@pharmatools/opengate 0.2.1 → 0.3.0

package/datasets/SCHEMA.md

@@ -7,7 +7,7 @@ One JSON file per case in `datasets/cases/` (files starting with `_` are ignored
 | `id` | yes | all | Unique slug. |
 | `title` / `notes` | no | — | Human context / provenance. |
 | `manuscript` | yes | claim-extraction | The pasted section, with citation markers exactly as authored. |
-| `goldClaims[]` | yes | citation-detection, claim-extraction | The verifiable claims a reviewer should extract. Each has `originalText` (with markers), `text` (clean), and `citations` (number array). |
+| `goldClaims[]` | yes | citation-detection, claim-extraction | The verifiable claims a reviewer should extract. Each has `originalText` (with markers), `text` (clean), and `citations` — an array of numbers and/or author-year string keys (e.g. "Smith 2020", "Meyer 2020a"). Numeric [N] markers are stripped from `text`; author-year mentions are grammatical prose and are never stripped, so for pure author-year claims `text === originalText`. |
 | `goldNonClaims[]` | no | claim-extraction | Sentences that should **not** be extracted (background, aims, transitions). Drives precision/leakage. |
 | `references{}` | online only | verdict-accuracy | Map of citation-number → `{ name, text }`. |
 | `goldVerdicts[]` | online only | verdict-accuracy | `{ claimText, citation, verdict }` where `verdict` ∈ the six-point scale. Mark `_requires: "online"`. |

package/datasets/cases/case-authoryear-harvard.json

@@ -0,0 +1,56 @@
+{
+  "id": "case-authoryear-harvard",
+  "title": "Author-year (Harvard) citation style — thyroid eye disease",
+  "notes": "SYNTHETIC manuscript exercising pure author-year citations. Unlike numeric [N] markers, author-year mentions are grammatical prose and are never stripped: text === originalText.",
+  "manuscript": "Teprotumumab reduced proptosis by 2.82 mm versus placebo (Douglas et al., 2020). Smith and Jones (2019) reported durable responses at 52 weeks in an open-label extension. The safety profile was consistent with earlier findings (Kim, 2023). This study aimed to characterise long-term outcomes in routine practice.",
+  "goldClaims": [
+    {
+      "originalText": "Teprotumumab reduced proptosis by 2.82 mm versus placebo (Douglas et al., 2020).",
+      "text": "Teprotumumab reduced proptosis by 2.82 mm versus placebo (Douglas et al., 2020).",
+      "citations": [
+        "Douglas 2020"
+      ]
+    },
+    {
+      "originalText": "Smith and Jones (2019) reported durable responses at 52 weeks in an open-label extension.",
+      "text": "Smith and Jones (2019) reported durable responses at 52 weeks in an open-label extension.",
+      "citations": [
+        "Smith 2019"
+      ]
+    },
+    {
+      "originalText": "The safety profile was consistent with earlier findings (Kim, 2023).",
+      "text": "The safety profile was consistent with earlier findings (Kim, 2023).",
+      "citations": [
+        "Kim 2023"
+      ]
+    }
+  ],
+  "goldNonClaims": [
+    "This study aimed to characterise long-term outcomes in routine practice."
+  ],
+  "references": {
+    "Douglas 2020": {
+      "name": "douglas-2020-nejm.txt",
+      "text": "In this randomised trial of teprotumumab for thyroid eye disease, the least-squares mean reduction in proptosis was 2.82 mm greater with teprotumumab than with placebo at week 24."
+    },
+    "Smith 2019": {
+      "name": "smith-2019-extension.txt",
+      "text": "In the 52-week open-label extension, proptosis responses were maintained in the majority of participants, indicating durable treatment effect."
+    }
+  },
+  "goldVerdicts": [
+    {
+      "_requires": "online",
+      "claimText": "Teprotumumab reduced proptosis by 2.82 mm versus placebo (Douglas et al., 2020).",
+      "citation": "Douglas 2020",
+      "verdict": "strong_support"
+    },
+    {
+      "_requires": "online",
+      "claimText": "Smith and Jones (2019) reported durable responses at 52 weeks in an open-label extension.",
+      "citation": "Smith 2019",
+      "verdict": "strong_support"
+    }
+  ]
+}

package/datasets/cases/case-authoryear-mixed.json

@@ -0,0 +1,48 @@
+{
+  "id": "case-authoryear-mixed",
+  "title": "Mixed numeric + author-year citations in one manuscript",
+  "notes": "SYNTHETIC. Exercises numeric superscript and author-year styles side by side, an apostrophe surname (O'Connor), and a 2020a/2020b suffix pair that must stay distinct.",
+  "manuscript": "Adalimumab improved ACR20 response rates versus placebo.1,2 O'Connor et al. (2021) confirmed the effect in a real-world cohort. Retention was higher with combination therapy (Meyer 2020a; Meyer 2020b). Baseline characteristics are summarised in Table 1.",
+  "goldClaims": [
+    {
+      "originalText": "Adalimumab improved ACR20 response rates versus placebo.1,2",
+      "text": "Adalimumab improved ACR20 response rates versus placebo.",
+      "citations": [
+        1,
+        2
+      ]
+    },
+    {
+      "originalText": "O'Connor et al. (2021) confirmed the effect in a real-world cohort.",
+      "text": "O'Connor et al. (2021) confirmed the effect in a real-world cohort.",
+      "citations": [
+        "O'Connor 2021"
+      ]
+    },
+    {
+      "originalText": "Retention was higher with combination therapy (Meyer 2020a; Meyer 2020b).",
+      "text": "Retention was higher with combination therapy (Meyer 2020a; Meyer 2020b).",
+      "citations": [
+        "Meyer 2020a",
+        "Meyer 2020b"
+      ]
+    }
+  ],
+  "goldNonClaims": [
+    "Baseline characteristics are summarised in Table 1."
+  ],
+  "references": {
+    "O'Connor 2021": {
+      "name": "oconnor-2021-cohort.txt",
+      "text": "In this prospective real-world cohort of 412 patients with rheumatoid arthritis, adalimumab was associated with significantly improved ACR20 response rates compared with historical placebo controls."
+    }
+  },
+  "goldVerdicts": [
+    {
+      "_requires": "online",
+      "claimText": "O'Connor et al. (2021) confirmed the effect in a real-world cohort.",
+      "citation": "O'Connor 2021",
+      "verdict": "strong_support"
+    }
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pharmatools/opengate",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "type": "module",
   "description": "OpenGATE — Open Grounded AI Testing & Evaluation. An open-source framework for evaluating evidence-grounded AI systems.",
   "license": "MIT",

package/src/lib/citations.mjs

@@ -69,7 +69,9 @@ const ETAL = String.raw`(?:et al\.?|and colleagues|and coworkers)`;
 /** Detect author-year citations in raw text; returns sorted unique keys. */
 export function detectAuthorYear(text) {
   const found = new Set();
-  const add = (name, year) => found.add(`${name} ${year.slice(0, 4)}`);
+  // Year suffixes are kept: "Smith 2020a" and "Smith 2020b" are distinct
+  // references in author-year styles.
+  const add = (name, year) => found.add(`${name} ${year}`);
 
   // 1. Narrative with parenthetical year: "Smith et al. (2020)",
   //    "Smith and Jones (2019)", "Smith (2020)".