medsci-skills 5.26.0 → 5.26.2

package/skills/fulltext-retrieval/fetch_oa.py

@@ -12,12 +12,13 @@ Usage:
     python fetch_oa.py worklist.csv -o pdfs/ -e user@example.com --report pdfs/retrieval_report.json
 
 Worklist formats: plain DOI-per-line, or TSV/CSV/Markdown-table with a DOI
-column (optional PMID and Title columns). A Title column enables a best-effort
-title cross-check (via `pdftotext` if installed) that flags mislabeled PDFs.
+column (optional PMID, Title and FirstAuthor columns). A separate source-identity
+report compares first-page title and identifiers via `pdftotext` if installed.
 """
 
 import argparse
 import csv
+import hashlib
 import io
 import json
 import logging
@@ -26,6 +27,7 @@ import re
 import shutil
 import subprocess
 import time
+import unicodedata
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -34,7 +36,7 @@ from pathlib import Path
 
 MIN_PDF_BYTES = 10 * 1024
 USER_AGENT = "medsci-skills/1.0"
-REPORT_SCHEMA_VERSION = 1
+REPORT_SCHEMA_VERSION = 2
 TITLE_MATCH_THRESHOLD = 0.6
 RETRIEVED_STATUSES = ("oa", "pmc", "arxiv", "skip")
 
@@ -85,14 +87,26 @@ def existing_pdf_ok(path: Path) -> bool:
         return False
 
 
+def pdf_sha256(path: Path) -> str:
+    """Hash the PDF bytes without loading a potentially large file into memory."""
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for block in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
 # ============================================================
-# Title cross-check (pure, offline-testable)
+# Source-identity evidence (pure, offline-testable; advisory, not verification)
 # ============================================================
 
 def normalize_title(text: str) -> str:
-    """Lowercase, strip punctuation, collapse whitespace."""
-    text = (text or "").lower()
-    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    """Normalize Unicode, punctuation and PDF line-end hyphenation."""
+    text = unicodedata.normalize("NFKD", text or "").casefold()
+    text = "".join(c for c in text if not unicodedata.combining(c))
+    text = text.replace("\u00ad", "")
+    text = re.sub(r"(?<=\w)[-\u2010]\s*\n\s*(?=\w)", "", text)
+    text = "".join(c if c.isalnum() or c.isspace() else " " for c in text)
     return " ".join(text.split())
 
 
@@ -113,16 +127,124 @@ def classify_title_match(expected_title: str, extracted_text,
                          threshold: float = TITLE_MATCH_THRESHOLD) -> str:
     """Tri-state title check: 'match' | 'mismatch' | 'unavailable'.
 
-    'unavailable' when no expected title is supplied or no extracted text is
-    available (e.g. pdftotext absent). A low overlap is 'mismatch' — flagged,
-    never used to auto-reject a downloaded PDF.
+    A match requires the complete normalized title on up to six consecutive
+    front-matter lines. Token overlap can only mark ambiguous text unavailable;
+    it can never establish a match. Mismatch is advisory, never auto-rejection.
     """
-    if not expected_title or not extracted_text:
+    expected = normalize_title(expected_title)
+    front = first_page_front_matter(extracted_text)
+    if not expected or not front:
         return "unavailable"
-    return "match" if title_overlap(expected_title, extracted_text) >= threshold else "mismatch"
+    lines = [re.sub(r"^title\s*:\s*", "", ln.strip(), flags=re.I)
+             for ln in front.splitlines() if ln.strip()]
+    for start in range(len(lines)):
+        for length in range(1, min(6, len(lines) - start) + 1):
+            if normalize_title("\n".join(lines[start:start + length])) == expected:
+                return "match"
+    return "unavailable" if title_overlap(expected_title, front) >= threshold else "mismatch"
+
+
+_BODY_START_RE = re.compile(
+    r"^\s*(?:(?:\d+|[IVX]+)[.\s]+)?"
+    r"(?:abstract|summary|introduction|background|references|bibliography|literature cited)"
+    r"(?:\s*[:.\u2014\u2013-]|\s*$)", re.I,
+)
 
 
-def extract_pdf_text(path: Path, max_pages: int = 2) -> str | None:
+def first_page_front_matter(extracted_text: str | None) -> str:
+    """Bound evidence to page 1 before a body/reference heading, at most 40 lines.
+
+    This is a conservative text-layout heuristic, not a PDF title-zone parser.
+    Cover sheets, unusual layouts and unrecognized headings need human review.
+    """
+    if not extracted_text:
+        return ""
+    lines = []
+    for line in extracted_text.split("\f", 1)[0][:4000].splitlines()[:40]:
+        if _BODY_START_RE.match(line):
+            break
+        lines.append(line)
+    return "\n".join(lines).strip()
+
+
+def normalize_identifier(value: str) -> str:
+    """Normalize DOI URL/prefix and equivalent arXiv DOI/id spellings."""
+    value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "",
+                   (value or "").strip(), flags=re.I).rstrip(".,;")
+    aid = arxiv_id_from_doi(value)
+    return (f"10.48550/arxiv.{aid}" if aid else value).casefold()
+
+
+def front_matter_identifiers(front: str) -> list[str]:
+    """Find DOI/arXiv evidence only in the bounded front matter, never PDF metadata."""
+    values = re.findall(r"\b10\.\d{4,9}/[^\s<>\"\[\]]+", front, re.I)
+    # Drop a citation's closing parenthesis, but retain balanced DOI suffixes.
+    cleaned = []
+    for value in values:
+        value = value.rstrip(".,;")
+        while value.endswith(")") and value.count(")") > value.count("("):
+            value = value[:-1]
+        cleaned.append(normalize_identifier(value))
+    for aid in re.findall(r"\barxiv\s*:\s*((?:\d{4}\.\d{4,5}|[a-z-]+/\d{7})(?:v\d+)?)\b",
+                          front, re.I):
+        cleaned.append(normalize_identifier("arXiv:" + aid))
+    return sorted(set(cleaned))
+
+
+def identifier_matches(expected: str, observed: str) -> bool:
+    """Unversioned arXiv requests accept a version; explicit versions must agree."""
+    if expected == observed:
+        return True
+    aid = arxiv_id_from_doi(expected)
+    return bool(aid and not re.search(r"v\d+$", aid, re.I)
+                and re.sub(r"v\d+$", "", observed, flags=re.I) == expected)
+
+
+def assess_source_identity(record: dict, extracted_text: str | None,
+                           threshold: float = TITLE_MATCH_THRESHOLD) -> dict:
+    """Report corroboration and uncertainty; never certify a source or delete it.
+
+    'consistent' needs a complete title and only compatible identifiers in the
+    same bounded first-page area. A supplied first-author name must also occur.
+    Matching titles with a different DOI may be another version: unresolved.
+    """
+    front = first_page_front_matter(extracted_text)
+    title_match = classify_title_match(record.get("title", ""), extracted_text, threshold)
+    expected = normalize_identifier(record["doi"])
+    observed = front_matter_identifiers(front)
+    doi_match = ("unavailable" if not observed else
+                 "match" if any(identifier_matches(expected, v) for v in observed)
+                 else "mismatch")
+    author = normalize_title(record.get("first_author", ""))
+    author_text = normalize_title(front).replace(normalize_title(record.get("title", "")), "", 1)
+    author_match = ("unavailable" if not author or not front else
+                    "match" if f" {author} " in f" {author_text} " else "mismatch")
+    status, reason = "unresolved", "title_not_matched"
+    if not extracted_text or not extracted_text.strip():
+        status, reason = "unavailable", "text_unavailable"
+    elif not front:
+        reason = "front_matter_unavailable"
+    elif not normalize_title(record.get("title", "")):
+        reason = "expected_title_missing"
+    elif title_match == "match":
+        if doi_match == "unavailable":
+            reason = "identifier_not_found"
+        elif doi_match == "mismatch":
+            reason = "title_matches_other_identifier"
+        elif not all(identifier_matches(expected, v) for v in observed):
+            reason = "multiple_identifiers"
+        elif author_match == "mismatch":
+            reason = "first_author_not_found"
+        else:
+            status, reason = "consistent", "title_and_identifier_agree"
+    elif title_match == "mismatch" and doi_match == "mismatch":
+        status, reason = "conflict", "title_and_identifier_differ"
+    return {"status": status, "reason": reason, "text_scope": "first_page_front_matter",
+            "title_match": title_match, "doi_match": doi_match,
+            "observed_identifiers": observed, "first_author_match": author_match}
+
+
+def extract_pdf_text(path: Path, max_pages: int = 1) -> str | None:
     """Best-effort first-page text via `pdftotext` (poppler). None if unavailable."""
     if not shutil.which("pdftotext"):
         return None
@@ -432,13 +554,16 @@ def process_doi(doi: str, outdir: Path, email: str,
 
 def build_report(records: list[dict], results: dict[str, tuple[str, str]],
                  outdir: Path, extracted_text_by_doi: dict[str, str] | None = None,
-                 threshold: float = TITLE_MATCH_THRESHOLD) -> dict:
+                 threshold: float = TITLE_MATCH_THRESHOLD, *,
+                 extracted_sha256_by_doi: dict[str, str] | None = None) -> dict:
     """Assemble a deterministic retrieval report (no network, no I/O writes).
 
-    records: list of {"doi", "pmid", "title"}.
+    records: list of {"doi", "pmid", "title"}, optionally "first_author".
     results: doi -> (status, source) as returned by process_doi.
     outdir:  directory where PDFs were written (used for file/size lookup).
     extracted_text_by_doi: optional doi -> first-page text for title cross-check.
+    extracted_sha256_by_doi: hashes captured before extraction by the CLI. When
+        supplied, missing/different hashes invalidate that text's assessment.
     """
     extracted_text_by_doi = extracted_text_by_doi or {}
     items = []
@@ -448,20 +573,27 @@ def build_report(records: list[dict], results: dict[str, tuple[str, str]],
         path = outdir / f"{safe_doi_name(doi)}.pdf"
         have_file = status in RETRIEVED_STATUSES and path.exists()
         size = path.stat().st_size if have_file else 0
-        if have_file:
-            title_match = classify_title_match(
-                rec.get("title", ""), extracted_text_by_doi.get(doi), threshold)
-        else:
-            title_match = "unavailable"
+        digest = pdf_sha256(path) if have_file else ""
+        text = extracted_text_by_doi.get(doi) if have_file else None
+        changed = bool(text and extracted_sha256_by_doi is not None
+                       and extracted_sha256_by_doi.get(doi) != digest)
+        identity = assess_source_identity(rec, None if changed else text, threshold)
+        if not have_file:
+            identity["reason"] = "pdf_not_available"
+        elif changed:
+            identity["reason"] = "pdf_changed_during_assessment"
         items.append({
             "doi": doi,
             "pmid": rec.get("pmid", ""),
             "title": rec.get("title", ""),
+            "first_author": rec.get("first_author", ""),
             "status": status,
             "source": source,
             "file": path.name if have_file else "",
             "size_bytes": size,
-            "title_match": title_match,
+            "file_sha256": digest,
+            "title_match": identity["title_match"],
+            "source_identity": identity,
         })
 
     retrieved = [i for i in items if i["status"] in RETRIEVED_STATUSES]
@@ -474,6 +606,10 @@ def build_report(records: list[dict], results: dict[str, tuple[str, str]],
             "retrieved": len(retrieved),
             "not_retrieved": len(not_retrieved),
             "title_mismatch": sum(1 for i in items if i["title_match"] == "mismatch"),
+            "source_identity": {
+                state: sum(i["source_identity"]["status"] == state for i in items)
+                for state in ("consistent", "conflict", "unresolved", "unavailable")
+            },
         },
         "items": items,
     }
@@ -489,7 +625,9 @@ def _records_from_dictrows(rows) -> list[dict]:
         rec = {"doi": "", "pmid": "", "title": ""}
         for k, v in row.items():
             nk = _norm_key(k)
-            if nk in rec:
+            if nk in ("firstauthor", "first_author", "first author"):
+                rec["first_author"] = (v or "").strip()
+            elif nk in rec:
                 rec[nk] = (v or "").strip()
         if rec["doi"]:
             records.append(rec)
@@ -514,11 +652,15 @@ def _records_from_markdown(lines: list[str]) -> list[dict]:
         row = dict(zip(header, c))
         doi = (row.get("doi") or "").strip()
         if doi:
-            records.append({
+            rec = {
                 "doi": doi,
                 "pmid": (row.get("pmid") or "").strip(),
                 "title": (row.get("title") or "").strip(),
-            })
+            }
+            for key in ("firstauthor", "first_author", "first author"):
+                if key in row:
+                    rec["first_author"] = (row[key] or "").strip()
+            records.append(rec)
     return records
 
 
@@ -526,8 +668,8 @@ def read_doi_file(path: Path) -> list[dict]:
     """Read a worklist of DOIs.
 
     Supports: plain DOI-per-line; TSV/CSV with a DOI header (optional PMID,
-    Title columns); and a Markdown pipe table with a DOI column. Each record is
-    {"doi", "pmid", "title"}.
+    Title and FirstAuthor columns); and a Markdown pipe table with a DOI column.
+    Each record is {"doi", "pmid", "title"}, optionally "first_author".
     """
     text = Path(path).read_text(encoding="utf-8")
     lines = text.splitlines()
@@ -560,7 +702,7 @@ def main():
         description="Batch download open-access PDFs by DOI.")
     parser.add_argument("input", type=Path,
                         help="Worklist: DOIs (one per line) or TSV/CSV/Markdown "
-                             "with a DOI column (optional PMID, Title)")
+                             "with a DOI column (optional PMID, Title, FirstAuthor)")
     parser.add_argument("-o", "--output", type=Path, default=Path("pdfs"),
                         help="Output directory (default: pdfs/)")
     parser.add_argument("-e", "--email", default=os.environ.get("MEDSCI_CONTACT_EMAIL"),
@@ -599,29 +741,29 @@ def main():
         results[doi] = (status, source)
         stats[status] += 1
 
-        labels = {"arxiv": "OK (arXiv)", "oa": "OK (OA)", "pmc": "OK (PMC)",
-                  "fail": "FAIL", "skip": "SKIP"}
+        labels = {"arxiv": "DOWNLOADED (arXiv)", "oa": "DOWNLOADED (OA)",
+                  "pmc": "DOWNLOADED (PMC)", "fail": "FAIL", "skip": "EXISTS"}
         print(labels[status])
         time.sleep(0.5)
 
-    # Best-effort title cross-check on successful downloads (needs pdftotext).
+    # Identity evidence is separate from retrieval, including DOI-only worklists.
     extracted: dict[str, str] = {}
-    have_titles = any(r.get("title") for r in records)
-    if have_titles and shutil.which("pdftotext"):
+    extracted_hashes: dict[str, str] = {}
+    if shutil.which("pdftotext"):
         for rec in records:
             doi = rec["doi"]
-            if not rec.get("title"):
-                continue
             status, _ = results.get(doi, ("fail", ""))
             if status not in RETRIEVED_STATUSES:
                 continue
             path = args.output / f"{safe_doi_name(doi)}.pdf"
             if path.exists():
+                extracted_hashes[doi] = pdf_sha256(path)
                 text = extract_pdf_text(path)
                 if text:
                     extracted[doi] = text
 
-    report = build_report(records, results, args.output, extracted)
+    report = build_report(records, results, args.output, extracted,
+                          extracted_sha256_by_doi=extracted_hashes)
     report_path.parent.mkdir(parents=True, exist_ok=True)
     report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
 
@@ -634,10 +776,14 @@ def main():
     total = stats["arxiv"] + stats["oa"] + stats["pmc"] + stats["fail"]
     if total > 0:
         pct = (stats["arxiv"] + stats["oa"] + stats["pmc"]) / total * 100
-        print(f"  Success: {pct:.0f}%")
+        print(f"  Download success: {pct:.0f}% (excludes existing files; not identity verification)")
     mismatches = report["counts"]["title_mismatch"]
     if mismatches:
         print(f"  Title mismatches flagged: {mismatches} (see report)")
+    identity_counts = report["counts"]["source_identity"]
+    print("  Source identity (advisory): " + ", ".join(
+        f"{state}={count}" for state, count in identity_counts.items()))
+    print("  Review source_identity and file_sha256 before using a PDF as evidence.")
     print(f"  Report:  {report_path}")
 
     # Write failed DOIs for manual retrieval

package/skills/fulltext-retrieval/fetch_oa_report_challenge/expected/projection.json

@@ -3,7 +3,8 @@
     "total": 4,
     "retrieved": 3,
     "not_retrieved": 1,
-    "title_mismatch": 1
+    "title_mismatch": 1,
+    "source_identity": {"consistent": 1, "conflict": 1, "unresolved": 0, "unavailable": 2}
   },
   "items": [
     {"doi": "10.1111/valid.match", "status": "oa", "source": "unpaywall", "title_match": "match"},

package/skills/fulltext-retrieval/fetch_oa_report_challenge/extracted_text.json

@@ -1,4 +1,4 @@
 {
-  "10.1111/valid.match": "Deep Learning for Pulmonary Nodule Detection on Chest CT\nAbstract: we present a convolutional neural network trained to detect pulmonary nodules.",
-  "10.2222/label.mismatch": "Genome-wide association study of type 2 diabetes in an Asian cohort\nIntroduction: we genotyped participants to identify susceptibility loci."
+  "10.1111/valid.match": "Deep Learning for Pulmonary Nodule Detection on Chest CT\nDOI: 10.1111/valid.match\nAbstract: we present a convolutional neural network trained to detect pulmonary nodules.",
+  "10.2222/label.mismatch": "Genome-wide association study of type 2 diabetes in an Asian cohort\nDOI: 10.1000/synthetic.other\nIntroduction: we genotyped participants to identify susceptibility loci."
 }

package/skills/fulltext-retrieval/skill.yml

@@ -8,12 +8,12 @@ when_to_use: "Batch-download open-access full-text PDFs from a DOI list; optiona
 when_NOT_to_use: "Finding or verifying citations (use search-lit / verify-refs). Retrieving paywalled or non-open-access content."
 
 inputs:
-  - path: "DOI worklist (.txt one-per-line, or .tsv/.csv/.md with a DOI column; optional PMID, Title)"
+  - path: "DOI worklist (.txt one-per-line, or .tsv/.csv/.md with a DOI column; optional PMID, Title, FirstAuthor)"
     schema: csv
     required: true
 outputs:
   - path: "downloaded open-access PDFs (pdfs/)"
-  - path: "pdfs/retrieval_report.json (per-DOI status/source/title_match tri-state)"
+  - path: "pdfs/retrieval_report.json (schema 2: retrieval status, source_identity evidence, file_sha256)"
   - path: "optional PDF-to-Markdown conversions"
   - path: "references/find_available_pdf.js (user-run Zotero 'Find Available PDF' batch snippet)"
 
@@ -38,10 +38,12 @@ safety_boundaries:
 known_limitations:
   - "Only open-access content is retrievable; non-OA DOIs fail by design rather than fetching from unauthorized sources."
   - "Higher-yield in-library retrieval (find_available_pdf.js) is user-initiated inside Zotero and uses the user's own proxy/OpenURL config; it is not reproducible CI evidence."
-  - "Title cross-check is best-effort: it needs a Title column plus pdftotext (poppler); otherwise title_match is 'unavailable'. A mismatch is flagged, never auto-rejected."
+  - "Retrieval success is not source verification. Source identity uses bounded first-page title/DOI evidence plus optional FirstAuthor; ambiguous or unavailable evidence stays visible, and no PDF is auto-rejected."
+  - "Identity is advisory: unusual layouts, cover sheets, changed titles, and preprint/publication DOI differences need review. Downstream consumers must preserve source_identity and file_sha256; old reports are unassessed."
   - "PDF-to-Markdown conversion requires the optional pymupdf4llm dependency (AGPL-3.0 or commercial license)."
 validation_commands:
-  - "bash fetch_oa_report_challenge/verify.sh   # offline report-builder + title tri-state (CI-wired)"
+  - "bash fetch_oa_report_challenge/verify.sh   # offline report projection (CI-wired)"
+  - "python3 tests/test_source_identity.py   # synthetic identity cases + real PDF CLI round trips with Poppler (CI-wired)"
   - "python fetch_oa.py dois.txt -o pdfs/ -e <email> --report pdfs/retrieval_report.json --verbose   # per-DOI source trace"
   - "verify each output begins with %PDF- and is at least 10 KB"
 evidence_surface: bundled_script

package/skills/fulltext-retrieval/tests/test_source_identity.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Synthetic source-identity regressions, including real PDF/CLI round trips.
+
+No source papers, private records, network requests or API credentials are used.
+The PDF integration cases use Poppler when available; pure cases are stdlib-only.
+"""
+import contextlib
+import hashlib
+import importlib.util
+import io
+import json
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+ENGINE = Path(__file__).resolve().parents[1] / "fetch_oa.py"
+SPEC = importlib.util.spec_from_file_location("fetch_oa_identity_test", ENGINE)
+m = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(m)
+
+TITLE = "Deep Learning for Pulmonary Nodule Detection on Chest CT"
+DOI = "10.1000/synthetic.nodules"
+OTHER = "10.1000/synthetic.crops"
+RECORD = {"doi": DOI, "title": TITLE, "pmid": ""}
+GOOD = f"{TITLE}\nAlex Example\nhttps://doi.org/{DOI}\nAbstract: Synthetic example."
+
+
+def write_pdf(path: Path, lines: list[str]) -> None:
+    """Write an actual single-page Helvetica PDF with a correct cross-reference table."""
+    commands = ["BT /F1 11 Tf 36 750 Td 16 TL"]
+    for line in lines:
+        line = line.replace("\\", "\\\\").replace("(", "\\(").replace(")", "\\)")
+        commands.append(f"({line}) Tj T*")
+    commands.append("ET")
+    stream = "\n".join(commands).encode("ascii")
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
+        b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+        f"<< /Length {len(stream)} >>\nstream\n".encode() + stream + b"\nendstream",
+    ]
+    data = bytearray(b"%PDF-1.4\n%" + b"synthetic-padding " * 700 + b"\n")
+    offsets = [0]
+    for number, obj in enumerate(objects, 1):
+        offsets.append(len(data))
+        data.extend(f"{number} 0 obj\n".encode() + obj + b"\nendobj\n")
+    xref = len(data)
+    data.extend(f"xref\n0 {len(offsets)}\n0000000000 65535 f \n".encode())
+    for offset in offsets[1:]:
+        data.extend(f"{offset:010d} 00000 n \n".encode())
+    data.extend(f"trailer\n<< /Size {len(offsets)} /Root 1 0 R >>\n"
+                f"startxref\n{xref}\n%%EOF\n".encode())
+    path.write_bytes(data)
+
+
+class SourceIdentityTests(unittest.TestCase):
+    def test_reference_and_body_mentions_do_not_establish_identity(self):
+        for boundary in ("References", "REFERENCES:", "1. Introduction", "Abstract:", "\f"):
+            text = f"Genome-wide analysis of crop yield\nDOI: {OTHER}\n{boundary}\n{TITLE}\n{DOI}"
+            with self.subTest(boundary=boundary):
+                result = m.assess_source_identity(RECORD, text)
+                self.assertEqual(result["title_match"], "mismatch")
+                self.assertEqual(result["status"], "conflict")
+                self.assertEqual(result["observed_identifiers"], [OTHER])
+
+    def test_scattered_title_words_and_reference_entries_are_not_titles(self):
+        texts = [
+            "Chest CT detection study\nNodule pulmonary learning on deep networks",
+            f"Unrelated article\nExample A. {TITLE}. doi:{DOI}",
+            f"Unrelated article\nWe discuss {TITLE} in this review.\n{DOI}",
+        ]
+        for text in texts:
+            with self.subTest(text=text):
+                # Even the former very permissive threshold cannot produce a match.
+                self.assertNotEqual(m.classify_title_match(TITLE, text, 0.1), "match")
+                self.assertNotEqual(m.assess_source_identity(RECORD, text)["status"], "consistent")
+
+    def test_wrapped_title_unicode_punctuation_and_doi_url(self):
+        text = ("Research Article\nDEEP LEARNING FOR PULMO-\n"
+                "NARY NODULE DETECTION ON CHEST CT\nAlex Example\n"
+                f"doi: https://doi.org/{DOI.upper()}\nAbstract: example")
+        result = m.assess_source_identity({**RECORD, "first_author": "Example"}, text)
+        self.assertEqual(result["status"], "consistent")
+        self.assertEqual(result["first_author_match"], "match")
+        self.assertEqual(m.classify_title_match("Café — α imaging", "CAFE: α imaging"), "match")
+
+    def test_title_alone_and_doi_alone_need_review(self):
+        self.assertEqual(m.assess_source_identity(RECORD, TITLE)["reason"], "identifier_not_found")
+        result = m.assess_source_identity({"doi": DOI}, GOOD)
+        self.assertEqual(result["status"], "unresolved")
+        self.assertEqual(result["doi_match"], "match")
+        self.assertEqual(result["reason"], "expected_title_missing")
+
+    def test_related_versions_and_multiple_identifiers_need_review(self):
+        result = m.assess_source_identity(RECORD, GOOD.replace(DOI, "10.48550/arXiv.2401.01234"))
+        self.assertEqual(result["status"], "unresolved")
+        self.assertEqual(result["reason"], "title_matches_other_identifier")
+        result = m.assess_source_identity(RECORD, GOOD.replace("Abstract:", f"{OTHER}\nAbstract:"))
+        self.assertEqual(result["status"], "unresolved")
+        self.assertEqual(result["reason"], "multiple_identifiers")
+
+    def test_arxiv_requested_version_must_not_be_silently_replaced(self):
+        text = f"{TITLE}\narXiv:2401.01234v2 [cs.CV]\nAbstract: example"
+        for doi in ("10.48550/arXiv.2401.01234", "arXiv:2401.01234v2"):
+            self.assertEqual(m.assess_source_identity({**RECORD, "doi": doi}, text)["status"],
+                             "consistent")
+        wrong = m.assess_source_identity({**RECORD, "doi": "arXiv:2401.01234v1"}, text)
+        self.assertEqual(wrong["status"], "unresolved")
+        old = f"{TITLE}\narXiv:hep-th/9901001v2\nAbstract: example"
+        self.assertEqual(m.assess_source_identity({**RECORD, "doi": "arXiv:hep-th/9901001"}, old)
+                         ["status"], "consistent")
+
+    def test_author_disagreement_cannot_be_hidden_by_title_and_doi(self):
+        result = m.assess_source_identity({**RECORD, "first_author": "Someone Else"}, GOOD)
+        self.assertEqual(result["status"], "unresolved")
+        self.assertEqual(result["reason"], "first_author_not_found")
+
+    def test_missing_extraction_and_unusable_front_matter_are_visible(self):
+        for text in (None, "", "   "):
+            self.assertEqual(m.assess_source_identity(RECORD, text)["status"], "unavailable")
+        self.assertEqual(m.assess_source_identity(RECORD, "Abstract:\n" + GOOD)["reason"],
+                         "front_matter_unavailable")
+        self.assertEqual(m.classify_title_match("!!!", GOOD), "unavailable")
+        with patch.object(m.shutil, "which", return_value=None):
+            self.assertIsNone(m.extract_pdf_text(Path("not-read.pdf")))
+        with patch.object(m.shutil, "which", return_value="pdftotext"), \
+                patch.object(m.subprocess, "run", side_effect=subprocess.TimeoutExpired("pdftotext", 20)):
+            self.assertIsNone(m.extract_pdf_text(Path("not-read.pdf")))
+
+    def test_doi_suffix_punctuation(self):
+        self.assertEqual(m.front_matter_identifiers("doi:10.1000/example(abc)."),
+                         ["10.1000/example(abc)"])
+        self.assertEqual(m.front_matter_identifiers("(https://doi.org/10.1000/example)."),
+                         ["10.1000/example"])
+
+    def test_optional_author_survives_each_worklist_format(self):
+        inputs = {
+            "tsv": f"DOI\tTitle\tFirstAuthor\n{DOI}\t{TITLE}\tExample\n",
+            "csv": f"DOI,Title,First Author\n{DOI},{TITLE},Example\n",
+            "md": f"| DOI | Title | First_Author |\n|---|---|---|\n|{DOI}|{TITLE}|Example|\n",
+        }
+        with tempfile.TemporaryDirectory() as tmp:
+            for suffix, text in inputs.items():
+                path = Path(tmp) / f"worklist.{suffix}"
+                path.write_text(text)
+                self.assertEqual(m.read_doi_file(path)[0]["first_author"], "Example")
+            path = Path(tmp) / "dois.txt"
+            path.write_text(DOI + "\n")
+            self.assertEqual(m.read_doi_file(path), [{"doi": DOI, "pmid": "", "title": ""}])
+
+    def test_report_preserves_retrieval_counts_and_binds_identity_to_file(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            pdf = root / (m.safe_doi_name(DOI) + ".pdf")
+            write_pdf(pdf, GOOD.splitlines())
+            report = m.build_report([RECORD], {DOI: ("skip", "existing")}, root, {DOI: GOOD})
+            self.assertEqual(report["schema_version"], 2)
+            self.assertEqual(report["counts"]["retrieved"], 1)
+            self.assertEqual(report["counts"]["source_identity"]["consistent"], 1)
+            item = report["items"][0]
+            self.assertEqual(item["file_sha256"], hashlib.sha256(pdf.read_bytes()).hexdigest())
+            before = item["file_sha256"]
+            write_pdf(pdf, ["Unrelated synthetic replacement", OTHER])
+            changed = m.build_report([RECORD], {DOI: ("skip", "existing")}, root, {DOI: GOOD},
+                                     extracted_sha256_by_doi={DOI: before})
+            self.assertEqual(changed["items"][0]["source_identity"]["status"], "unavailable")
+            self.assertEqual(changed["items"][0]["source_identity"]["reason"],
+                             "pdf_changed_during_assessment")
+            self.assertNotEqual(changed["items"][0]["file_sha256"], before)
+            pdf.unlink()
+            missing = m.build_report([RECORD], {DOI: ("oa", "unpaywall")}, root, {DOI: GOOD})
+            self.assertEqual(missing["counts"]["retrieved"], 1)  # resolver-result semantics unchanged
+            self.assertEqual(missing["items"][0]["source_identity"]["reason"], "pdf_not_available")
+            self.assertEqual(missing["items"][0]["file_sha256"], "")
+
+
+@unittest.skipUnless(shutil.which("pdftotext"), "Poppler needed for actual PDF/CLI round trips")
+class PDFIntegrationTests(unittest.TestCase):
+    def test_real_pdfs_through_cli_distinguish_download_from_identity(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            pdfs = root / "pdfs"
+            pdfs.mkdir()
+            requested_other = "10.1000/synthetic.other-request"
+            for doi, lines in [
+                (DOI, GOOD.splitlines()),
+                (requested_other, ["Genome-wide analysis of crop yield", f"DOI: {OTHER}",
+                                   "Abstract: unrelated synthetic example.", "References", TITLE,
+                                   requested_other]),
+            ]:
+                write_pdf(pdfs / (m.safe_doi_name(doi) + ".pdf"), lines)
+            worklist = root / "worklist.tsv"
+            worklist.write_text(f"DOI\tTitle\n{DOI}\t{TITLE}\n{requested_other}\t{TITLE}\n")
+            output = io.StringIO()
+            with patch.object(sys, "argv", [str(ENGINE), str(worklist), "-o", str(pdfs),
+                                            "-e", "test@example.com"]), \
+                    patch.object(m.time, "sleep"), \
+                    patch.object(m.urllib.request, "urlopen", side_effect=AssertionError("network forbidden")), \
+                    contextlib.redirect_stdout(output):
+                m.main()
+            report = json.loads((pdfs / "retrieval_report.json").read_text())
+            self.assertEqual(report["counts"]["retrieved"], 2)
+            self.assertEqual(report["counts"]["source_identity"],
+                             {"consistent": 1, "conflict": 1, "unresolved": 0, "unavailable": 0})
+            self.assertIn("Source identity (advisory): consistent=1, conflict=1", output.getvalue())
+            self.assertTrue((pdfs / (m.safe_doi_name(requested_other) + ".pdf")).is_file())
+
+    def test_doi_only_cli_still_extracts_identifiers_and_reports_missing_title(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            write_pdf(root / (m.safe_doi_name(DOI) + ".pdf"), GOOD.splitlines())
+            worklist = root / "dois.txt"
+            worklist.write_text(DOI + "\n")
+            with patch.object(sys, "argv", [str(ENGINE), str(worklist), "-o", str(root),
+                                            "-e", "test@example.com"]), \
+                    patch.object(m.time, "sleep"), \
+                    patch.object(m.urllib.request, "urlopen", side_effect=AssertionError("network forbidden")), \
+                    contextlib.redirect_stdout(io.StringIO()):
+                m.main()
+            item = json.loads((root / "retrieval_report.json").read_text())["items"][0]
+            self.assertEqual(item["source_identity"]["observed_identifiers"], [DOI])
+            self.assertEqual(item["source_identity"]["status"], "unresolved")
+            self.assertEqual(item["source_identity"]["reason"], "expected_title_missing")
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)