opencode-bioresearcher 1.9.0 → 1.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,13 @@ Contract:
10
10
  - Fill-missing-only merging: never overwrite a non-null stored value.
11
11
  - Loud gaps: missing fields render as [MISSING field: ...]; unknown bib keys as
12
12
  [MISSING record <key>] with a non-zero exit. Never fabricate.
13
+ - render is the single numbering authority: drafts cite [@key] markers; render
14
+ assigns numbers by first appearance, rewrites the markers, and appends the
15
+ References section from the ledger. Any unresolved key or [MISSING ...] entry
16
+ fails the render (exit 1) WITHOUT writing the output file.
17
+ - check validates a ledger end-to-end (exit 1 on quarantined lines; with
18
+ --markers also on unresolved/mixed [@key] markers in the given markdown
19
+ files - exit 1 iff anything this invocation validated failed).
13
20
  - Verb banners: every subcommand prints "[evidence-ledger] <verb>: ..." so
14
21
  test graders can anchor on deterministic stdout.
15
22
 
@@ -39,7 +46,7 @@ LEDGER_TYPES = {
39
46
  "drug", "disease", "dataset", "web", "other",
40
47
  }
41
48
  KEY_NAMESPACES = (
42
- "pmid", "doi", "pmcid", "nct", "patent", "geo", "sra", "gb",
49
+ "pmid", "doi", "pmcid", "nct", "patent", "geo", "sra", "gb", "pdb",
43
50
  "gene", "clinvar", "chembl", "chebi", "unii",
44
51
  "mondo", "doid", "omim", "efo", "url", "title",
45
52
  )
@@ -63,6 +70,8 @@ ID_ALIASES = {
63
70
  "geo_id": "geo",
64
71
  "sra_id": "sra",
65
72
  "gb_acc": "genbank",
73
+ "pdb_id": "pdb", # biomcp pdb
74
+ "rcsb_id": "pdb",
66
75
  "disease_id": None, # context-dependent: prefix-sniffed below
67
76
  }
68
77
 
@@ -74,6 +83,7 @@ FOLD_FIELDS = {
74
83
  "patent": ["assignee", "status"],
75
84
  "gene": ["symbol", "full_name"],
76
85
  "variant": ["gene", "protein_change", "significance"],
86
+ "dataset": ["method", "experimental_method", "resolution"],
77
87
  }
78
88
 
79
89
 
@@ -105,7 +115,7 @@ def _canonicalize_id_value(key: str, value: str) -> str:
105
115
  if not value.isdigit():
106
116
  raise ValueError(f"pmid must be digits, got {value!r}")
107
117
  return value.lstrip("0") or "0"
108
- if key in ("pmcid", "nct", "patent"):
118
+ if key in ("pmcid", "nct", "patent", "pdb"):
109
119
  return value.upper()
110
120
  return value
111
121
 
@@ -161,6 +171,14 @@ def normalize_record(raw, require_provenance_aspect=None) -> dict:
161
171
  title = raw.get("title")
162
172
  if title is not None:
163
173
  title = str(title).strip() or None
174
+ if title is None and rtype != "article":
175
+ # biomcp record shapes often carry `name` instead of `title` (drugs,
176
+ # genes, diseases): fold it fill-only so bibliographies never render
177
+ # [MISSING field: title] for schema-shaped records. Articles keep
178
+ # their hard id requirement (a name-only article must not verify-gate).
179
+ name = raw.get("name")
180
+ if isinstance(name, str) and name.strip():
181
+ title = name.strip()
164
182
  if not title and not ids:
165
183
  raise ValueError("record needs a title or at least one id")
166
184
  if raw.get("meta") is not None and not isinstance(raw.get("meta"), dict):
@@ -217,6 +235,11 @@ def _title_key(rtype: str, title) -> str:
217
235
 
218
236
 
219
237
  def derive_dataset_key(ids: dict, title) -> str:
238
+ if ids.get("pdb"):
239
+ return f"pdb:{ids['pdb'].upper()}"
240
+ for k, v in ids.items():
241
+ if (k or "").lower() in ("pdb", "pdb_id"):
242
+ return f"pdb:{str(v).upper()}"
220
243
  for v in ids.values():
221
244
  v = str(v)
222
245
  if v.upper().startswith("GSE"):
@@ -230,7 +253,7 @@ def derive_dataset_key(ids: dict, title) -> str:
230
253
  for k, v in ids.items():
231
254
  if (k or "").lower() in ("genbank", "gb", "accession"):
232
255
  return f"gb:{v}"
233
- raise ValueError("dataset record needs a geo (GSE/GDS), sra (SRR/SRP), or genbank accession id")
256
+ raise ValueError("dataset record needs a pdb, geo (GSE/GDS), sra (SRR/SRP), or genbank accession id")
234
257
 
235
258
 
236
259
  def derive_web_key(ids: dict, title) -> str:
@@ -441,16 +464,46 @@ def append_quarantine(out_path: Path, entries: list) -> Path:
441
464
  # Subcommands
442
465
  # ---------------------------------------------------------------------------
443
466
 
467
+ def _parse_payload(text: str) -> list:
468
+ """Parse JSON array, single JSON object, or newline-delimited JSON (.jsonl).
469
+ Strips optional markdown code fences, leading UTF-8 BOM, and blank lines."""
470
+ text = (text or "").lstrip("\ufeff")
471
+ lines = text.strip().splitlines()
472
+ if lines and lines[0].strip().startswith("```"):
473
+ lines = lines[1:]
474
+ if lines and lines[-1].strip().startswith("```"):
475
+ lines = lines[:-1]
476
+ cleaned = "\n".join(lines).strip()
477
+ if not cleaned:
478
+ raise ValueError("empty record input payload")
479
+ try:
480
+ payload = json.loads(cleaned)
481
+ return payload if isinstance(payload, list) else [payload]
482
+ except json.JSONDecodeError:
483
+ records = []
484
+ for line_no, line in enumerate(cleaned.splitlines(), start=1):
485
+ line = line.strip()
486
+ if not line:
487
+ continue
488
+ try:
489
+ item = json.loads(line)
490
+ except json.JSONDecodeError as e:
491
+ raise ValueError(f"malformed JSON at line {line_no}: {e}") from e
492
+ if isinstance(item, list):
493
+ records.extend(item)
494
+ else:
495
+ records.append(item)
496
+ return records
497
+
498
+
444
499
  def _parse_incoming(args) -> list:
445
500
  if args.stdin:
446
- payload = json.loads(sys.stdin.read())
447
- return payload if isinstance(payload, list) else [payload]
501
+ return _parse_payload(sys.stdin.read())
448
502
  if getattr(args, "record", None) is None:
449
503
  raise ValueError("provide a record JSON, @file, or --stdin")
450
504
  if args.record.startswith("@"):
451
- payload = json.loads(Path(args.record[1:]).read_text(encoding="utf-8-sig"))
452
- return payload if isinstance(payload, list) else [payload]
453
- return [json.loads(args.record)]
505
+ return _parse_payload(Path(args.record[1:]).read_text(encoding="utf-8-sig"))
506
+ return _parse_payload(args.record)
454
507
 
455
508
 
456
509
  def cmd_add(args) -> int:
@@ -462,16 +515,23 @@ def cmd_add(args) -> int:
462
515
  qpath = append_quarantine(path, led.quarantined)
463
516
  warn(f"{len(led.quarantined)} pre-existing malformed line(s) quarantined to {qpath} (excluded from rewrite)")
464
517
  accepted = rejected = 0
518
+ derived_keys: list = []
465
519
  for raw in _parse_incoming(args):
466
520
  try:
467
- led.upsert(normalize_record(raw, require_provenance_aspect=args.aspect))
521
+ rec = normalize_record(raw, require_provenance_aspect=args.aspect)
468
522
  except (ValueError, TypeError) as e:
469
523
  rejected += 1
470
524
  warn(f"rejected record ({e}): {json.dumps(raw, ensure_ascii=False)[:200]}")
471
525
  continue
526
+ led.upsert(rec)
527
+ derived_keys.append(rec["key"])
472
528
  accepted += 1
473
529
  led.write(path)
474
530
  banner("add", f"{accepted} record(s) accepted, {rejected} rejected -> {path}")
531
+ if derived_keys:
532
+ # Echo the derived canonical keys so workers cite exactly what the
533
+ # ledger keyed (dataset/alias-derived keys are otherwise guesswork).
534
+ banner("add", f"derived keys: {', '.join(derived_keys)}")
475
535
  return 0
476
536
 
477
537
 
@@ -510,14 +570,27 @@ def cmd_merge(args) -> int:
510
570
  return 0
511
571
 
512
572
 
573
+ def _token_overlap(t1: str, t2: str) -> float:
574
+ toks1 = set(re.findall(r"[a-z0-9]{3,}", (t1 or "").lower()))
575
+ toks2 = set(re.findall(r"[a-z0-9]{3,}", (t2 or "").lower()))
576
+ if not toks1 or not toks2:
577
+ return 1.0
578
+ return len(toks1 & toks2) / min(len(toks1), len(toks2))
579
+
580
+
513
581
  def _esummary_locator(doc: dict) -> dict:
514
582
  m = re.search(r"\b(19\d\d|20\d\d)\b", str(doc.get("pubdate", "")))
515
583
  year = m.group(1) if m else str(doc.get("sortpubdate") or "")[:4]
584
+ pages = str(doc.get("pages", "")).strip() or None
585
+ if not pages:
586
+ eloc = str(doc.get("elocationid", "")).strip()
587
+ if eloc and not eloc.lower().startswith("doi:"):
588
+ pages = re.sub(r"^(?:pii|articleno|article):\s*", "", eloc, flags=re.IGNORECASE) or None
516
589
  return {
517
590
  "year": year or None,
518
591
  "volume": str(doc.get("volume", "")).strip() or None,
519
592
  "issue": str(doc.get("issue", "")).strip() or None,
520
- "pages": str(doc.get("pages", "")).strip() or None,
593
+ "pages": pages,
521
594
  }
522
595
 
523
596
 
@@ -535,6 +608,14 @@ def _esummary_doi(doc: dict):
535
608
  return None
536
609
 
537
610
 
611
+ def _esummary_authors(doc: dict) -> list:
612
+ out = []
613
+ for a in doc.get("authors", []) or []:
614
+ if isinstance(a, dict) and a.get("name"):
615
+ out.append(str(a["name"]).strip())
616
+ return out
617
+
618
+
538
619
  def cmd_verify(args) -> int:
539
620
  path = Path(args.file)
540
621
  led = read_ledger(path)
@@ -543,7 +624,9 @@ def cmd_verify(args) -> int:
543
624
  if r.get("type") in verifiable and (r.get("ids") or {}).get("pmid") and not r.get("verified")})
544
625
  docs = fetch_ncbi_summaries(pmids, timeout=args.timeout) if pmids else {}
545
626
  now = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
546
- filled = title_fixed = clean = unreachable = 0
627
+ filled = title_fixed = clean = unreachable = mismatches = 0
628
+ already = sum(1 for r in led.records()
629
+ if r.get("type") in verifiable and (r.get("ids") or {}).get("pmid") and r.get("verified"))
547
630
  skipped = sum(1 for r in led.records() if r.get("type") not in verifiable)
548
631
  for rec in led.records():
549
632
  if rec.get("type") not in verifiable:
@@ -555,6 +638,28 @@ def cmd_verify(args) -> int:
555
638
  if not doc:
556
639
  unreachable += 1
557
640
  continue
641
+
642
+ # Conflict detection
643
+ mismatch_reasons = []
644
+ ncbi_title = _esummary_title(doc)
645
+ if rec.get("title") and ncbi_title and len(ncbi_title) > 20:
646
+ overlap = _token_overlap(rec["title"], ncbi_title)
647
+ if overlap < 0.30:
648
+ mismatch_reasons.append(f"title overlap {overlap:.2f} < 0.30")
649
+
650
+ ncbi_doi = _esummary_doi(doc)
651
+ rec_doi = (rec.get("ids") or {}).get("doi")
652
+ if rec_doi and ncbi_doi:
653
+ if rec_doi.lower().rstrip(".") != ncbi_doi.lower().rstrip("."):
654
+ mismatch_reasons.append(f"doi conflict ({rec_doi} vs {ncbi_doi})")
655
+
656
+ if mismatch_reasons:
657
+ rec["verified"] = False
658
+ rec["verification_notes"] = f"mismatch: {'; '.join(mismatch_reasons)}"
659
+ warn(f"pmid:{pmid} metadata mismatch: {'; '.join(mismatch_reasons)}. Record left unverified.")
660
+ mismatches += 1
661
+ continue
662
+
558
663
  changed = False
559
664
  loc = _esummary_locator(doc)
560
665
  for field in ("year", "volume", "issue", "pages"):
@@ -563,9 +668,8 @@ def cmd_verify(args) -> int:
563
668
  rec["backfilled"].append(field)
564
669
  changed = True
565
670
  if not (rec.get("ids") or {}).get("doi"):
566
- doi = _esummary_doi(doc)
567
- if doi:
568
- rec["ids"]["doi"] = doi
671
+ if ncbi_doi:
672
+ rec["ids"]["doi"] = ncbi_doi
569
673
  rec["backfilled"].append("doi")
570
674
  changed = True
571
675
  if not rec.get("journal"):
@@ -575,16 +679,22 @@ def cmd_verify(args) -> int:
575
679
  rec["backfilled"].append("journal")
576
680
  changed = True
577
681
  if not rec.get("title"):
578
- t = _esummary_title(doc)
579
- if t:
580
- rec["title"] = t
682
+ if ncbi_title:
683
+ rec["title"] = ncbi_title
581
684
  rec["title_original"] = None
582
685
  rec["backfilled"].append("title")
583
686
  changed = True
584
687
  title_fixed += 1
688
+ if not rec.get("authors"):
689
+ es_authors = _esummary_authors(doc)
690
+ if es_authors:
691
+ rec["authors"] = es_authors
692
+ rec["backfilled"].append("authors")
693
+ changed = True
585
694
  rec["verified"] = True
586
695
  rec["verified_source"] = "ncbi-esummary"
587
696
  rec["verified_at"] = now
697
+ rec.pop("verification_notes", None)
588
698
  if changed:
589
699
  filled += 1
590
700
  else:
@@ -594,11 +704,13 @@ def cmd_verify(args) -> int:
594
704
  if led.quarantined:
595
705
  qpath = append_quarantine(path, led.quarantined)
596
706
  warn(f"{len(led.quarantined)} pre-existing malformed line(s) quarantined to {qpath} (excluded from rewrite)")
707
+ mismatch_note = f", {mismatches} metadata mismatch(es) (unverified)" if mismatches else ""
597
708
  banner(
598
709
  "verify",
599
710
  f"{len(docs)} PubMed record(s) checked; {filled} backfilled, {title_fixed} title(s) set, "
600
- f"{clean} verified clean, {unreachable} unreachable (fail-safe, left unverified); "
711
+ f"{clean} verified clean{mismatch_note}, {unreachable} unreachable (fail-safe, left unverified); "
601
712
  f"{skipped} record(s) of unverified type(s) skipped (no verifier configured)"
713
+ + (f"; {already} already verified (not rechecked)" if already else "")
602
714
  + ("; --apply written" if args.apply else " (dry-run, no changes written)"),
603
715
  )
604
716
  return 0
@@ -643,8 +755,20 @@ def vancouver_author(name: str) -> str:
643
755
  surname, given = name, []
644
756
  else:
645
757
  surname, given = tokens[0], tokens[1:]
646
- initials = "".join(t[0].upper() for t in given if t and t[0].isalpha())
647
- return f"{surname} {initials}" if initials else surname
758
+ surname = surname.rstrip(",")
759
+ # Handle generational suffixes (Jr, Sr, 2nd, 3rd, II, III, IV)
760
+ suffix = ""
761
+ if given and given[-1].lower() in {"jr", "jr.", "sr", "sr.", "ii", "iii", "iv", "2nd", "3rd"}:
762
+ suffix = " " + given[-1].rstrip(".")
763
+ given = given[:-1]
764
+ cleaned_given = [t.replace(".", "") for t in given if t]
765
+ # Idempotency guard: if given is already uppercase initials (e.g. ["PB"], ["J.W."], ["SH"])
766
+ if len(cleaned_given) == 1 and cleaned_given[0].isupper() and cleaned_given[0].isalpha() and len(cleaned_given[0]) <= 4:
767
+ initials = cleaned_given[0]
768
+ else:
769
+ initials = "".join(t[0].upper() for t in given if t and t[0].isalpha())
770
+ res = f"{surname} {initials}" if initials else surname
771
+ return f"{res}{suffix}"
648
772
 
649
773
 
650
774
  def _author_list(rec: dict, max_authors: int = 3) -> str:
@@ -653,7 +777,9 @@ def _author_list(rec: dict, max_authors: int = 3) -> str:
653
777
  return "[MISSING field: authors]"
654
778
  out = ", ".join(filter(None, (vancouver_author(str(a)) for a in authors[:max_authors])))
655
779
  if len(authors) > max_authors:
656
- out += ", et al"
780
+ out += ", et al."
781
+ else:
782
+ out = _close_segment(out)
657
783
  return out
658
784
 
659
785
 
@@ -696,6 +822,25 @@ def _close(s: str) -> str:
696
822
  return s if s.endswith(".") else s + "."
697
823
 
698
824
 
825
+ def _close_segment(s: str) -> str:
826
+ """Ensure a metadata segment terminates with exactly one period, stripping any
827
+ pre-existing trailing period or whitespace (e.g. 'Tesaro, Inc.' -> 'Tesaro, Inc.')."""
828
+ if not s:
829
+ return ""
830
+ stripped = str(s).strip()
831
+ if not stripped:
832
+ return ""
833
+ return stripped.rstrip(".") + "."
834
+
835
+
836
+ def _close_title(v) -> str:
837
+ """Render a title and close with a single period (registry titles often
838
+ already end with one - never emit 'Title..')."""
839
+ if v in (None, ""):
840
+ return f"[MISSING field: title]"
841
+ return _close(str(v).strip().rstrip("."))
842
+
843
+
699
844
  def render_article(rec: dict, expand: bool) -> str:
700
845
  ids = rec.get("ids") or {}
701
846
  title = _close(_need(rec, "title").strip())
@@ -716,16 +861,16 @@ def render_trial(rec: dict) -> str:
716
861
  ids = rec.get("ids") or {}
717
862
  meta = rec.get("meta") or {}
718
863
  nct = ids.get("nct") or "[MISSING field: ids.nct]"
719
- out = f"{nct}: {_need(rec, 'title')}."
864
+ out = f"{nct}: {_close_title(rec.get('title'))}"
720
865
  phase = str(meta.get("phase") or "").strip().rstrip(".")
721
866
  if phase:
722
867
  # Workers copy phase verbatim from biomcp/CTgov ("Phase 2", "PHASE3",
723
868
  # "2"): never double the prefix.
724
869
  out += f" {phase}." if phase.lower().startswith("phase") else f" Phase {phase}."
725
870
  if meta.get("sponsor"):
726
- out += f" Sponsor: {meta['sponsor']}."
871
+ out += f" Sponsor: {_close_segment(meta['sponsor'])}"
727
872
  if meta.get("status"):
728
- out += f" Status: {meta['status']}."
873
+ out += f" Status: {_close_segment(meta['status'])}"
729
874
  out += " " + (rec.get("url") or f"https://clinicaltrials.gov/study/{nct}")
730
875
  return out
731
876
 
@@ -734,10 +879,11 @@ def render_patent(rec: dict) -> str:
734
879
  ids = rec.get("ids") or {}
735
880
  meta = rec.get("meta") or {}
736
881
  num = ids.get("patent") or "[MISSING field: ids.patent]"
737
- assignee = meta.get("assignee") or "[MISSING field: meta.assignee]"
882
+ assignee = (meta.get("assignee") or "").strip()
883
+ assignee_str = f"{_close_segment(assignee)} " if assignee else "[MISSING field: meta.assignee]. "
738
884
  status = f" ({meta['status']})" if meta.get("status") else ""
739
885
  url = rec.get("url") or f"https://patents.google.com/patent/{num}"
740
- return f"{assignee}. {_need(rec, 'title')}. {num}{status}. {url}"
886
+ return f"{assignee_str}{_close_title(rec.get('title'))} {num}{status}. {url}"
741
887
 
742
888
 
743
889
  def render_gene(rec: dict) -> str:
@@ -771,20 +917,31 @@ def render_drug(rec: dict) -> str:
771
917
  "[MISSING field: ids.chembl|chebi|unii]")
772
918
  ind = f" Indication: {meta['indication']}." if meta.get("indication") else ""
773
919
  url = rec.get("url") or ""
774
- return f"{_need(rec, 'title')}.{ind} {dbid}. {url}".strip()
920
+ return f"{_close_title(rec.get('title'))}{ind} {dbid}. {url}".strip()
775
921
 
776
922
 
777
923
  def render_disease(rec: dict) -> str:
778
924
  ids = rec.get("ids") or {}
779
925
  oid = next((f"{k.upper()}:{ids[k]}" for k in ("mondo", "doid", "omim", "efo") if ids.get(k)),
780
926
  "[MISSING field: ids.mondo|doid|omim|efo]")
781
- return f"{_need(rec, 'title')}. {oid}. {rec.get('url') or ''}".strip()
927
+ return f"{_close_title(rec.get('title'))} {oid}. {rec.get('url') or ''}".strip()
782
928
 
783
929
 
784
930
  def render_dataset(rec: dict) -> str:
785
931
  ids = rec.get("ids") or {}
786
932
  key = rec.get("key", "")
787
- title = _need(rec, "title")
933
+ title = _need(rec, "title").strip().rstrip(".")
934
+ if key.startswith("pdb:"):
935
+ acc = ids.get("pdb") or key[len("pdb:"):]
936
+ meta = rec.get("meta") or {}
937
+ extras = []
938
+ method = meta.get("method") or meta.get("experimental_method")
939
+ if method:
940
+ extras.append(f"[{method}]")
941
+ if meta.get("resolution"):
942
+ extras.append(f"Resolution: {meta['resolution']}.")
943
+ extra_str = f" {' '.join(extras)}" if extras else ""
944
+ return f"PDB structure {acc}: {title}.{extra_str} https://www.rcsb.org/structure/{acc}"
788
945
  if key.startswith("geo:"):
789
946
  acc = ids.get("geo") or ids.get("accession") or key[len("geo:"):]
790
947
  return f"GEO series {acc}: {title}. https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={acc}"
@@ -797,10 +954,12 @@ def render_dataset(rec: dict) -> str:
797
954
 
798
955
  def render_web(rec: dict) -> str:
799
956
  meta = rec.get("meta") or {}
800
- updated = f" Updated {meta['updated']}." if meta.get("updated") else ""
957
+ updated = f" Updated {_close_segment(meta['updated'])}" if meta.get("updated") else ""
801
958
  url = rec.get("url") or (rec.get("ids") or {}).get("url") or "[MISSING field: url]"
802
959
  accessed = meta.get("accessed") or "[MISSING field: meta.accessed]"
803
- return f"{_need(rec, 'title')}. {meta.get('organization') or '[MISSING field: meta.organization]'}.{updated} {url}. Accessed: {accessed}."
960
+ org = (meta.get("organization") or "").strip()
961
+ org_str = f" {_close_segment(org)}" if org else " [MISSING field: meta.organization]."
962
+ return f"{_close_title(rec.get('title'))}{org_str}{updated} {url}. Accessed: {accessed}."
804
963
 
805
964
 
806
965
  def render_other(rec: dict) -> str:
@@ -903,6 +1062,299 @@ def cmd_stats(args) -> int:
903
1062
  return 0
904
1063
 
905
1064
 
1065
+ # ---------------------------------------------------------------------------
1066
+ # Cite-key rendering (draft -> numbered report; the single numbering authority)
1067
+ # ---------------------------------------------------------------------------
1068
+
1069
+ # A cite-key marker: [@ns:value] or [@a; @b]. A bracket is a citation group
1070
+ # only if EVERY non-empty token is a recognized namespace with a shape-valid
1071
+ # value; anything else (prose [@home], pandoc [@Chapman2011], [@gene:BRAF]
1072
+ # symbols) passes through verbatim.
1073
+ MARKER_RE = re.compile(r"\[@([^\[\]]+)\]")
1074
+
1075
+ # Canonical value shapes per key namespace (see KEY_NAMESPACES). Namespaces
1076
+ # without an entry accept any non-empty value.
1077
+ NS_VALUE_SHAPES = {
1078
+ "pmid": re.compile(r"^\d+$"),
1079
+ "doi": re.compile(r"^10\.\S+$", re.IGNORECASE),
1080
+ "pmcid": re.compile(r"^PMC\d+$", re.IGNORECASE),
1081
+ "nct": re.compile(r"^NCT\d+$", re.IGNORECASE),
1082
+ "patent": re.compile(r"^[A-Z]{2}\d+", re.IGNORECASE),
1083
+ "geo": re.compile(r"^GS[ED]\d+$", re.IGNORECASE),
1084
+ "sra": re.compile(r"^SR[RP]\d+$", re.IGNORECASE),
1085
+ "gb": re.compile(r"^[A-Z]{2,}\d+(\.\d+)?$", re.IGNORECASE),
1086
+ "pdb": re.compile(r"^[0-9][a-z0-9]{3}$", re.IGNORECASE),
1087
+ "gene": re.compile(r"^\d+$"),
1088
+ "clinvar": re.compile(r"^\d+$"),
1089
+ "chembl": re.compile(r"^CHEMBL\d+$", re.IGNORECASE),
1090
+ "chebi": re.compile(r"^CHEBI[:_ ]?\d+$", re.IGNORECASE),
1091
+ "unii": re.compile(r"^[A-Z0-9]{4,10}$", re.IGNORECASE),
1092
+ "mondo": re.compile(r"^MONDO[:_ ]?\d+$", re.IGNORECASE),
1093
+ "doid": re.compile(r"^DOID[:_ ]?\d+$", re.IGNORECASE),
1094
+ "omim": re.compile(r"^\d{5,7}$"),
1095
+ "efo": re.compile(r"^[A-Z]{2,}[_:]?\d+", re.IGNORECASE),
1096
+ "url": re.compile(r"^[0-9a-f]{16}$", re.IGNORECASE),
1097
+ "title": re.compile(r"^[0-9a-f]{16}$", re.IGNORECASE),
1098
+ }
1099
+
1100
+ REFS_STRIP_RE = re.compile(
1101
+ r"(?ims)^#{1,6}[ \t]*(?:references?|bibliography|literature cited|citations)\b[^\n]*\n"
1102
+ r".*?(?=\n#{1,6}[ \t]*\S|\Z)"
1103
+ )
1104
+
1105
+ # Hand-typed numeric citation brackets (render's input is authored with
1106
+ # [@key] markers, so any [N]/[N, M]/[N-M] bracket in a draft is suspect).
1107
+ # Negative lookarounds exclude markdown links [1](url), reference defs [1]:,
1108
+ # and wikilinks [[1,2]].
1109
+ HAND_TYPED_NUM_RE = re.compile(r"(?<!\[)\[\d{1,3}(?:\s*[,\u2013\-]\s*\d{1,3})*\](?![:(\[])")
1110
+
1111
+ CODE_BLOCK_RE = re.compile(r"(?ms)^(?:```|~~~)[^\n]*\n.*?^(?:```|~~~)[ \t]*$")
1112
+
1113
+
1114
+ def mask_code_blocks(text: str) -> str:
1115
+ """Fenced code blocks -> same-length newline filler (offsets preserved), so
1116
+ section detection and marker scans never fire on example/documentation
1117
+ content inside fences."""
1118
+ return CODE_BLOCK_RE.sub(lambda m: "\n" * (m.end() - m.start()), text)
1119
+
1120
+
1121
+ def classify_token(token: str):
1122
+ """Classify a marker token: ('cite', ns, value) | ('shape', ns, value) | ('plain', tok, None)."""
1123
+ tok = token.strip()
1124
+ if tok.startswith("@"):
1125
+ tok = tok[1:].strip()
1126
+ if ":" not in tok:
1127
+ return ("plain", tok, None)
1128
+ ns, _, value = tok.partition(":")
1129
+ ns, value = ns.strip().lower(), value.strip()
1130
+ if not value or ns not in KEY_NAMESPACES:
1131
+ return ("plain", tok, None)
1132
+ shape = NS_VALUE_SHAPES.get(ns)
1133
+ if shape is None or shape.match(value):
1134
+ return ("cite", ns, value)
1135
+ return ("shape", ns, value)
1136
+
1137
+
1138
+ def _canonical_marker_key(ns: str, value: str) -> str:
1139
+ try:
1140
+ return f"{ns}:{_canonicalize_id_value(ns, value)}"
1141
+ except ValueError:
1142
+ return f"{ns}:{value}"
1143
+
1144
+
1145
+ def resolve_key(led: Ledger, ns: str, value: str):
1146
+ """Resolve a marker key to a ledger key: direct -> secondary-id twin -> None."""
1147
+ cand = _canonical_marker_key(ns, value)
1148
+ if cand in led.by_key:
1149
+ return cand
1150
+ if ns in ("doi", "pmcid"):
1151
+ # merge promotes doi:/pmcid: twins to their pmid key; the marker may
1152
+ # legitimately cite the pre-promotion namespace.
1153
+ canon_val = cand.split(":", 1)[1].lower()
1154
+ twin = led.sec_index.get(f"{ns}:{canon_val}")
1155
+ if twin and twin in led.by_key:
1156
+ return twin
1157
+ return None
1158
+
1159
+
1160
+ def _compress_numbers(nums: list) -> str:
1161
+ """[1,2,3,5] -> '1-3, 5'; [1,2] -> '1, 2'."""
1162
+ nums = sorted(set(nums))
1163
+ parts, i = [], 0
1164
+ while i < len(nums):
1165
+ j = i
1166
+ while j + 1 < len(nums) and nums[j + 1] == nums[j] + 1:
1167
+ j += 1
1168
+ if j - i >= 2:
1169
+ parts.append(f"{nums[i]}-{nums[j]}")
1170
+ elif j == i + 1:
1171
+ parts.append(f"{nums[i]}, {nums[j]}")
1172
+ else:
1173
+ parts.append(str(nums[i]))
1174
+ i = j + 1
1175
+ return ", ".join(parts)
1176
+
1177
+
1178
+ def cmd_render(args) -> int:
1179
+ led = read_ledger(Path(args.ledger))
1180
+ text = Path(args.draft).read_text(encoding="utf-8-sig")
1181
+ out_path = Path(args.out)
1182
+
1183
+ # Strip pre-existing References-like sections, fence-aware: detect on a
1184
+ # code-block-masked copy (offsets preserved), cut from the real text in
1185
+ # reverse so earlier spans stay valid.
1186
+ masked = mask_code_blocks(text)
1187
+ had_entries = False
1188
+ for m in reversed(list(REFS_STRIP_RE.finditer(masked))):
1189
+ chunk = text[m.start():m.end()]
1190
+ if re.search(r"(?m)^\s*[-*]?\s*\[\d+\]", chunk):
1191
+ had_entries = True
1192
+ if "[@" in chunk:
1193
+ warn("a pre-existing References-like section contained [@key] marker(s); "
1194
+ "the section was stripped - cite those sources in the body instead")
1195
+ text = text[:m.start()] + text[m.end():]
1196
+ body = text.rstrip() + "\n"
1197
+
1198
+ failures: list = []
1199
+ numbers: dict = {}
1200
+ order: list = []
1201
+
1202
+ def render_marker(bracket: str) -> str:
1203
+ inner = bracket[2:-1] if bracket.endswith("]") else bracket[2:]
1204
+ tokens = [t for t in inner.split(";") if t.strip()]
1205
+ kinds = [classify_token(t) for t in tokens]
1206
+ cites = [k for k in kinds if k[0] == "cite"]
1207
+ if not tokens or not cites:
1208
+ for kind, ns, _ in kinds:
1209
+ if kind == "shape":
1210
+ warn(f"bracket {bracket!r} uses the {ns}: namespace but its value is not shape-valid; left verbatim")
1211
+ return bracket
1212
+ if any(k[0] != "cite" for k in kinds):
1213
+ # A group with at least one real cite-key must not silently drop
1214
+ # its non-citation tokens - that would lose citations quietly.
1215
+ failures.append(f"mixed citation group {bracket!r}: every token must be a cite-key")
1216
+ return bracket
1217
+ keys = []
1218
+ for _, ns, value in cites:
1219
+ key = resolve_key(led, ns, value)
1220
+ if key is None:
1221
+ suggestions = [k for k in sorted(led.by_key) if k.startswith(ns + ":")][:5]
1222
+ failures.append(
1223
+ f"unknown citation key {ns}:{value}"
1224
+ + (f" (did you mean: {', '.join(suggestions)}?)" if suggestions else "")
1225
+ )
1226
+ continue
1227
+ keys.append(key)
1228
+ if failures:
1229
+ return bracket
1230
+ for key in keys:
1231
+ if key not in numbers:
1232
+ numbers[key] = len(numbers) + 1
1233
+ order.append(key)
1234
+ return "[" + _compress_numbers([numbers[k] for k in keys]) + "]"
1235
+
1236
+ # Marker substitution, fence-aware: scan the masked copy (offsets equal),
1237
+ # splice replacements into the real body.
1238
+ masked_body = mask_code_blocks(body)
1239
+ # Hand-typed numeric brackets in the DRAFT are the residual leak class:
1240
+ # render owns numbering, so warn loudly (non-fatal - prose ranges like
1241
+ # [140, 155] are legitimate; links/wikilinks/reference defs are excluded).
1242
+ hand_typed = [m.start() for m in HAND_TYPED_NUM_RE.finditer(masked_body)]
1243
+ if hand_typed:
1244
+ lines = sorted({masked_body.count("\n", 0, pos) + 1 for pos in hand_typed[:5]})
1245
+ warn(f"draft contains {len(hand_typed)} hand-typed numeric citation bracket(s) "
1246
+ f"(first at line(s) {lines}): render owns numbering - remove hand-typed [N] brackets from the draft")
1247
+ parts: list = []
1248
+ last = 0
1249
+ for m in MARKER_RE.finditer(masked_body):
1250
+ parts.append(body[last:m.start()])
1251
+ parts.append(render_marker(body[m.start():m.end()]))
1252
+ last = m.end()
1253
+ parts.append(body[last:])
1254
+ rendered_body = "".join(parts)
1255
+
1256
+ # Double-render guard: an already-rendered document has no markers left.
1257
+ if not order and had_entries:
1258
+ banner("render", "FAILED: no [@key] citation markers found, but a References section with entries was present "
1259
+ "(already-rendered document?); nothing written")
1260
+ return 1
1261
+ if failures:
1262
+ for f in failures:
1263
+ warn(f"unresolved citation: {f}")
1264
+ banner("render", f"FAILED: {len(failures)} unresolved citation key(s); nothing written")
1265
+ return 1
1266
+
1267
+ entries = []
1268
+ missing = []
1269
+ for key in order:
1270
+ entry = render_record(led.by_key[key], args.expand_pages)
1271
+ if "[MISSING" in entry:
1272
+ missing.append(key)
1273
+ entries.append(f"[{numbers[key]}] {entry}")
1274
+ if missing:
1275
+ for key in missing:
1276
+ warn(f"record {key} would render with [MISSING ...] markers (incomplete ledger record)")
1277
+ banner("render", f"FAILED: {len(missing)} record(s) render with [MISSING ...] gaps "
1278
+ f"({', '.join(missing[:5])}{' ...' if len(missing) > 5 else ''}); nothing written")
1279
+ return 1
1280
+
1281
+ out_path.parent.mkdir(parents=True, exist_ok=True)
1282
+ refs = "\n".join(entries) if entries else "(no cited sources)"
1283
+ out_path.write_text(rendered_body + "\n## References\n\n" + refs + "\n", encoding="utf-8")
1284
+ banner("render", f"{len(order)} citation(s) numbered, {len(entries)} reference(s) rendered from {args.ledger} -> {args.out}")
1285
+ return 0
1286
+
1287
+
1288
+ def _check_markers(led: Ledger, paths: list) -> int:
1289
+ """Cross-validate [@key] markers in markdown files against the ledger.
1290
+
1291
+ Mirrors render's group semantics exactly: groups with zero cite tokens are
1292
+ prose (ignored); all-cite groups must fully resolve (direct or sec_index);
1293
+ groups mixing >=1 cite token with any non-cite token are failures;
1294
+ shape-kind tokens (recognized namespace, invalid shape) warn only.
1295
+ Returns the number of failures; a missing marker file counts as one
1296
+ (never a vacuous pass)."""
1297
+ failures = 0
1298
+ for raw in paths:
1299
+ p = Path(raw)
1300
+ if not p.is_file():
1301
+ banner("check", f"markers: marker file not found: {p}")
1302
+ failures += 1
1303
+ continue
1304
+ text = p.read_text(encoding="utf-8-sig")
1305
+ masked = mask_code_blocks(text)
1306
+ groups = resolved = 0
1307
+ for m in MARKER_RE.finditer(masked):
1308
+ tokens = [t for t in m.group(1).split(";") if t.strip()]
1309
+ kinds = [classify_token(t) for t in tokens]
1310
+ cites = [k for k in kinds if k[0] == "cite"]
1311
+ if not tokens or not cites:
1312
+ continue # prose bracket - render leaves it verbatim
1313
+ line = masked.count("\n", 0, m.start()) + 1
1314
+ groups += 1
1315
+ if any(k[0] != "cite" for k in kinds):
1316
+ warn(f"{p.name}:{line}: mixed citation group {m.group(0)!r}: every token must be a cite-key")
1317
+ failures += 1
1318
+ continue
1319
+ for kind, ns, value in cites:
1320
+ if kind == "shape":
1321
+ warn(f"{p.name}:{line}: bracket {m.group(0)!r} uses the {ns}: namespace but its value is not shape-valid (left verbatim)")
1322
+ continue
1323
+ key = resolve_key(led, ns, value)
1324
+ if key is None:
1325
+ warn(f"{p.name}:{line}: unresolved marker {ns}:{value} (no matching ledger record)")
1326
+ failures += 1
1327
+ else:
1328
+ resolved += 1
1329
+ print(f"markers[{p.name}]: {groups} citation group(s), {resolved} marker(s) resolved")
1330
+ return failures
1331
+
1332
+
1333
+ def cmd_check(args) -> int:
1334
+ markers = getattr(args, "markers", None) or []
1335
+ led = read_ledger(Path(args.file))
1336
+ types: dict = {}
1337
+ for rec in led.records():
1338
+ types[rec.get("type", "?")] = types.get(rec.get("type", "?"), 0) + 1
1339
+ for k in sorted(led.by_key):
1340
+ print(k)
1341
+ status = "OK" if not led.quarantined else "FAIL"
1342
+ banner("check", f"{len(led.by_key)} record(s) across {len(types)} type(s) "
1343
+ f"({', '.join(f'{t}:{c}' for t, c in sorted(types.items())) or 'none'}); "
1344
+ f"quarantined {len(led.quarantined)}; {status}")
1345
+ if led.quarantined:
1346
+ for q in led.quarantined:
1347
+ warn(f"quarantined {q.get('file')}:{q.get('line')}: {q.get('error')}")
1348
+ return 1
1349
+ if markers:
1350
+ marker_failures = _check_markers(led, markers)
1351
+ banner("check", f"markers: {marker_failures} problem(s) across {len(markers)} file(s); "
1352
+ f"{'FAIL' if marker_failures else 'OK'}")
1353
+ if marker_failures:
1354
+ return 1
1355
+ return 0
1356
+
1357
+
906
1358
  # ---------------------------------------------------------------------------
907
1359
  # Hermetic selftest (CI; no network)
908
1360
  # ---------------------------------------------------------------------------
@@ -1057,6 +1509,7 @@ def selftest() -> int:
1057
1509
  "articleids": [{"idtype": "doi", "value": "10.1111/cas.70480"}]},
1058
1510
  "99900001": {"pubdate": "2011 Jun 30", "volume": "364", "issue": "26", "pages": "2507-16",
1059
1511
  "source": "N Engl J Med", "title": "Mocked title for hint record.",
1512
+ "authors": [{"name": "Chapman PB", "authtype": "Author"}, {"name": "Hauschild A", "authtype": "Author"}],
1060
1513
  "articleids": [{"idtype": "doi", "value": "10.1056/NEJMoa1103782"}]},
1061
1514
  }
1062
1515
  mod = sys.modules[__name__]
@@ -1073,9 +1526,48 @@ def selftest() -> int:
1073
1526
  h = led.by_key["pmid:99900001"]
1074
1527
  assert h["title"] == "Mocked title for hint record", "hint title not backfilled"
1075
1528
  assert h["volume"] == "364" and h["pages"] == "2507-16", "locators not backfilled"
1076
- assert "volume" in h["backfilled"] and "title" in h["backfilled"]
1529
+ assert h["authors"] == ["Chapman PB", "Hauschild A"], f"authors not backfilled: {h['authors']}"
1530
+ assert "volume" in h["backfilled"] and "title" in h["backfilled"] and "authors" in h["backfilled"]
1077
1531
  check("verify-backfill", st_verify_backfill)
1078
1532
 
1533
+ # ---- verify mismatch (conflict detection) --------------------------------
1534
+ def st_verify_mismatch():
1535
+ f = d / "vm.jsonl"
1536
+ # Title mismatch: completely unrelated title put into article record
1537
+ bad_title = _fixture_article(pmid="540362", ids={"pmid": "540362", "doi": "10.1248/cpb.27.1942"},
1538
+ title="Unrelated Subject Matter on Plant Photosynthesis")
1539
+ # DOI mismatch
1540
+ bad_doi = _fixture_article(pmid="21639808", ids={"pmid": "21639808", "doi": "10.1000/wrong.doi"},
1541
+ title="Improved survival with vemurafenib in melanoma with BRAF V600E mutation")
1542
+ f.write_text(json.dumps(bad_title) + "\n" + json.dumps(bad_doi) + "\n", encoding="utf-8")
1543
+ docs = {
1544
+ "540362": {"pubdate": "1979", "volume": "27", "issue": "8", "pages": "1942-4",
1545
+ "source": "Chem Pharm Bull (Tokyo)",
1546
+ "title": "Solution structure of alpha-conotoxin EI from Conus ermineus.",
1547
+ "articleids": [{"idtype": "doi", "value": "10.1248/cpb.27.1942"}]},
1548
+ "21639808": {"pubdate": "2011", "volume": "364", "issue": "26", "pages": "2507-16",
1549
+ "source": "N Engl J Med",
1550
+ "title": "Improved survival with vemurafenib in melanoma with BRAF V600E mutation.",
1551
+ "articleids": [{"idtype": "doi", "value": "10.1056/nejmoa1103782"}]},
1552
+ }
1553
+ mod = sys.modules[__name__]
1554
+ original = mod.fetch_ncbi_summaries
1555
+ mod.fetch_ncbi_summaries = lambda pmids, timeout=15.0: docs
1556
+ try:
1557
+ rc, out = _capture(cmd_verify, argparse.Namespace(file=str(f), apply=True, timeout=1.0))
1558
+ finally:
1559
+ mod.fetch_ncbi_summaries = original
1560
+ assert rc == 0
1561
+ assert "2 metadata mismatch(es) (unverified)" in out
1562
+ led = read_ledger(f)
1563
+ rec_title = led.by_key["pmid:540362"]
1564
+ assert rec_title["verified"] is False, "title mismatch record must NOT be verified"
1565
+ assert "mismatch: title overlap" in rec_title.get("verification_notes", "")
1566
+ rec_doi = led.by_key["pmid:21639808"]
1567
+ assert rec_doi["verified"] is False, "doi mismatch record must NOT be verified"
1568
+ assert "mismatch: doi conflict" in rec_doi.get("verification_notes", "")
1569
+ check("verify-mismatch", st_verify_mismatch)
1570
+
1079
1571
  # ---- bib -------------------------------------------------------------------
1080
1572
  def st_bib():
1081
1573
  # Vancouver initials: standard, particle surnames, group passthrough
@@ -1085,6 +1577,11 @@ def selftest() -> int:
1085
1577
  assert vancouver_author("World Health Organization") == "World Health Organization"
1086
1578
  assert vancouver_author("Li Jiang") == "Li J"
1087
1579
  assert vancouver_author("WHO") == "WHO"
1580
+ # Idempotency of vancouver_author on pre-formatted initials & punctuation
1581
+ assert vancouver_author("Chapman PB") == "Chapman PB", vancouver_author("Chapman PB")
1582
+ assert vancouver_author("Taniguchi SH") == "Taniguchi SH", vancouver_author("Taniguchi SH")
1583
+ assert vancouver_author("Schmidberger, J.W.") == "Schmidberger JW", vancouver_author("Schmidberger, J.W.")
1584
+ assert vancouver_author("Smith JA Jr") == "Smith JA Jr", vancouver_author("Smith JA Jr")
1088
1585
  # expand_pages guards
1089
1586
  assert expand_pages("2507-16") == "2507-2516"
1090
1587
  assert expand_pages("2507-2516") == "2507-2516"
@@ -1095,7 +1592,7 @@ def selftest() -> int:
1095
1592
  rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:21639808", expand_pages=False, offset=0))
1096
1593
  assert rc == 0
1097
1594
  first = out.splitlines()[0]
1098
- assert "Chapman PB, Hauschild A, Robert C, et al" in first, f"Vancouver initials wrong: {first}"
1595
+ assert "Chapman PB, Hauschild A, Robert C, et al." in first, f"Vancouver initials / period wrong: {first}"
1099
1596
  assert "2011;364(26):2507-16" in first, f"locator wrong: {first}"
1100
1597
  assert "PMID: 21639808." in first and "DOI: 10.1056/nejmoa1103782." in first, first
1101
1598
  rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:21639808", expand_pages=True, offset=0))
@@ -1108,7 +1605,7 @@ def selftest() -> int:
1108
1605
  _, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:42487519", expand_pages=False, offset=0))
1109
1606
  line = out.splitlines()[0]
1110
1607
  assert "Cancer Sci. 2026." in line and "364" not in line and "DOI: 10.1111/cas.70480." in line, f"epub form wrong: {line}"
1111
- assert "Takahashi M, Taniguchi SH" in line, "multi-initial author wrong"
1608
+ assert "Takahashi M, Taniguchi SH." in line, f"author list period wrong: {line}"
1112
1609
  # unknown key -> loud + non-zero
1113
1610
  rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:42487519,pmid:nope", expand_pages=False, offset=0))
1114
1611
  assert rc != 0, "unknown key must exit non-zero"
@@ -1314,6 +1811,20 @@ def selftest() -> int:
1314
1811
  {"type": "article", "title": "no ids", "ids": {}}]
1315
1812
  rc, out = add_stdin(f, json.dumps(mixed))
1316
1813
  assert rc == 0 and "1 record(s) accepted, 1 rejected" in out, f"mixed batch accounting wrong: {out}"
1814
+ # newline-delimited JSON (.jsonl) batch via --stdin
1815
+ jsonl_batch = (json.dumps(_fixture_article(pmid="10000006", doi="10.1000/b6", title="Batch six", ids={"pmid": "10000006", "doi": "10.1000/b6", "pmcid": "PMC1000006"})) + "\n" +
1816
+ json.dumps(_fixture_article(pmid="10000007", doi="10.1000/b7", title="Batch seven", ids={"pmid": "10000007", "doi": "10.1000/b7", "pmcid": "PMC1000007"})) + "\n")
1817
+ rc, out = add_stdin(f, jsonl_batch)
1818
+ assert rc == 0 and "add: 2 record(s) accepted, 0 rejected" in out, f"stdin jsonl batch failed: {out}"
1819
+ # @file with newline-delimited JSONL
1820
+ jf = d / "bt-lines.jsonl"
1821
+ jf.write_text(jsonl_batch, encoding="utf-8")
1822
+ rc, out = _capture(cmd_add, argparse.Namespace(file=str(f), record="@" + str(jf), stdin=False, aspect=None))
1823
+ assert rc == 0 and "add: 2 record(s) accepted" in out, f"@file jsonl batch failed: {out}"
1824
+ # markdown code-fenced payload in stdin
1825
+ fenced = "```json\n" + json.dumps([_fixture_article(pmid="10000008", doi="10.1000/b8", title="Batch eight", ids={"pmid": "10000008", "doi": "10.1000/b8", "pmcid": "PMC1000008"})]) + "\n```\n"
1826
+ rc, out = add_stdin(f, fenced)
1827
+ assert rc == 0 and "add: 1 record(s) accepted" in out, f"fenced payload failed: {out}"
1317
1828
  # title-key rules: web/dataset keep hard identity fields; article
1318
1829
  # rejects title-only input AND explicit title: keys; other falls
1319
1830
  # back to a title: key.
@@ -1345,6 +1856,247 @@ def selftest() -> int:
1345
1856
  assert len(led3.by_key) == 1 and not led3.quarantined, f"BOM re-read failed: {led3.quarantined}"
1346
1857
  check("batch/title-rules", st_batch_and_title_rules)
1347
1858
 
1859
+ # ---- name -> title fold (biomcp `name`-shaped records) -------------
1860
+ def st_name_fold():
1861
+ f = d / "nf.jsonl"
1862
+ # drug carrying `name` instead of `title` (biomcp drug_get shape)
1863
+ drug = {"type": "drug", "ids": {"chembl": "CHEMBL1229517"}, "name": "vemurafenib",
1864
+ "meta": {"indication": "BRAF V600E-mutant melanoma"}, "provenance": [{"aspect": "a"}]}
1865
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(drug), stdin=False, aspect=None))
1866
+ led = read_ledger(f)
1867
+ assert "chembl:CHEMBL1229517" in led.by_key, "name-bearing drug not accepted"
1868
+ assert led.by_key["chembl:CHEMBL1229517"]["title"] == "vemurafenib", "name not folded into title"
1869
+ # fold is idempotent across re-reads and preserves the original field
1870
+ before = json.dumps(led.by_key["chembl:CHEMBL1229517"], sort_keys=True)
1871
+ after = json.dumps(read_ledger(f).by_key["chembl:CHEMBL1229517"], sort_keys=True)
1872
+ assert before == after, "name fold not idempotent on re-read"
1873
+ assert led.by_key["chembl:CHEMBL1229517"].get("name") == "vemurafenib", "original name field lost"
1874
+ # gene name-only now accepted too (acceptance widening, CHANGELOG-noted)
1875
+ gene = {"type": "gene", "ids": {"entrez_id": "673"}, "name": "B-Raf proto-oncogene",
1876
+ "provenance": [{"aspect": "a"}]}
1877
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(gene), stdin=False, aspect=None))
1878
+ assert "gene:673" in read_ledger(f).by_key, "name-bearing gene not accepted"
1879
+ # article keeps its hard id requirement: name-only rejected
1880
+ _capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "article", "name": "x", "ids": {}}', stdin=False, aspect=None))
1881
+ # web/dataset keep their hard identity fields: name-only rejected
1882
+ _capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "web", "name": "Just a page", "ids": {}}', stdin=False, aspect=None))
1883
+ _capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "dataset", "name": "Just a series", "ids": {}}', stdin=False, aspect=None))
1884
+ keys = set(read_ledger(f).by_key)
1885
+ assert not any(k.startswith("title:") for k in keys), "name-only article/web/dataset must not derive title keys"
1886
+ assert len(keys) == 2, f"name-fold acceptance boundary wrong: {sorted(keys)}"
1887
+ # renders clean (no [MISSING field: title])
1888
+ _, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="chembl:CHEMBL1229517", expand_pages=False, offset=0))
1889
+ assert "[MISSING" not in out.splitlines()[0], f"drug still renders MISSING: {out.splitlines()[0]}"
1890
+ assert "Vemurafenib" in out.splitlines()[0] or "vemurafenib" in out.splitlines()[0], out.splitlines()[0]
1891
+ check("name-fold", st_name_fold)
1892
+
1893
+ # ---- render: cite-key markers -> numbered citations + References ---
1894
+ def _render_ledger(d):
1895
+ f = d / "render.jsonl"
1896
+ batch = [
1897
+ _fixture_article(),
1898
+ _fixture_article(pmid="30000001", doi="10.1000/r1", title="Render one", ids={"pmid": "30000001", "doi": "10.1000/r1", "pmcid": "PMC3000001"}),
1899
+ _fixture_article(pmid="30000002", doi="10.1000/r2", title="Render two", ids={"pmid": "30000002", "doi": "10.1000/r2", "pmcid": "PMC3000002"}),
1900
+ _fixture_article(pmid="30000003", doi="10.1000/r3", title="Render three", ids={"pmid": "30000003", "doi": "10.1000/r3", "pmcid": "PMC3000003"}),
1901
+ {"key": "nct:NCT04280705", "type": "trial", "ids": {"nct": "NCT04280705"},
1902
+ "title": "Encorafenib Plus Cetuximab", "meta": {"phase": "Phase 2", "sponsor": "Pfizer", "status": "Completed"},
1903
+ "provenance": [{"aspect": "a"}]},
1904
+ {"type": "drug", "ids": {"chembl": "CHEMBL1229517"}, "name": "vemurafenib",
1905
+ "meta": {"indication": "BRAF V600E-mutant melanoma"}, "provenance": [{"aspect": "a"}]},
1906
+ ]
1907
+ old_stdin, sys.stdin = sys.stdin, io.StringIO(json.dumps(batch))
1908
+ try:
1909
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=None, stdin=True, aspect=None))
1910
+ finally:
1911
+ sys.stdin = old_stdin
1912
+ return f
1913
+
1914
+ def st_render():
1915
+ f = _render_ledger(d)
1916
+ draft = d / "report.draft.md"
1917
+ draft.write_text(
1918
+ "# Report\n\n"
1919
+ "Vemurafenib [@chembl:CHEMBL1229517] improves survival [@pmid:21639808].\n\n"
1920
+ "Prose [@home] and pandoc [@Chapman2011] and symbol-ish [@gene:BRAF] stay verbatim.\n\n"
1921
+ "Trial plus article [@nct:NCT04280705; @pmid:21639808] group. Again [@pmid:21639808].\n\n"
1922
+ "Three more [@pmid:30000001; @pmid:30000002; @pmid:30000003] in one group.\n",
1923
+ encoding="utf-8",
1924
+ )
1925
+ out = d / "final.md"
1926
+ rc, stdout = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out), expand_pages=False))
1927
+ assert rc == 0, f"render failed: {stdout}"
1928
+ text = out.read_text(encoding="utf-8")
1929
+ assert "Vemurafenib [1] improves survival [2]." in text, f"numbering wrong:\n{text}"
1930
+ assert "[@home]" in text and "[@Chapman2011]" in text and "[@gene:BRAF]" in text, "prose brackets rewritten"
1931
+ assert "[2, 3]" in text, f"group not sorted/compressed: {text}"
1932
+ assert "Again [2]." in text, "duplicate key not reusing its number"
1933
+ assert "[4-6]" in text, f"consecutive run not range-compressed: {text}"
1934
+ assert "## References" in text and "[1] vemurafenib." in text and "[2] Chapman PB" in text
1935
+ # registry titles ending in a period never render doubled ("Title.. Status")
1936
+ # and corporate sponsors ending in periods (e.g. "Tesaro, Inc.") do not double
1937
+ tdot = d / "tdot.jsonl"
1938
+ tdot.write_text(json.dumps({
1939
+ "key": "nct:NCT01844986", "type": "trial", "ids": {"nct": "NCT01844986"},
1940
+ "title": "Olaparib Maintenance Monotherapy in Patients With BRCA Mutated Ovarian Cancer Following First Line Platinum Based Chemotherapy.",
1941
+ "meta": {"phase": "Phase 3.", "sponsor": "Tesaro, Inc.", "status": "ACTIVE_NOT_RECRUITING."},
1942
+ "provenance": [{"aspect": "a"}]}) + "\n", encoding="utf-8")
1943
+ _, tout = _capture(cmd_bib, argparse.Namespace(file=str(tdot), keys="nct:NCT01844986", expand_pages=False, offset=0))
1944
+ tline = tout.splitlines()[0]
1945
+ assert "Chemotherapy. Phase 3. Sponsor: Tesaro, Inc. Status: ACTIVE_NOT_RECRUITING." in tline and ".." not in tline, f"double period rendered: {tline}"
1946
+ assert "[MISSING" not in text
1947
+ refs = text.split("## References", 1)[1]
1948
+ nums = re.findall(r"(?m)^\[(\d+)\]", refs)
1949
+ assert nums == [str(i) for i in range(1, 7)], f"bibliography not 1..N contiguous: {nums}"
1950
+ # idempotent re-render of the SAME draft
1951
+ out2 = d / "final2.md"
1952
+ _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out2), expand_pages=False))
1953
+ assert out2.read_text(encoding="utf-8") == text, "re-render of the same draft is not idempotent"
1954
+ # sec-index: doi: marker resolves to the promoted pmid twin
1955
+ doi_draft = d / "doi.draft.md"
1956
+ doi_draft.write_text("Only one [@doi:10.1056/nejmoa1103782].\n", encoding="utf-8")
1957
+ doi_out = d / "doi.md"
1958
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(doi_draft), out=str(doi_out), expand_pages=False))
1959
+ dtext = doi_out.read_text(encoding="utf-8")
1960
+ assert rc == 0 and "Only one [1]." in dtext and "PMID: 21639808." in dtext, f"doi twin resolution failed: {dtext}"
1961
+ assert len(re.findall(r"(?m)^\[\d+\]", dtext.split("## References", 1)[1])) == 1
1962
+ # unknown key: exit 1, output NOT written
1963
+ bad = d / "bad.draft.md"
1964
+ bad.write_text("Broken [@pmid:99999999].\n", encoding="utf-8")
1965
+ bad_out = d / "bad.md"
1966
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(bad), out=str(bad_out), expand_pages=False))
1967
+ assert rc != 0 and not bad_out.exists(), "unknown key must exit 1 without writing output"
1968
+ # [MISSING ...] entry: exit 1, output NOT written
1969
+ tonly = {"type": "trial", "title": "Title-only degraded trial", "provenance": [{"aspect": "a"}]}
1970
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(tonly), stdin=False, aspect=None))
1971
+ tkey = next(k for k in read_ledger(f).by_key if k.startswith("title:"))
1972
+ miss = d / "miss.draft.md"
1973
+ miss.write_text(f"Degraded [@{tkey}] cite.\n", encoding="utf-8")
1974
+ miss_out = d / "miss.md"
1975
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(miss), out=str(miss_out), expand_pages=False))
1976
+ assert rc != 0 and not miss_out.exists(), "MISSING-rendering record must exit 1 without writing output"
1977
+ # double-render guard: rendering an already-rendered report fails
1978
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(out), out=str(d / "double.md"), expand_pages=False))
1979
+ assert rc != 0, "double-render must fail (no markers, References present)"
1980
+ assert not (d / "double.md").exists()
1981
+ # fence-awareness: a fenced example References section / marker is
1982
+ # never stripped and never numbered
1983
+ fence_draft = d / "fence.draft.md"
1984
+ fence_draft.write_text(
1985
+ "# F\n\nReal cite [@pmid:21639808].\n\nExample (do not touch):\n\n```\n"
1986
+ "## References\n\n[1] example entry.\n\nCite like [@pmid:21639808].\n```\n\nTail kept.\n",
1987
+ encoding="utf-8",
1988
+ )
1989
+ fence_out = d / "fence.md"
1990
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(fence_draft), out=str(fence_out), expand_pages=False))
1991
+ ftext = fence_out.read_text(encoding="utf-8")
1992
+ assert rc == 0, "fenced content must not break render"
1993
+ assert "## References\n\n[1] example entry." in ftext, "fenced References example was stripped"
1994
+ assert "[@pmid:21639808]" in ftext, "marker inside a fence was rewritten"
1995
+ assert "Tail kept." in ftext, "content after a fenced References example was truncated"
1996
+ assert "Real cite [1]." in ftext and ftext.rstrip().endswith("[1] Chapman PB, Hauschild A, Robert C, et al. Improved survival with vemurafenib in melanoma with BRAF V600E mutation. N Engl J Med. 2011;364(26):2507-16. DOI: 10.1056/nejmoa1103782. PMID: 21639808."), \
1997
+ f"real marker/References wrong:\n{ftext}"
1998
+ # mixed citation group: at least one cite-key + a non-citation token
1999
+ # must hard-fail (never silently drop the citation)
2000
+ mix = d / "mix.draft.md"
2001
+ mix.write_text("Mixed [@pmid:21639808; see note] group.\n", encoding="utf-8")
2002
+ mix_out = d / "mix.md"
2003
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(mix), out=str(mix_out), expand_pages=False))
2004
+ assert rc != 0 and not mix_out.exists(), "mixed citation group must exit 1 without writing output"
2005
+ check("render", st_render)
2006
+
2007
+ # ---- check: worker validity gate -----------------------------------
2008
+ def st_check():
2009
+ f = d / "ck.jsonl"
2010
+ f.write_text("", encoding="utf-8")
2011
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f)))
2012
+ assert rc == 0 and "0 record(s)" in out and "quarantined 0" in out, f"empty ledger must pass: {out}"
2013
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
2014
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f)))
2015
+ assert rc == 0 and "1 record(s)" in out and "pmid:21639808" in out and "; OK" in out, out
2016
+ bad = d / "ck2.jsonl"
2017
+ bad.write_text(json.dumps(_fixture_article()) + "\nnot json\n", encoding="utf-8")
2018
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(bad)))
2019
+ assert rc == 1 and "quarantined 1" in out and "FAIL" in out, f"quarantined ledger must fail: {out}"
2020
+ check("check", st_check)
2021
+
2022
+ # ---- check --markers: worker marker cross-validation gate ------------
2023
+ def st_check_markers():
2024
+ f = d / "ckm.jsonl"
2025
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
2026
+ good = d / "good.md"
2027
+ good.write_text(
2028
+ "Real [@pmid:21639808] plus its doi twin [@doi:10.1056/NEJMOA1103782].\n"
2029
+ "Prose [@home] and shape-ish [@gene:BRAF] stay verbatim.\n"
2030
+ "CI text [95% CI 78-89] and a link [1](http://x) are not citations.\n"
2031
+ "Fenced example:\n\n```\n[@pmid:99999999]\n```\n",
2032
+ encoding="utf-8",
2033
+ )
2034
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(good)]))
2035
+ assert rc == 0, f"resolvable markers must pass:\n{out}"
2036
+ assert "markers[good.md]: 2 citation group(s), 2 marker(s) resolved" in out
2037
+ assert "markers: 0 problem(s) across 1 file(s); OK" in out
2038
+ # unresolved marker -> exit 1
2039
+ bad = d / "bad.md"
2040
+ bad.write_text("Broken [@pmid:99999999] cite.\n", encoding="utf-8")
2041
+ buf = io.StringIO()
2042
+ with contextlib.redirect_stderr(buf):
2043
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(bad)]))
2044
+ assert rc == 1 and "unresolved marker pmid:99999999" in buf.getvalue(), out + buf.getvalue()
2045
+ # mixed group (cite + plain token) -> exit 1, mirroring render
2046
+ mix = d / "mix.md"
2047
+ mix.write_text("Mixed [@pmid:21639808; see note] group.\n", encoding="utf-8")
2048
+ buf = io.StringIO()
2049
+ with contextlib.redirect_stderr(buf):
2050
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(mix)]))
2051
+ assert rc == 1 and "mixed citation group" in buf.getvalue(), out + buf.getvalue()
2052
+ # missing marker file -> exit 1 (never a vacuous pass)
2053
+ rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(d / "nope.md")]))
2054
+ assert rc == 1 and "marker file not found" in out, out
2055
+ check("check-markers", st_check_markers)
2056
+
2057
+ # ---- render: hand-typed numeric bracket warning (non-fatal) ---------
2058
+ def st_render_handtyped_warning():
2059
+ f = d / "ht.jsonl"
2060
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
2061
+ draft = d / "ht.draft.md"
2062
+ draft.write_text(
2063
+ "Clean [@pmid:21639808] marker.\n\nBut a hand-typed [1] leak and [2, 3] too.\n\n"
2064
+ "Not citations: [95% CI 78-89], link [4](http://x), wiki [[5, 6]], fence:\n\n```\n[7]\n```\n",
2065
+ encoding="utf-8",
2066
+ )
2067
+ out_path = d / "ht.md"
2068
+ buf_err = io.StringIO()
2069
+ with contextlib.redirect_stderr(buf_err):
2070
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out_path), expand_pages=False))
2071
+ err = buf_err.getvalue()
2072
+ assert rc == 0, "hand-typed brackets must NOT fail render"
2073
+ assert "hand-typed numeric citation bracket" in err and "line(s) [3]" in err, err
2074
+ assert "[7]" not in err.replace("line(s)", ""), "fenced [7] must not warn"
2075
+ # clean draft: no warning at all
2076
+ clean = d / "ht2.draft.md"
2077
+ clean.write_text("Only [@pmid:21639808] here.\n", encoding="utf-8")
2078
+ buf_err2 = io.StringIO()
2079
+ with contextlib.redirect_stderr(buf_err2):
2080
+ rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(clean), out=str(d / "ht2.md"), expand_pages=False))
2081
+ assert rc == 0 and "hand-typed" not in buf_err2.getvalue(), buf_err2.getvalue()
2082
+ check("render-handtyped-warning", st_render_handtyped_warning)
2083
+
2084
+ # ---- pdb dataset support ---------------------------------------------
2085
+ def st_pdb_dataset():
2086
+ f = d / "pdb.jsonl"
2087
+ rec = {"type": "dataset", "ids": {"pdb": "6N65"}, "title": "KRAS G-quadruplex G16T mutant",
2088
+ "meta": {"method": "X-RAY DIFFRACTION", "resolution": "1.6 Å"},
2089
+ "provenance": [{"aspect": "pdb_aspect"}]}
2090
+ _capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(rec), stdin=False, aspect=None))
2091
+ led = read_ledger(f)
2092
+ assert "pdb:6N65" in led.by_key, "pdb:6N65 key not derived"
2093
+ _, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pdb:6N65", expand_pages=False, offset=0))
2094
+ line = out.splitlines()[0]
2095
+ assert "PDB structure 6N65: KRAS G-quadruplex G16T mutant." in line
2096
+ assert "[X-RAY DIFFRACTION]" in line and "Resolution: 1.6 Å." in line
2097
+ assert "https://www.rcsb.org/structure/6N65" in line
2098
+ check("pdb-dataset", st_pdb_dataset)
2099
+
1348
2100
  failed = [r for r in results if r[1] is not None]
1349
2101
  for name, err in results:
1350
2102
  print(f"{'PASS' if err is None else 'FAIL'} {name}" + (f": {err}" if err else ""))
@@ -1394,6 +2146,19 @@ def main() -> int:
1394
2146
  p.add_argument("file")
1395
2147
  p.set_defaults(fn=cmd_stats)
1396
2148
 
2149
+ p = sub.add_parser("render", help="Number cite-key markers in a draft and append the References section")
2150
+ p.add_argument("ledger", help="merged ledger file (evidence/sources.jsonl)")
2151
+ p.add_argument("draft", help="markdown draft authored with [@key] markers")
2152
+ p.add_argument("-o", "--out", required=True, help="output report path (never written on failure)")
2153
+ p.add_argument("--expand-pages", action="store_true", help="expand abbreviated page ranges (2507-16 -> 2507-2516)")
2154
+ p.set_defaults(fn=cmd_render)
2155
+
2156
+ p = sub.add_parser("check", help="Validate a ledger file (exit 1 on quarantined lines; with --markers also on unresolved/mixed markers)")
2157
+ p.add_argument("file")
2158
+ p.add_argument("--markers", nargs="+", metavar="MD",
2159
+ help="markdown file(s) whose [@key] markers must resolve in this ledger")
2160
+ p.set_defaults(fn=cmd_check)
2161
+
1397
2162
  p = sub.add_parser("selftest", help="Hermetic feature-matrix selftest (no network)")
1398
2163
  p.set_defaults(fn=lambda a: selftest())
1399
2164