opencode-bioresearcher 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/bioresearcher-dr-worker.md +31 -22
- package/connector-meta.json +1 -1
- package/package.json +1 -1
- package/skills/bioresearcher-deep-research/SKILL.md +190 -220
- package/skills/bioresearcher-deep-research/references/analysis-methods.md +40 -2
- package/skills/bioresearcher-deep-research/references/best-practices.md +5 -5
- package/skills/bioresearcher-deep-research/references/citations.md +40 -24
- package/skills/bioresearcher-deep-research/references/clinical-trials.md +1 -1
- package/skills/bioresearcher-deep-research/references/ensembl-pdb.md +2 -0
- package/skills/bioresearcher-deep-research/references/report-template.md +16 -14
- package/skills/bioresearcher-deep-research/references/tool-selection.md +2 -0
- package/skills/bioresearcher-deep-research/references/worker-protocol.md +130 -42
- package/skills/bioresearcher-deep-research/scripts/evidence-ledger.py +799 -34
- package/skills/bioresearcher-deep-research/scripts/vet-references.py +362 -28
|
@@ -10,6 +10,13 @@ Contract:
|
|
|
10
10
|
- Fill-missing-only merging: never overwrite a non-null stored value.
|
|
11
11
|
- Loud gaps: missing fields render as [MISSING field: ...]; unknown bib keys as
|
|
12
12
|
[MISSING record <key>] with a non-zero exit. Never fabricate.
|
|
13
|
+
- render is the single numbering authority: drafts cite [@key] markers; render
|
|
14
|
+
assigns numbers by first appearance, rewrites the markers, and appends the
|
|
15
|
+
References section from the ledger. Any unresolved key or [MISSING ...] entry
|
|
16
|
+
fails the render (exit 1) WITHOUT writing the output file.
|
|
17
|
+
- check validates a ledger end-to-end (exit 1 on quarantined lines; with
|
|
18
|
+
--markers also on unresolved/mixed [@key] markers in the given markdown
|
|
19
|
+
files - exit 1 iff anything this invocation validated failed).
|
|
13
20
|
- Verb banners: every subcommand prints "[evidence-ledger] <verb>: ..." so
|
|
14
21
|
test graders can anchor on deterministic stdout.
|
|
15
22
|
|
|
@@ -39,7 +46,7 @@ LEDGER_TYPES = {
|
|
|
39
46
|
"drug", "disease", "dataset", "web", "other",
|
|
40
47
|
}
|
|
41
48
|
KEY_NAMESPACES = (
|
|
42
|
-
"pmid", "doi", "pmcid", "nct", "patent", "geo", "sra", "gb",
|
|
49
|
+
"pmid", "doi", "pmcid", "nct", "patent", "geo", "sra", "gb", "pdb",
|
|
43
50
|
"gene", "clinvar", "chembl", "chebi", "unii",
|
|
44
51
|
"mondo", "doid", "omim", "efo", "url", "title",
|
|
45
52
|
)
|
|
@@ -63,6 +70,8 @@ ID_ALIASES = {
|
|
|
63
70
|
"geo_id": "geo",
|
|
64
71
|
"sra_id": "sra",
|
|
65
72
|
"gb_acc": "genbank",
|
|
73
|
+
"pdb_id": "pdb", # biomcp pdb
|
|
74
|
+
"rcsb_id": "pdb",
|
|
66
75
|
"disease_id": None, # context-dependent: prefix-sniffed below
|
|
67
76
|
}
|
|
68
77
|
|
|
@@ -74,6 +83,7 @@ FOLD_FIELDS = {
|
|
|
74
83
|
"patent": ["assignee", "status"],
|
|
75
84
|
"gene": ["symbol", "full_name"],
|
|
76
85
|
"variant": ["gene", "protein_change", "significance"],
|
|
86
|
+
"dataset": ["method", "experimental_method", "resolution"],
|
|
77
87
|
}
|
|
78
88
|
|
|
79
89
|
|
|
@@ -105,7 +115,7 @@ def _canonicalize_id_value(key: str, value: str) -> str:
|
|
|
105
115
|
if not value.isdigit():
|
|
106
116
|
raise ValueError(f"pmid must be digits, got {value!r}")
|
|
107
117
|
return value.lstrip("0") or "0"
|
|
108
|
-
if key in ("pmcid", "nct", "patent"):
|
|
118
|
+
if key in ("pmcid", "nct", "patent", "pdb"):
|
|
109
119
|
return value.upper()
|
|
110
120
|
return value
|
|
111
121
|
|
|
@@ -161,6 +171,14 @@ def normalize_record(raw, require_provenance_aspect=None) -> dict:
|
|
|
161
171
|
title = raw.get("title")
|
|
162
172
|
if title is not None:
|
|
163
173
|
title = str(title).strip() or None
|
|
174
|
+
if title is None and rtype != "article":
|
|
175
|
+
# biomcp record shapes often carry `name` instead of `title` (drugs,
|
|
176
|
+
# genes, diseases): fold it fill-only so bibliographies never render
|
|
177
|
+
# [MISSING field: title] for schema-shaped records. Articles keep
|
|
178
|
+
# their hard id requirement (a name-only article must not verify-gate).
|
|
179
|
+
name = raw.get("name")
|
|
180
|
+
if isinstance(name, str) and name.strip():
|
|
181
|
+
title = name.strip()
|
|
164
182
|
if not title and not ids:
|
|
165
183
|
raise ValueError("record needs a title or at least one id")
|
|
166
184
|
if raw.get("meta") is not None and not isinstance(raw.get("meta"), dict):
|
|
@@ -217,6 +235,11 @@ def _title_key(rtype: str, title) -> str:
|
|
|
217
235
|
|
|
218
236
|
|
|
219
237
|
def derive_dataset_key(ids: dict, title) -> str:
|
|
238
|
+
if ids.get("pdb"):
|
|
239
|
+
return f"pdb:{ids['pdb'].upper()}"
|
|
240
|
+
for k, v in ids.items():
|
|
241
|
+
if (k or "").lower() in ("pdb", "pdb_id"):
|
|
242
|
+
return f"pdb:{str(v).upper()}"
|
|
220
243
|
for v in ids.values():
|
|
221
244
|
v = str(v)
|
|
222
245
|
if v.upper().startswith("GSE"):
|
|
@@ -230,7 +253,7 @@ def derive_dataset_key(ids: dict, title) -> str:
|
|
|
230
253
|
for k, v in ids.items():
|
|
231
254
|
if (k or "").lower() in ("genbank", "gb", "accession"):
|
|
232
255
|
return f"gb:{v}"
|
|
233
|
-
raise ValueError("dataset record needs a geo (GSE/GDS), sra (SRR/SRP), or genbank accession id")
|
|
256
|
+
raise ValueError("dataset record needs a pdb, geo (GSE/GDS), sra (SRR/SRP), or genbank accession id")
|
|
234
257
|
|
|
235
258
|
|
|
236
259
|
def derive_web_key(ids: dict, title) -> str:
|
|
@@ -441,16 +464,46 @@ def append_quarantine(out_path: Path, entries: list) -> Path:
|
|
|
441
464
|
# Subcommands
|
|
442
465
|
# ---------------------------------------------------------------------------
|
|
443
466
|
|
|
467
|
+
def _parse_payload(text: str) -> list:
|
|
468
|
+
"""Parse JSON array, single JSON object, or newline-delimited JSON (.jsonl).
|
|
469
|
+
Strips optional markdown code fences, leading UTF-8 BOM, and blank lines."""
|
|
470
|
+
text = (text or "").lstrip("\ufeff")
|
|
471
|
+
lines = text.strip().splitlines()
|
|
472
|
+
if lines and lines[0].strip().startswith("```"):
|
|
473
|
+
lines = lines[1:]
|
|
474
|
+
if lines and lines[-1].strip().startswith("```"):
|
|
475
|
+
lines = lines[:-1]
|
|
476
|
+
cleaned = "\n".join(lines).strip()
|
|
477
|
+
if not cleaned:
|
|
478
|
+
raise ValueError("empty record input payload")
|
|
479
|
+
try:
|
|
480
|
+
payload = json.loads(cleaned)
|
|
481
|
+
return payload if isinstance(payload, list) else [payload]
|
|
482
|
+
except json.JSONDecodeError:
|
|
483
|
+
records = []
|
|
484
|
+
for line_no, line in enumerate(cleaned.splitlines(), start=1):
|
|
485
|
+
line = line.strip()
|
|
486
|
+
if not line:
|
|
487
|
+
continue
|
|
488
|
+
try:
|
|
489
|
+
item = json.loads(line)
|
|
490
|
+
except json.JSONDecodeError as e:
|
|
491
|
+
raise ValueError(f"malformed JSON at line {line_no}: {e}") from e
|
|
492
|
+
if isinstance(item, list):
|
|
493
|
+
records.extend(item)
|
|
494
|
+
else:
|
|
495
|
+
records.append(item)
|
|
496
|
+
return records
|
|
497
|
+
|
|
498
|
+
|
|
444
499
|
def _parse_incoming(args) -> list:
|
|
445
500
|
if args.stdin:
|
|
446
|
-
|
|
447
|
-
return payload if isinstance(payload, list) else [payload]
|
|
501
|
+
return _parse_payload(sys.stdin.read())
|
|
448
502
|
if getattr(args, "record", None) is None:
|
|
449
503
|
raise ValueError("provide a record JSON, @file, or --stdin")
|
|
450
504
|
if args.record.startswith("@"):
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
return [json.loads(args.record)]
|
|
505
|
+
return _parse_payload(Path(args.record[1:]).read_text(encoding="utf-8-sig"))
|
|
506
|
+
return _parse_payload(args.record)
|
|
454
507
|
|
|
455
508
|
|
|
456
509
|
def cmd_add(args) -> int:
|
|
@@ -462,16 +515,23 @@ def cmd_add(args) -> int:
|
|
|
462
515
|
qpath = append_quarantine(path, led.quarantined)
|
|
463
516
|
warn(f"{len(led.quarantined)} pre-existing malformed line(s) quarantined to {qpath} (excluded from rewrite)")
|
|
464
517
|
accepted = rejected = 0
|
|
518
|
+
derived_keys: list = []
|
|
465
519
|
for raw in _parse_incoming(args):
|
|
466
520
|
try:
|
|
467
|
-
|
|
521
|
+
rec = normalize_record(raw, require_provenance_aspect=args.aspect)
|
|
468
522
|
except (ValueError, TypeError) as e:
|
|
469
523
|
rejected += 1
|
|
470
524
|
warn(f"rejected record ({e}): {json.dumps(raw, ensure_ascii=False)[:200]}")
|
|
471
525
|
continue
|
|
526
|
+
led.upsert(rec)
|
|
527
|
+
derived_keys.append(rec["key"])
|
|
472
528
|
accepted += 1
|
|
473
529
|
led.write(path)
|
|
474
530
|
banner("add", f"{accepted} record(s) accepted, {rejected} rejected -> {path}")
|
|
531
|
+
if derived_keys:
|
|
532
|
+
# Echo the derived canonical keys so workers cite exactly what the
|
|
533
|
+
# ledger keyed (dataset/alias-derived keys are otherwise guesswork).
|
|
534
|
+
banner("add", f"derived keys: {', '.join(derived_keys)}")
|
|
475
535
|
return 0
|
|
476
536
|
|
|
477
537
|
|
|
@@ -510,14 +570,27 @@ def cmd_merge(args) -> int:
|
|
|
510
570
|
return 0
|
|
511
571
|
|
|
512
572
|
|
|
573
|
+
def _token_overlap(t1: str, t2: str) -> float:
|
|
574
|
+
toks1 = set(re.findall(r"[a-z0-9]{3,}", (t1 or "").lower()))
|
|
575
|
+
toks2 = set(re.findall(r"[a-z0-9]{3,}", (t2 or "").lower()))
|
|
576
|
+
if not toks1 or not toks2:
|
|
577
|
+
return 1.0
|
|
578
|
+
return len(toks1 & toks2) / min(len(toks1), len(toks2))
|
|
579
|
+
|
|
580
|
+
|
|
513
581
|
def _esummary_locator(doc: dict) -> dict:
|
|
514
582
|
m = re.search(r"\b(19\d\d|20\d\d)\b", str(doc.get("pubdate", "")))
|
|
515
583
|
year = m.group(1) if m else str(doc.get("sortpubdate") or "")[:4]
|
|
584
|
+
pages = str(doc.get("pages", "")).strip() or None
|
|
585
|
+
if not pages:
|
|
586
|
+
eloc = str(doc.get("elocationid", "")).strip()
|
|
587
|
+
if eloc and not eloc.lower().startswith("doi:"):
|
|
588
|
+
pages = re.sub(r"^(?:pii|articleno|article):\s*", "", eloc, flags=re.IGNORECASE) or None
|
|
516
589
|
return {
|
|
517
590
|
"year": year or None,
|
|
518
591
|
"volume": str(doc.get("volume", "")).strip() or None,
|
|
519
592
|
"issue": str(doc.get("issue", "")).strip() or None,
|
|
520
|
-
"pages":
|
|
593
|
+
"pages": pages,
|
|
521
594
|
}
|
|
522
595
|
|
|
523
596
|
|
|
@@ -535,6 +608,14 @@ def _esummary_doi(doc: dict):
|
|
|
535
608
|
return None
|
|
536
609
|
|
|
537
610
|
|
|
611
|
+
def _esummary_authors(doc: dict) -> list:
|
|
612
|
+
out = []
|
|
613
|
+
for a in doc.get("authors", []) or []:
|
|
614
|
+
if isinstance(a, dict) and a.get("name"):
|
|
615
|
+
out.append(str(a["name"]).strip())
|
|
616
|
+
return out
|
|
617
|
+
|
|
618
|
+
|
|
538
619
|
def cmd_verify(args) -> int:
|
|
539
620
|
path = Path(args.file)
|
|
540
621
|
led = read_ledger(path)
|
|
@@ -543,7 +624,9 @@ def cmd_verify(args) -> int:
|
|
|
543
624
|
if r.get("type") in verifiable and (r.get("ids") or {}).get("pmid") and not r.get("verified")})
|
|
544
625
|
docs = fetch_ncbi_summaries(pmids, timeout=args.timeout) if pmids else {}
|
|
545
626
|
now = _dt.datetime.now(_dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
546
|
-
filled = title_fixed = clean = unreachable = 0
|
|
627
|
+
filled = title_fixed = clean = unreachable = mismatches = 0
|
|
628
|
+
already = sum(1 for r in led.records()
|
|
629
|
+
if r.get("type") in verifiable and (r.get("ids") or {}).get("pmid") and r.get("verified"))
|
|
547
630
|
skipped = sum(1 for r in led.records() if r.get("type") not in verifiable)
|
|
548
631
|
for rec in led.records():
|
|
549
632
|
if rec.get("type") not in verifiable:
|
|
@@ -555,6 +638,28 @@ def cmd_verify(args) -> int:
|
|
|
555
638
|
if not doc:
|
|
556
639
|
unreachable += 1
|
|
557
640
|
continue
|
|
641
|
+
|
|
642
|
+
# Conflict detection
|
|
643
|
+
mismatch_reasons = []
|
|
644
|
+
ncbi_title = _esummary_title(doc)
|
|
645
|
+
if rec.get("title") and ncbi_title and len(ncbi_title) > 20:
|
|
646
|
+
overlap = _token_overlap(rec["title"], ncbi_title)
|
|
647
|
+
if overlap < 0.30:
|
|
648
|
+
mismatch_reasons.append(f"title overlap {overlap:.2f} < 0.30")
|
|
649
|
+
|
|
650
|
+
ncbi_doi = _esummary_doi(doc)
|
|
651
|
+
rec_doi = (rec.get("ids") or {}).get("doi")
|
|
652
|
+
if rec_doi and ncbi_doi:
|
|
653
|
+
if rec_doi.lower().rstrip(".") != ncbi_doi.lower().rstrip("."):
|
|
654
|
+
mismatch_reasons.append(f"doi conflict ({rec_doi} vs {ncbi_doi})")
|
|
655
|
+
|
|
656
|
+
if mismatch_reasons:
|
|
657
|
+
rec["verified"] = False
|
|
658
|
+
rec["verification_notes"] = f"mismatch: {'; '.join(mismatch_reasons)}"
|
|
659
|
+
warn(f"pmid:{pmid} metadata mismatch: {'; '.join(mismatch_reasons)}. Record left unverified.")
|
|
660
|
+
mismatches += 1
|
|
661
|
+
continue
|
|
662
|
+
|
|
558
663
|
changed = False
|
|
559
664
|
loc = _esummary_locator(doc)
|
|
560
665
|
for field in ("year", "volume", "issue", "pages"):
|
|
@@ -563,9 +668,8 @@ def cmd_verify(args) -> int:
|
|
|
563
668
|
rec["backfilled"].append(field)
|
|
564
669
|
changed = True
|
|
565
670
|
if not (rec.get("ids") or {}).get("doi"):
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
rec["ids"]["doi"] = doi
|
|
671
|
+
if ncbi_doi:
|
|
672
|
+
rec["ids"]["doi"] = ncbi_doi
|
|
569
673
|
rec["backfilled"].append("doi")
|
|
570
674
|
changed = True
|
|
571
675
|
if not rec.get("journal"):
|
|
@@ -575,16 +679,22 @@ def cmd_verify(args) -> int:
|
|
|
575
679
|
rec["backfilled"].append("journal")
|
|
576
680
|
changed = True
|
|
577
681
|
if not rec.get("title"):
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
rec["title"] = t
|
|
682
|
+
if ncbi_title:
|
|
683
|
+
rec["title"] = ncbi_title
|
|
581
684
|
rec["title_original"] = None
|
|
582
685
|
rec["backfilled"].append("title")
|
|
583
686
|
changed = True
|
|
584
687
|
title_fixed += 1
|
|
688
|
+
if not rec.get("authors"):
|
|
689
|
+
es_authors = _esummary_authors(doc)
|
|
690
|
+
if es_authors:
|
|
691
|
+
rec["authors"] = es_authors
|
|
692
|
+
rec["backfilled"].append("authors")
|
|
693
|
+
changed = True
|
|
585
694
|
rec["verified"] = True
|
|
586
695
|
rec["verified_source"] = "ncbi-esummary"
|
|
587
696
|
rec["verified_at"] = now
|
|
697
|
+
rec.pop("verification_notes", None)
|
|
588
698
|
if changed:
|
|
589
699
|
filled += 1
|
|
590
700
|
else:
|
|
@@ -594,11 +704,13 @@ def cmd_verify(args) -> int:
|
|
|
594
704
|
if led.quarantined:
|
|
595
705
|
qpath = append_quarantine(path, led.quarantined)
|
|
596
706
|
warn(f"{len(led.quarantined)} pre-existing malformed line(s) quarantined to {qpath} (excluded from rewrite)")
|
|
707
|
+
mismatch_note = f", {mismatches} metadata mismatch(es) (unverified)" if mismatches else ""
|
|
597
708
|
banner(
|
|
598
709
|
"verify",
|
|
599
710
|
f"{len(docs)} PubMed record(s) checked; {filled} backfilled, {title_fixed} title(s) set, "
|
|
600
|
-
f"{clean} verified clean, {unreachable} unreachable (fail-safe, left unverified); "
|
|
711
|
+
f"{clean} verified clean{mismatch_note}, {unreachable} unreachable (fail-safe, left unverified); "
|
|
601
712
|
f"{skipped} record(s) of unverified type(s) skipped (no verifier configured)"
|
|
713
|
+
+ (f"; {already} already verified (not rechecked)" if already else "")
|
|
602
714
|
+ ("; --apply written" if args.apply else " (dry-run, no changes written)"),
|
|
603
715
|
)
|
|
604
716
|
return 0
|
|
@@ -643,8 +755,20 @@ def vancouver_author(name: str) -> str:
|
|
|
643
755
|
surname, given = name, []
|
|
644
756
|
else:
|
|
645
757
|
surname, given = tokens[0], tokens[1:]
|
|
646
|
-
|
|
647
|
-
|
|
758
|
+
surname = surname.rstrip(",")
|
|
759
|
+
# Handle generational suffixes (Jr, Sr, 2nd, 3rd, II, III, IV)
|
|
760
|
+
suffix = ""
|
|
761
|
+
if given and given[-1].lower() in {"jr", "jr.", "sr", "sr.", "ii", "iii", "iv", "2nd", "3rd"}:
|
|
762
|
+
suffix = " " + given[-1].rstrip(".")
|
|
763
|
+
given = given[:-1]
|
|
764
|
+
cleaned_given = [t.replace(".", "") for t in given if t]
|
|
765
|
+
# Idempotency guard: if given is already uppercase initials (e.g. ["PB"], ["J.W."], ["SH"])
|
|
766
|
+
if len(cleaned_given) == 1 and cleaned_given[0].isupper() and cleaned_given[0].isalpha() and len(cleaned_given[0]) <= 4:
|
|
767
|
+
initials = cleaned_given[0]
|
|
768
|
+
else:
|
|
769
|
+
initials = "".join(t[0].upper() for t in given if t and t[0].isalpha())
|
|
770
|
+
res = f"{surname} {initials}" if initials else surname
|
|
771
|
+
return f"{res}{suffix}"
|
|
648
772
|
|
|
649
773
|
|
|
650
774
|
def _author_list(rec: dict, max_authors: int = 3) -> str:
|
|
@@ -653,7 +777,9 @@ def _author_list(rec: dict, max_authors: int = 3) -> str:
|
|
|
653
777
|
return "[MISSING field: authors]"
|
|
654
778
|
out = ", ".join(filter(None, (vancouver_author(str(a)) for a in authors[:max_authors])))
|
|
655
779
|
if len(authors) > max_authors:
|
|
656
|
-
out += ", et al"
|
|
780
|
+
out += ", et al."
|
|
781
|
+
else:
|
|
782
|
+
out = _close_segment(out)
|
|
657
783
|
return out
|
|
658
784
|
|
|
659
785
|
|
|
@@ -696,6 +822,25 @@ def _close(s: str) -> str:
|
|
|
696
822
|
return s if s.endswith(".") else s + "."
|
|
697
823
|
|
|
698
824
|
|
|
825
|
+
def _close_segment(s: str) -> str:
|
|
826
|
+
"""Ensure a metadata segment terminates with exactly one period, stripping any
|
|
827
|
+
pre-existing trailing period or whitespace (e.g. 'Tesaro, Inc.' -> 'Tesaro, Inc.')."""
|
|
828
|
+
if not s:
|
|
829
|
+
return ""
|
|
830
|
+
stripped = str(s).strip()
|
|
831
|
+
if not stripped:
|
|
832
|
+
return ""
|
|
833
|
+
return stripped.rstrip(".") + "."
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def _close_title(v) -> str:
|
|
837
|
+
"""Render a title and close with a single period (registry titles often
|
|
838
|
+
already end with one - never emit 'Title..')."""
|
|
839
|
+
if v in (None, ""):
|
|
840
|
+
return f"[MISSING field: title]"
|
|
841
|
+
return _close(str(v).strip().rstrip("."))
|
|
842
|
+
|
|
843
|
+
|
|
699
844
|
def render_article(rec: dict, expand: bool) -> str:
|
|
700
845
|
ids = rec.get("ids") or {}
|
|
701
846
|
title = _close(_need(rec, "title").strip())
|
|
@@ -716,16 +861,16 @@ def render_trial(rec: dict) -> str:
|
|
|
716
861
|
ids = rec.get("ids") or {}
|
|
717
862
|
meta = rec.get("meta") or {}
|
|
718
863
|
nct = ids.get("nct") or "[MISSING field: ids.nct]"
|
|
719
|
-
out = f"{nct}: {
|
|
864
|
+
out = f"{nct}: {_close_title(rec.get('title'))}"
|
|
720
865
|
phase = str(meta.get("phase") or "").strip().rstrip(".")
|
|
721
866
|
if phase:
|
|
722
867
|
# Workers copy phase verbatim from biomcp/CTgov ("Phase 2", "PHASE3",
|
|
723
868
|
# "2"): never double the prefix.
|
|
724
869
|
out += f" {phase}." if phase.lower().startswith("phase") else f" Phase {phase}."
|
|
725
870
|
if meta.get("sponsor"):
|
|
726
|
-
out += f" Sponsor: {meta['sponsor']}
|
|
871
|
+
out += f" Sponsor: {_close_segment(meta['sponsor'])}"
|
|
727
872
|
if meta.get("status"):
|
|
728
|
-
out += f" Status: {meta['status']}
|
|
873
|
+
out += f" Status: {_close_segment(meta['status'])}"
|
|
729
874
|
out += " " + (rec.get("url") or f"https://clinicaltrials.gov/study/{nct}")
|
|
730
875
|
return out
|
|
731
876
|
|
|
@@ -734,10 +879,11 @@ def render_patent(rec: dict) -> str:
|
|
|
734
879
|
ids = rec.get("ids") or {}
|
|
735
880
|
meta = rec.get("meta") or {}
|
|
736
881
|
num = ids.get("patent") or "[MISSING field: ids.patent]"
|
|
737
|
-
assignee = meta.get("assignee") or "
|
|
882
|
+
assignee = (meta.get("assignee") or "").strip()
|
|
883
|
+
assignee_str = f"{_close_segment(assignee)} " if assignee else "[MISSING field: meta.assignee]. "
|
|
738
884
|
status = f" ({meta['status']})" if meta.get("status") else ""
|
|
739
885
|
url = rec.get("url") or f"https://patents.google.com/patent/{num}"
|
|
740
|
-
return f"{
|
|
886
|
+
return f"{assignee_str}{_close_title(rec.get('title'))} {num}{status}. {url}"
|
|
741
887
|
|
|
742
888
|
|
|
743
889
|
def render_gene(rec: dict) -> str:
|
|
@@ -771,20 +917,31 @@ def render_drug(rec: dict) -> str:
|
|
|
771
917
|
"[MISSING field: ids.chembl|chebi|unii]")
|
|
772
918
|
ind = f" Indication: {meta['indication']}." if meta.get("indication") else ""
|
|
773
919
|
url = rec.get("url") or ""
|
|
774
|
-
return f"{
|
|
920
|
+
return f"{_close_title(rec.get('title'))}{ind} {dbid}. {url}".strip()
|
|
775
921
|
|
|
776
922
|
|
|
777
923
|
def render_disease(rec: dict) -> str:
|
|
778
924
|
ids = rec.get("ids") or {}
|
|
779
925
|
oid = next((f"{k.upper()}:{ids[k]}" for k in ("mondo", "doid", "omim", "efo") if ids.get(k)),
|
|
780
926
|
"[MISSING field: ids.mondo|doid|omim|efo]")
|
|
781
|
-
return f"{
|
|
927
|
+
return f"{_close_title(rec.get('title'))} {oid}. {rec.get('url') or ''}".strip()
|
|
782
928
|
|
|
783
929
|
|
|
784
930
|
def render_dataset(rec: dict) -> str:
|
|
785
931
|
ids = rec.get("ids") or {}
|
|
786
932
|
key = rec.get("key", "")
|
|
787
|
-
title = _need(rec, "title")
|
|
933
|
+
title = _need(rec, "title").strip().rstrip(".")
|
|
934
|
+
if key.startswith("pdb:"):
|
|
935
|
+
acc = ids.get("pdb") or key[len("pdb:"):]
|
|
936
|
+
meta = rec.get("meta") or {}
|
|
937
|
+
extras = []
|
|
938
|
+
method = meta.get("method") or meta.get("experimental_method")
|
|
939
|
+
if method:
|
|
940
|
+
extras.append(f"[{method}]")
|
|
941
|
+
if meta.get("resolution"):
|
|
942
|
+
extras.append(f"Resolution: {meta['resolution']}.")
|
|
943
|
+
extra_str = f" {' '.join(extras)}" if extras else ""
|
|
944
|
+
return f"PDB structure {acc}: {title}.{extra_str} https://www.rcsb.org/structure/{acc}"
|
|
788
945
|
if key.startswith("geo:"):
|
|
789
946
|
acc = ids.get("geo") or ids.get("accession") or key[len("geo:"):]
|
|
790
947
|
return f"GEO series {acc}: {title}. https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc={acc}"
|
|
@@ -797,10 +954,12 @@ def render_dataset(rec: dict) -> str:
|
|
|
797
954
|
|
|
798
955
|
def render_web(rec: dict) -> str:
|
|
799
956
|
meta = rec.get("meta") or {}
|
|
800
|
-
updated = f" Updated {meta['updated']}
|
|
957
|
+
updated = f" Updated {_close_segment(meta['updated'])}" if meta.get("updated") else ""
|
|
801
958
|
url = rec.get("url") or (rec.get("ids") or {}).get("url") or "[MISSING field: url]"
|
|
802
959
|
accessed = meta.get("accessed") or "[MISSING field: meta.accessed]"
|
|
803
|
-
|
|
960
|
+
org = (meta.get("organization") or "").strip()
|
|
961
|
+
org_str = f" {_close_segment(org)}" if org else " [MISSING field: meta.organization]."
|
|
962
|
+
return f"{_close_title(rec.get('title'))}{org_str}{updated} {url}. Accessed: {accessed}."
|
|
804
963
|
|
|
805
964
|
|
|
806
965
|
def render_other(rec: dict) -> str:
|
|
@@ -903,6 +1062,299 @@ def cmd_stats(args) -> int:
|
|
|
903
1062
|
return 0
|
|
904
1063
|
|
|
905
1064
|
|
|
1065
|
+
# ---------------------------------------------------------------------------
|
|
1066
|
+
# Cite-key rendering (draft -> numbered report; the single numbering authority)
|
|
1067
|
+
# ---------------------------------------------------------------------------
|
|
1068
|
+
|
|
1069
|
+
# A cite-key marker: [@ns:value] or [@a; @b]. A bracket is a citation group
|
|
1070
|
+
# only if EVERY non-empty token is a recognized namespace with a shape-valid
|
|
1071
|
+
# value; anything else (prose [@home], pandoc [@Chapman2011], [@gene:BRAF]
|
|
1072
|
+
# symbols) passes through verbatim.
|
|
1073
|
+
MARKER_RE = re.compile(r"\[@([^\[\]]+)\]")
|
|
1074
|
+
|
|
1075
|
+
# Canonical value shapes per key namespace (see KEY_NAMESPACES). Namespaces
|
|
1076
|
+
# without an entry accept any non-empty value.
|
|
1077
|
+
NS_VALUE_SHAPES = {
|
|
1078
|
+
"pmid": re.compile(r"^\d+$"),
|
|
1079
|
+
"doi": re.compile(r"^10\.\S+$", re.IGNORECASE),
|
|
1080
|
+
"pmcid": re.compile(r"^PMC\d+$", re.IGNORECASE),
|
|
1081
|
+
"nct": re.compile(r"^NCT\d+$", re.IGNORECASE),
|
|
1082
|
+
"patent": re.compile(r"^[A-Z]{2}\d+", re.IGNORECASE),
|
|
1083
|
+
"geo": re.compile(r"^GS[ED]\d+$", re.IGNORECASE),
|
|
1084
|
+
"sra": re.compile(r"^SR[RP]\d+$", re.IGNORECASE),
|
|
1085
|
+
"gb": re.compile(r"^[A-Z]{2,}\d+(\.\d+)?$", re.IGNORECASE),
|
|
1086
|
+
"pdb": re.compile(r"^[0-9][a-z0-9]{3}$", re.IGNORECASE),
|
|
1087
|
+
"gene": re.compile(r"^\d+$"),
|
|
1088
|
+
"clinvar": re.compile(r"^\d+$"),
|
|
1089
|
+
"chembl": re.compile(r"^CHEMBL\d+$", re.IGNORECASE),
|
|
1090
|
+
"chebi": re.compile(r"^CHEBI[:_ ]?\d+$", re.IGNORECASE),
|
|
1091
|
+
"unii": re.compile(r"^[A-Z0-9]{4,10}$", re.IGNORECASE),
|
|
1092
|
+
"mondo": re.compile(r"^MONDO[:_ ]?\d+$", re.IGNORECASE),
|
|
1093
|
+
"doid": re.compile(r"^DOID[:_ ]?\d+$", re.IGNORECASE),
|
|
1094
|
+
"omim": re.compile(r"^\d{5,7}$"),
|
|
1095
|
+
"efo": re.compile(r"^[A-Z]{2,}[_:]?\d+", re.IGNORECASE),
|
|
1096
|
+
"url": re.compile(r"^[0-9a-f]{16}$", re.IGNORECASE),
|
|
1097
|
+
"title": re.compile(r"^[0-9a-f]{16}$", re.IGNORECASE),
|
|
1098
|
+
}
|
|
1099
|
+
|
|
1100
|
+
REFS_STRIP_RE = re.compile(
|
|
1101
|
+
r"(?ims)^#{1,6}[ \t]*(?:references?|bibliography|literature cited|citations)\b[^\n]*\n"
|
|
1102
|
+
r".*?(?=\n#{1,6}[ \t]*\S|\Z)"
|
|
1103
|
+
)
|
|
1104
|
+
|
|
1105
|
+
# Hand-typed numeric citation brackets (render's input is authored with
|
|
1106
|
+
# [@key] markers, so any [N]/[N, M]/[N-M] bracket in a draft is suspect).
|
|
1107
|
+
# Negative lookarounds exclude markdown links [1](url), reference defs [1]:,
|
|
1108
|
+
# and wikilinks [[1,2]].
|
|
1109
|
+
HAND_TYPED_NUM_RE = re.compile(r"(?<!\[)\[\d{1,3}(?:\s*[,\u2013\-]\s*\d{1,3})*\](?![:(\[])")
|
|
1110
|
+
|
|
1111
|
+
CODE_BLOCK_RE = re.compile(r"(?ms)^(?:```|~~~)[^\n]*\n.*?^(?:```|~~~)[ \t]*$")
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def mask_code_blocks(text: str) -> str:
|
|
1115
|
+
"""Fenced code blocks -> same-length newline filler (offsets preserved), so
|
|
1116
|
+
section detection and marker scans never fire on example/documentation
|
|
1117
|
+
content inside fences."""
|
|
1118
|
+
return CODE_BLOCK_RE.sub(lambda m: "\n" * (m.end() - m.start()), text)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def classify_token(token: str):
|
|
1122
|
+
"""Classify a marker token: ('cite', ns, value) | ('shape', ns, value) | ('plain', tok, None)."""
|
|
1123
|
+
tok = token.strip()
|
|
1124
|
+
if tok.startswith("@"):
|
|
1125
|
+
tok = tok[1:].strip()
|
|
1126
|
+
if ":" not in tok:
|
|
1127
|
+
return ("plain", tok, None)
|
|
1128
|
+
ns, _, value = tok.partition(":")
|
|
1129
|
+
ns, value = ns.strip().lower(), value.strip()
|
|
1130
|
+
if not value or ns not in KEY_NAMESPACES:
|
|
1131
|
+
return ("plain", tok, None)
|
|
1132
|
+
shape = NS_VALUE_SHAPES.get(ns)
|
|
1133
|
+
if shape is None or shape.match(value):
|
|
1134
|
+
return ("cite", ns, value)
|
|
1135
|
+
return ("shape", ns, value)
|
|
1136
|
+
|
|
1137
|
+
|
|
1138
|
+
def _canonical_marker_key(ns: str, value: str) -> str:
|
|
1139
|
+
try:
|
|
1140
|
+
return f"{ns}:{_canonicalize_id_value(ns, value)}"
|
|
1141
|
+
except ValueError:
|
|
1142
|
+
return f"{ns}:{value}"
|
|
1143
|
+
|
|
1144
|
+
|
|
1145
|
+
def resolve_key(led: Ledger, ns: str, value: str):
|
|
1146
|
+
"""Resolve a marker key to a ledger key: direct -> secondary-id twin -> None."""
|
|
1147
|
+
cand = _canonical_marker_key(ns, value)
|
|
1148
|
+
if cand in led.by_key:
|
|
1149
|
+
return cand
|
|
1150
|
+
if ns in ("doi", "pmcid"):
|
|
1151
|
+
# merge promotes doi:/pmcid: twins to their pmid key; the marker may
|
|
1152
|
+
# legitimately cite the pre-promotion namespace.
|
|
1153
|
+
canon_val = cand.split(":", 1)[1].lower()
|
|
1154
|
+
twin = led.sec_index.get(f"{ns}:{canon_val}")
|
|
1155
|
+
if twin and twin in led.by_key:
|
|
1156
|
+
return twin
|
|
1157
|
+
return None
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
def _compress_numbers(nums: list) -> str:
|
|
1161
|
+
"""[1,2,3,5] -> '1-3, 5'; [1,2] -> '1, 2'."""
|
|
1162
|
+
nums = sorted(set(nums))
|
|
1163
|
+
parts, i = [], 0
|
|
1164
|
+
while i < len(nums):
|
|
1165
|
+
j = i
|
|
1166
|
+
while j + 1 < len(nums) and nums[j + 1] == nums[j] + 1:
|
|
1167
|
+
j += 1
|
|
1168
|
+
if j - i >= 2:
|
|
1169
|
+
parts.append(f"{nums[i]}-{nums[j]}")
|
|
1170
|
+
elif j == i + 1:
|
|
1171
|
+
parts.append(f"{nums[i]}, {nums[j]}")
|
|
1172
|
+
else:
|
|
1173
|
+
parts.append(str(nums[i]))
|
|
1174
|
+
i = j + 1
|
|
1175
|
+
return ", ".join(parts)
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
def cmd_render(args) -> int:
|
|
1179
|
+
led = read_ledger(Path(args.ledger))
|
|
1180
|
+
text = Path(args.draft).read_text(encoding="utf-8-sig")
|
|
1181
|
+
out_path = Path(args.out)
|
|
1182
|
+
|
|
1183
|
+
# Strip pre-existing References-like sections, fence-aware: detect on a
|
|
1184
|
+
# code-block-masked copy (offsets preserved), cut from the real text in
|
|
1185
|
+
# reverse so earlier spans stay valid.
|
|
1186
|
+
masked = mask_code_blocks(text)
|
|
1187
|
+
had_entries = False
|
|
1188
|
+
for m in reversed(list(REFS_STRIP_RE.finditer(masked))):
|
|
1189
|
+
chunk = text[m.start():m.end()]
|
|
1190
|
+
if re.search(r"(?m)^\s*[-*]?\s*\[\d+\]", chunk):
|
|
1191
|
+
had_entries = True
|
|
1192
|
+
if "[@" in chunk:
|
|
1193
|
+
warn("a pre-existing References-like section contained [@key] marker(s); "
|
|
1194
|
+
"the section was stripped - cite those sources in the body instead")
|
|
1195
|
+
text = text[:m.start()] + text[m.end():]
|
|
1196
|
+
body = text.rstrip() + "\n"
|
|
1197
|
+
|
|
1198
|
+
failures: list = []
|
|
1199
|
+
numbers: dict = {}
|
|
1200
|
+
order: list = []
|
|
1201
|
+
|
|
1202
|
+
def render_marker(bracket: str) -> str:
|
|
1203
|
+
inner = bracket[2:-1] if bracket.endswith("]") else bracket[2:]
|
|
1204
|
+
tokens = [t for t in inner.split(";") if t.strip()]
|
|
1205
|
+
kinds = [classify_token(t) for t in tokens]
|
|
1206
|
+
cites = [k for k in kinds if k[0] == "cite"]
|
|
1207
|
+
if not tokens or not cites:
|
|
1208
|
+
for kind, ns, _ in kinds:
|
|
1209
|
+
if kind == "shape":
|
|
1210
|
+
warn(f"bracket {bracket!r} uses the {ns}: namespace but its value is not shape-valid; left verbatim")
|
|
1211
|
+
return bracket
|
|
1212
|
+
if any(k[0] != "cite" for k in kinds):
|
|
1213
|
+
# A group with at least one real cite-key must not silently drop
|
|
1214
|
+
# its non-citation tokens - that would lose citations quietly.
|
|
1215
|
+
failures.append(f"mixed citation group {bracket!r}: every token must be a cite-key")
|
|
1216
|
+
return bracket
|
|
1217
|
+
keys = []
|
|
1218
|
+
for _, ns, value in cites:
|
|
1219
|
+
key = resolve_key(led, ns, value)
|
|
1220
|
+
if key is None:
|
|
1221
|
+
suggestions = [k for k in sorted(led.by_key) if k.startswith(ns + ":")][:5]
|
|
1222
|
+
failures.append(
|
|
1223
|
+
f"unknown citation key {ns}:{value}"
|
|
1224
|
+
+ (f" (did you mean: {', '.join(suggestions)}?)" if suggestions else "")
|
|
1225
|
+
)
|
|
1226
|
+
continue
|
|
1227
|
+
keys.append(key)
|
|
1228
|
+
if failures:
|
|
1229
|
+
return bracket
|
|
1230
|
+
for key in keys:
|
|
1231
|
+
if key not in numbers:
|
|
1232
|
+
numbers[key] = len(numbers) + 1
|
|
1233
|
+
order.append(key)
|
|
1234
|
+
return "[" + _compress_numbers([numbers[k] for k in keys]) + "]"
|
|
1235
|
+
|
|
1236
|
+
# Marker substitution, fence-aware: scan the masked copy (offsets equal),
|
|
1237
|
+
# splice replacements into the real body.
|
|
1238
|
+
masked_body = mask_code_blocks(body)
|
|
1239
|
+
# Hand-typed numeric brackets in the DRAFT are the residual leak class:
|
|
1240
|
+
# render owns numbering, so warn loudly (non-fatal - prose ranges like
|
|
1241
|
+
# [140, 155] are legitimate; links/wikilinks/reference defs are excluded).
|
|
1242
|
+
hand_typed = [m.start() for m in HAND_TYPED_NUM_RE.finditer(masked_body)]
|
|
1243
|
+
if hand_typed:
|
|
1244
|
+
lines = sorted({masked_body.count("\n", 0, pos) + 1 for pos in hand_typed[:5]})
|
|
1245
|
+
warn(f"draft contains {len(hand_typed)} hand-typed numeric citation bracket(s) "
|
|
1246
|
+
f"(first at line(s) {lines}): render owns numbering - remove hand-typed [N] brackets from the draft")
|
|
1247
|
+
parts: list = []
|
|
1248
|
+
last = 0
|
|
1249
|
+
for m in MARKER_RE.finditer(masked_body):
|
|
1250
|
+
parts.append(body[last:m.start()])
|
|
1251
|
+
parts.append(render_marker(body[m.start():m.end()]))
|
|
1252
|
+
last = m.end()
|
|
1253
|
+
parts.append(body[last:])
|
|
1254
|
+
rendered_body = "".join(parts)
|
|
1255
|
+
|
|
1256
|
+
# Double-render guard: an already-rendered document has no markers left.
|
|
1257
|
+
if not order and had_entries:
|
|
1258
|
+
banner("render", "FAILED: no [@key] citation markers found, but a References section with entries was present "
|
|
1259
|
+
"(already-rendered document?); nothing written")
|
|
1260
|
+
return 1
|
|
1261
|
+
if failures:
|
|
1262
|
+
for f in failures:
|
|
1263
|
+
warn(f"unresolved citation: {f}")
|
|
1264
|
+
banner("render", f"FAILED: {len(failures)} unresolved citation key(s); nothing written")
|
|
1265
|
+
return 1
|
|
1266
|
+
|
|
1267
|
+
entries = []
|
|
1268
|
+
missing = []
|
|
1269
|
+
for key in order:
|
|
1270
|
+
entry = render_record(led.by_key[key], args.expand_pages)
|
|
1271
|
+
if "[MISSING" in entry:
|
|
1272
|
+
missing.append(key)
|
|
1273
|
+
entries.append(f"[{numbers[key]}] {entry}")
|
|
1274
|
+
if missing:
|
|
1275
|
+
for key in missing:
|
|
1276
|
+
warn(f"record {key} would render with [MISSING ...] markers (incomplete ledger record)")
|
|
1277
|
+
banner("render", f"FAILED: {len(missing)} record(s) render with [MISSING ...] gaps "
|
|
1278
|
+
f"({', '.join(missing[:5])}{' ...' if len(missing) > 5 else ''}); nothing written")
|
|
1279
|
+
return 1
|
|
1280
|
+
|
|
1281
|
+
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1282
|
+
refs = "\n".join(entries) if entries else "(no cited sources)"
|
|
1283
|
+
out_path.write_text(rendered_body + "\n## References\n\n" + refs + "\n", encoding="utf-8")
|
|
1284
|
+
banner("render", f"{len(order)} citation(s) numbered, {len(entries)} reference(s) rendered from {args.ledger} -> {args.out}")
|
|
1285
|
+
return 0
|
|
1286
|
+
|
|
1287
|
+
|
|
1288
|
+
def _check_markers(led: Ledger, paths: list) -> int:
|
|
1289
|
+
"""Cross-validate [@key] markers in markdown files against the ledger.
|
|
1290
|
+
|
|
1291
|
+
Mirrors render's group semantics exactly: groups with zero cite tokens are
|
|
1292
|
+
prose (ignored); all-cite groups must fully resolve (direct or sec_index);
|
|
1293
|
+
groups mixing >=1 cite token with any non-cite token are failures;
|
|
1294
|
+
shape-kind tokens (recognized namespace, invalid shape) warn only.
|
|
1295
|
+
Returns the number of failures; a missing marker file counts as one
|
|
1296
|
+
(never a vacuous pass)."""
|
|
1297
|
+
failures = 0
|
|
1298
|
+
for raw in paths:
|
|
1299
|
+
p = Path(raw)
|
|
1300
|
+
if not p.is_file():
|
|
1301
|
+
banner("check", f"markers: marker file not found: {p}")
|
|
1302
|
+
failures += 1
|
|
1303
|
+
continue
|
|
1304
|
+
text = p.read_text(encoding="utf-8-sig")
|
|
1305
|
+
masked = mask_code_blocks(text)
|
|
1306
|
+
groups = resolved = 0
|
|
1307
|
+
for m in MARKER_RE.finditer(masked):
|
|
1308
|
+
tokens = [t for t in m.group(1).split(";") if t.strip()]
|
|
1309
|
+
kinds = [classify_token(t) for t in tokens]
|
|
1310
|
+
cites = [k for k in kinds if k[0] == "cite"]
|
|
1311
|
+
if not tokens or not cites:
|
|
1312
|
+
continue # prose bracket - render leaves it verbatim
|
|
1313
|
+
line = masked.count("\n", 0, m.start()) + 1
|
|
1314
|
+
groups += 1
|
|
1315
|
+
if any(k[0] != "cite" for k in kinds):
|
|
1316
|
+
warn(f"{p.name}:{line}: mixed citation group {m.group(0)!r}: every token must be a cite-key")
|
|
1317
|
+
failures += 1
|
|
1318
|
+
continue
|
|
1319
|
+
for kind, ns, value in cites:
|
|
1320
|
+
if kind == "shape":
|
|
1321
|
+
warn(f"{p.name}:{line}: bracket {m.group(0)!r} uses the {ns}: namespace but its value is not shape-valid (left verbatim)")
|
|
1322
|
+
continue
|
|
1323
|
+
key = resolve_key(led, ns, value)
|
|
1324
|
+
if key is None:
|
|
1325
|
+
warn(f"{p.name}:{line}: unresolved marker {ns}:{value} (no matching ledger record)")
|
|
1326
|
+
failures += 1
|
|
1327
|
+
else:
|
|
1328
|
+
resolved += 1
|
|
1329
|
+
print(f"markers[{p.name}]: {groups} citation group(s), {resolved} marker(s) resolved")
|
|
1330
|
+
return failures
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def cmd_check(args) -> int:
|
|
1334
|
+
markers = getattr(args, "markers", None) or []
|
|
1335
|
+
led = read_ledger(Path(args.file))
|
|
1336
|
+
types: dict = {}
|
|
1337
|
+
for rec in led.records():
|
|
1338
|
+
types[rec.get("type", "?")] = types.get(rec.get("type", "?"), 0) + 1
|
|
1339
|
+
for k in sorted(led.by_key):
|
|
1340
|
+
print(k)
|
|
1341
|
+
status = "OK" if not led.quarantined else "FAIL"
|
|
1342
|
+
banner("check", f"{len(led.by_key)} record(s) across {len(types)} type(s) "
|
|
1343
|
+
f"({', '.join(f'{t}:{c}' for t, c in sorted(types.items())) or 'none'}); "
|
|
1344
|
+
f"quarantined {len(led.quarantined)}; {status}")
|
|
1345
|
+
if led.quarantined:
|
|
1346
|
+
for q in led.quarantined:
|
|
1347
|
+
warn(f"quarantined {q.get('file')}:{q.get('line')}: {q.get('error')}")
|
|
1348
|
+
return 1
|
|
1349
|
+
if markers:
|
|
1350
|
+
marker_failures = _check_markers(led, markers)
|
|
1351
|
+
banner("check", f"markers: {marker_failures} problem(s) across {len(markers)} file(s); "
|
|
1352
|
+
f"{'FAIL' if marker_failures else 'OK'}")
|
|
1353
|
+
if marker_failures:
|
|
1354
|
+
return 1
|
|
1355
|
+
return 0
|
|
1356
|
+
|
|
1357
|
+
|
|
906
1358
|
# ---------------------------------------------------------------------------
|
|
907
1359
|
# Hermetic selftest (CI; no network)
|
|
908
1360
|
# ---------------------------------------------------------------------------
|
|
@@ -1057,6 +1509,7 @@ def selftest() -> int:
|
|
|
1057
1509
|
"articleids": [{"idtype": "doi", "value": "10.1111/cas.70480"}]},
|
|
1058
1510
|
"99900001": {"pubdate": "2011 Jun 30", "volume": "364", "issue": "26", "pages": "2507-16",
|
|
1059
1511
|
"source": "N Engl J Med", "title": "Mocked title for hint record.",
|
|
1512
|
+
"authors": [{"name": "Chapman PB", "authtype": "Author"}, {"name": "Hauschild A", "authtype": "Author"}],
|
|
1060
1513
|
"articleids": [{"idtype": "doi", "value": "10.1056/NEJMoa1103782"}]},
|
|
1061
1514
|
}
|
|
1062
1515
|
mod = sys.modules[__name__]
|
|
@@ -1073,9 +1526,48 @@ def selftest() -> int:
|
|
|
1073
1526
|
h = led.by_key["pmid:99900001"]
|
|
1074
1527
|
assert h["title"] == "Mocked title for hint record", "hint title not backfilled"
|
|
1075
1528
|
assert h["volume"] == "364" and h["pages"] == "2507-16", "locators not backfilled"
|
|
1076
|
-
assert "
|
|
1529
|
+
assert h["authors"] == ["Chapman PB", "Hauschild A"], f"authors not backfilled: {h['authors']}"
|
|
1530
|
+
assert "volume" in h["backfilled"] and "title" in h["backfilled"] and "authors" in h["backfilled"]
|
|
1077
1531
|
check("verify-backfill", st_verify_backfill)
|
|
1078
1532
|
|
|
1533
|
+
# ---- verify mismatch (conflict detection) --------------------------------
|
|
1534
|
+
def st_verify_mismatch():
|
|
1535
|
+
f = d / "vm.jsonl"
|
|
1536
|
+
# Title mismatch: completely unrelated title put into article record
|
|
1537
|
+
bad_title = _fixture_article(pmid="540362", ids={"pmid": "540362", "doi": "10.1248/cpb.27.1942"},
|
|
1538
|
+
title="Unrelated Subject Matter on Plant Photosynthesis")
|
|
1539
|
+
# DOI mismatch
|
|
1540
|
+
bad_doi = _fixture_article(pmid="21639808", ids={"pmid": "21639808", "doi": "10.1000/wrong.doi"},
|
|
1541
|
+
title="Improved survival with vemurafenib in melanoma with BRAF V600E mutation")
|
|
1542
|
+
f.write_text(json.dumps(bad_title) + "\n" + json.dumps(bad_doi) + "\n", encoding="utf-8")
|
|
1543
|
+
docs = {
|
|
1544
|
+
"540362": {"pubdate": "1979", "volume": "27", "issue": "8", "pages": "1942-4",
|
|
1545
|
+
"source": "Chem Pharm Bull (Tokyo)",
|
|
1546
|
+
"title": "Solution structure of alpha-conotoxin EI from Conus ermineus.",
|
|
1547
|
+
"articleids": [{"idtype": "doi", "value": "10.1248/cpb.27.1942"}]},
|
|
1548
|
+
"21639808": {"pubdate": "2011", "volume": "364", "issue": "26", "pages": "2507-16",
|
|
1549
|
+
"source": "N Engl J Med",
|
|
1550
|
+
"title": "Improved survival with vemurafenib in melanoma with BRAF V600E mutation.",
|
|
1551
|
+
"articleids": [{"idtype": "doi", "value": "10.1056/nejmoa1103782"}]},
|
|
1552
|
+
}
|
|
1553
|
+
mod = sys.modules[__name__]
|
|
1554
|
+
original = mod.fetch_ncbi_summaries
|
|
1555
|
+
mod.fetch_ncbi_summaries = lambda pmids, timeout=15.0: docs
|
|
1556
|
+
try:
|
|
1557
|
+
rc, out = _capture(cmd_verify, argparse.Namespace(file=str(f), apply=True, timeout=1.0))
|
|
1558
|
+
finally:
|
|
1559
|
+
mod.fetch_ncbi_summaries = original
|
|
1560
|
+
assert rc == 0
|
|
1561
|
+
assert "2 metadata mismatch(es) (unverified)" in out
|
|
1562
|
+
led = read_ledger(f)
|
|
1563
|
+
rec_title = led.by_key["pmid:540362"]
|
|
1564
|
+
assert rec_title["verified"] is False, "title mismatch record must NOT be verified"
|
|
1565
|
+
assert "mismatch: title overlap" in rec_title.get("verification_notes", "")
|
|
1566
|
+
rec_doi = led.by_key["pmid:21639808"]
|
|
1567
|
+
assert rec_doi["verified"] is False, "doi mismatch record must NOT be verified"
|
|
1568
|
+
assert "mismatch: doi conflict" in rec_doi.get("verification_notes", "")
|
|
1569
|
+
check("verify-mismatch", st_verify_mismatch)
|
|
1570
|
+
|
|
1079
1571
|
# ---- bib -------------------------------------------------------------------
|
|
1080
1572
|
def st_bib():
|
|
1081
1573
|
# Vancouver initials: standard, particle surnames, group passthrough
|
|
@@ -1085,6 +1577,11 @@ def selftest() -> int:
|
|
|
1085
1577
|
assert vancouver_author("World Health Organization") == "World Health Organization"
|
|
1086
1578
|
assert vancouver_author("Li Jiang") == "Li J"
|
|
1087
1579
|
assert vancouver_author("WHO") == "WHO"
|
|
1580
|
+
# Idempotency of vancouver_author on pre-formatted initials & punctuation
|
|
1581
|
+
assert vancouver_author("Chapman PB") == "Chapman PB", vancouver_author("Chapman PB")
|
|
1582
|
+
assert vancouver_author("Taniguchi SH") == "Taniguchi SH", vancouver_author("Taniguchi SH")
|
|
1583
|
+
assert vancouver_author("Schmidberger, J.W.") == "Schmidberger JW", vancouver_author("Schmidberger, J.W.")
|
|
1584
|
+
assert vancouver_author("Smith JA Jr") == "Smith JA Jr", vancouver_author("Smith JA Jr")
|
|
1088
1585
|
# expand_pages guards
|
|
1089
1586
|
assert expand_pages("2507-16") == "2507-2516"
|
|
1090
1587
|
assert expand_pages("2507-2516") == "2507-2516"
|
|
@@ -1095,7 +1592,7 @@ def selftest() -> int:
|
|
|
1095
1592
|
rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:21639808", expand_pages=False, offset=0))
|
|
1096
1593
|
assert rc == 0
|
|
1097
1594
|
first = out.splitlines()[0]
|
|
1098
|
-
assert "Chapman PB, Hauschild A, Robert C, et al" in first, f"Vancouver initials wrong: {first}"
|
|
1595
|
+
assert "Chapman PB, Hauschild A, Robert C, et al." in first, f"Vancouver initials / period wrong: {first}"
|
|
1099
1596
|
assert "2011;364(26):2507-16" in first, f"locator wrong: {first}"
|
|
1100
1597
|
assert "PMID: 21639808." in first and "DOI: 10.1056/nejmoa1103782." in first, first
|
|
1101
1598
|
rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:21639808", expand_pages=True, offset=0))
|
|
@@ -1108,7 +1605,7 @@ def selftest() -> int:
|
|
|
1108
1605
|
_, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:42487519", expand_pages=False, offset=0))
|
|
1109
1606
|
line = out.splitlines()[0]
|
|
1110
1607
|
assert "Cancer Sci. 2026." in line and "364" not in line and "DOI: 10.1111/cas.70480." in line, f"epub form wrong: {line}"
|
|
1111
|
-
assert "Takahashi M, Taniguchi SH" in line, "
|
|
1608
|
+
assert "Takahashi M, Taniguchi SH." in line, f"author list period wrong: {line}"
|
|
1112
1609
|
# unknown key -> loud + non-zero
|
|
1113
1610
|
rc, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pmid:42487519,pmid:nope", expand_pages=False, offset=0))
|
|
1114
1611
|
assert rc != 0, "unknown key must exit non-zero"
|
|
@@ -1314,6 +1811,20 @@ def selftest() -> int:
|
|
|
1314
1811
|
{"type": "article", "title": "no ids", "ids": {}}]
|
|
1315
1812
|
rc, out = add_stdin(f, json.dumps(mixed))
|
|
1316
1813
|
assert rc == 0 and "1 record(s) accepted, 1 rejected" in out, f"mixed batch accounting wrong: {out}"
|
|
1814
|
+
# newline-delimited JSON (.jsonl) batch via --stdin
|
|
1815
|
+
jsonl_batch = (json.dumps(_fixture_article(pmid="10000006", doi="10.1000/b6", title="Batch six", ids={"pmid": "10000006", "doi": "10.1000/b6", "pmcid": "PMC1000006"})) + "\n" +
|
|
1816
|
+
json.dumps(_fixture_article(pmid="10000007", doi="10.1000/b7", title="Batch seven", ids={"pmid": "10000007", "doi": "10.1000/b7", "pmcid": "PMC1000007"})) + "\n")
|
|
1817
|
+
rc, out = add_stdin(f, jsonl_batch)
|
|
1818
|
+
assert rc == 0 and "add: 2 record(s) accepted, 0 rejected" in out, f"stdin jsonl batch failed: {out}"
|
|
1819
|
+
# @file with newline-delimited JSONL
|
|
1820
|
+
jf = d / "bt-lines.jsonl"
|
|
1821
|
+
jf.write_text(jsonl_batch, encoding="utf-8")
|
|
1822
|
+
rc, out = _capture(cmd_add, argparse.Namespace(file=str(f), record="@" + str(jf), stdin=False, aspect=None))
|
|
1823
|
+
assert rc == 0 and "add: 2 record(s) accepted" in out, f"@file jsonl batch failed: {out}"
|
|
1824
|
+
# markdown code-fenced payload in stdin
|
|
1825
|
+
fenced = "```json\n" + json.dumps([_fixture_article(pmid="10000008", doi="10.1000/b8", title="Batch eight", ids={"pmid": "10000008", "doi": "10.1000/b8", "pmcid": "PMC1000008"})]) + "\n```\n"
|
|
1826
|
+
rc, out = add_stdin(f, fenced)
|
|
1827
|
+
assert rc == 0 and "add: 1 record(s) accepted" in out, f"fenced payload failed: {out}"
|
|
1317
1828
|
# title-key rules: web/dataset keep hard identity fields; article
|
|
1318
1829
|
# rejects title-only input AND explicit title: keys; other falls
|
|
1319
1830
|
# back to a title: key.
|
|
@@ -1345,6 +1856,247 @@ def selftest() -> int:
|
|
|
1345
1856
|
assert len(led3.by_key) == 1 and not led3.quarantined, f"BOM re-read failed: {led3.quarantined}"
|
|
1346
1857
|
check("batch/title-rules", st_batch_and_title_rules)
|
|
1347
1858
|
|
|
1859
|
+
# ---- name -> title fold (biomcp `name`-shaped records) -------------
|
|
1860
|
+
def st_name_fold():
|
|
1861
|
+
f = d / "nf.jsonl"
|
|
1862
|
+
# drug carrying `name` instead of `title` (biomcp drug_get shape)
|
|
1863
|
+
drug = {"type": "drug", "ids": {"chembl": "CHEMBL1229517"}, "name": "vemurafenib",
|
|
1864
|
+
"meta": {"indication": "BRAF V600E-mutant melanoma"}, "provenance": [{"aspect": "a"}]}
|
|
1865
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(drug), stdin=False, aspect=None))
|
|
1866
|
+
led = read_ledger(f)
|
|
1867
|
+
assert "chembl:CHEMBL1229517" in led.by_key, "name-bearing drug not accepted"
|
|
1868
|
+
assert led.by_key["chembl:CHEMBL1229517"]["title"] == "vemurafenib", "name not folded into title"
|
|
1869
|
+
# fold is idempotent across re-reads and preserves the original field
|
|
1870
|
+
before = json.dumps(led.by_key["chembl:CHEMBL1229517"], sort_keys=True)
|
|
1871
|
+
after = json.dumps(read_ledger(f).by_key["chembl:CHEMBL1229517"], sort_keys=True)
|
|
1872
|
+
assert before == after, "name fold not idempotent on re-read"
|
|
1873
|
+
assert led.by_key["chembl:CHEMBL1229517"].get("name") == "vemurafenib", "original name field lost"
|
|
1874
|
+
# gene name-only now accepted too (acceptance widening, CHANGELOG-noted)
|
|
1875
|
+
gene = {"type": "gene", "ids": {"entrez_id": "673"}, "name": "B-Raf proto-oncogene",
|
|
1876
|
+
"provenance": [{"aspect": "a"}]}
|
|
1877
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(gene), stdin=False, aspect=None))
|
|
1878
|
+
assert "gene:673" in read_ledger(f).by_key, "name-bearing gene not accepted"
|
|
1879
|
+
# article keeps its hard id requirement: name-only rejected
|
|
1880
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "article", "name": "x", "ids": {}}', stdin=False, aspect=None))
|
|
1881
|
+
# web/dataset keep their hard identity fields: name-only rejected
|
|
1882
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "web", "name": "Just a page", "ids": {}}', stdin=False, aspect=None))
|
|
1883
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record='{"type": "dataset", "name": "Just a series", "ids": {}}', stdin=False, aspect=None))
|
|
1884
|
+
keys = set(read_ledger(f).by_key)
|
|
1885
|
+
assert not any(k.startswith("title:") for k in keys), "name-only article/web/dataset must not derive title keys"
|
|
1886
|
+
assert len(keys) == 2, f"name-fold acceptance boundary wrong: {sorted(keys)}"
|
|
1887
|
+
# renders clean (no [MISSING field: title])
|
|
1888
|
+
_, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="chembl:CHEMBL1229517", expand_pages=False, offset=0))
|
|
1889
|
+
assert "[MISSING" not in out.splitlines()[0], f"drug still renders MISSING: {out.splitlines()[0]}"
|
|
1890
|
+
assert "Vemurafenib" in out.splitlines()[0] or "vemurafenib" in out.splitlines()[0], out.splitlines()[0]
|
|
1891
|
+
check("name-fold", st_name_fold)
|
|
1892
|
+
|
|
1893
|
+
# ---- render: cite-key markers -> numbered citations + References ---
|
|
1894
|
+
def _render_ledger(d):
|
|
1895
|
+
f = d / "render.jsonl"
|
|
1896
|
+
batch = [
|
|
1897
|
+
_fixture_article(),
|
|
1898
|
+
_fixture_article(pmid="30000001", doi="10.1000/r1", title="Render one", ids={"pmid": "30000001", "doi": "10.1000/r1", "pmcid": "PMC3000001"}),
|
|
1899
|
+
_fixture_article(pmid="30000002", doi="10.1000/r2", title="Render two", ids={"pmid": "30000002", "doi": "10.1000/r2", "pmcid": "PMC3000002"}),
|
|
1900
|
+
_fixture_article(pmid="30000003", doi="10.1000/r3", title="Render three", ids={"pmid": "30000003", "doi": "10.1000/r3", "pmcid": "PMC3000003"}),
|
|
1901
|
+
{"key": "nct:NCT04280705", "type": "trial", "ids": {"nct": "NCT04280705"},
|
|
1902
|
+
"title": "Encorafenib Plus Cetuximab", "meta": {"phase": "Phase 2", "sponsor": "Pfizer", "status": "Completed"},
|
|
1903
|
+
"provenance": [{"aspect": "a"}]},
|
|
1904
|
+
{"type": "drug", "ids": {"chembl": "CHEMBL1229517"}, "name": "vemurafenib",
|
|
1905
|
+
"meta": {"indication": "BRAF V600E-mutant melanoma"}, "provenance": [{"aspect": "a"}]},
|
|
1906
|
+
]
|
|
1907
|
+
old_stdin, sys.stdin = sys.stdin, io.StringIO(json.dumps(batch))
|
|
1908
|
+
try:
|
|
1909
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=None, stdin=True, aspect=None))
|
|
1910
|
+
finally:
|
|
1911
|
+
sys.stdin = old_stdin
|
|
1912
|
+
return f
|
|
1913
|
+
|
|
1914
|
+
def st_render():
|
|
1915
|
+
f = _render_ledger(d)
|
|
1916
|
+
draft = d / "report.draft.md"
|
|
1917
|
+
draft.write_text(
|
|
1918
|
+
"# Report\n\n"
|
|
1919
|
+
"Vemurafenib [@chembl:CHEMBL1229517] improves survival [@pmid:21639808].\n\n"
|
|
1920
|
+
"Prose [@home] and pandoc [@Chapman2011] and symbol-ish [@gene:BRAF] stay verbatim.\n\n"
|
|
1921
|
+
"Trial plus article [@nct:NCT04280705; @pmid:21639808] group. Again [@pmid:21639808].\n\n"
|
|
1922
|
+
"Three more [@pmid:30000001; @pmid:30000002; @pmid:30000003] in one group.\n",
|
|
1923
|
+
encoding="utf-8",
|
|
1924
|
+
)
|
|
1925
|
+
out = d / "final.md"
|
|
1926
|
+
rc, stdout = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out), expand_pages=False))
|
|
1927
|
+
assert rc == 0, f"render failed: {stdout}"
|
|
1928
|
+
text = out.read_text(encoding="utf-8")
|
|
1929
|
+
assert "Vemurafenib [1] improves survival [2]." in text, f"numbering wrong:\n{text}"
|
|
1930
|
+
assert "[@home]" in text and "[@Chapman2011]" in text and "[@gene:BRAF]" in text, "prose brackets rewritten"
|
|
1931
|
+
assert "[2, 3]" in text, f"group not sorted/compressed: {text}"
|
|
1932
|
+
assert "Again [2]." in text, "duplicate key not reusing its number"
|
|
1933
|
+
assert "[4-6]" in text, f"consecutive run not range-compressed: {text}"
|
|
1934
|
+
assert "## References" in text and "[1] vemurafenib." in text and "[2] Chapman PB" in text
|
|
1935
|
+
# registry titles ending in a period never render doubled ("Title.. Status")
|
|
1936
|
+
# and corporate sponsors ending in periods (e.g. "Tesaro, Inc.") do not double
|
|
1937
|
+
tdot = d / "tdot.jsonl"
|
|
1938
|
+
tdot.write_text(json.dumps({
|
|
1939
|
+
"key": "nct:NCT01844986", "type": "trial", "ids": {"nct": "NCT01844986"},
|
|
1940
|
+
"title": "Olaparib Maintenance Monotherapy in Patients With BRCA Mutated Ovarian Cancer Following First Line Platinum Based Chemotherapy.",
|
|
1941
|
+
"meta": {"phase": "Phase 3.", "sponsor": "Tesaro, Inc.", "status": "ACTIVE_NOT_RECRUITING."},
|
|
1942
|
+
"provenance": [{"aspect": "a"}]}) + "\n", encoding="utf-8")
|
|
1943
|
+
_, tout = _capture(cmd_bib, argparse.Namespace(file=str(tdot), keys="nct:NCT01844986", expand_pages=False, offset=0))
|
|
1944
|
+
tline = tout.splitlines()[0]
|
|
1945
|
+
assert "Chemotherapy. Phase 3. Sponsor: Tesaro, Inc. Status: ACTIVE_NOT_RECRUITING." in tline and ".." not in tline, f"double period rendered: {tline}"
|
|
1946
|
+
assert "[MISSING" not in text
|
|
1947
|
+
refs = text.split("## References", 1)[1]
|
|
1948
|
+
nums = re.findall(r"(?m)^\[(\d+)\]", refs)
|
|
1949
|
+
assert nums == [str(i) for i in range(1, 7)], f"bibliography not 1..N contiguous: {nums}"
|
|
1950
|
+
# idempotent re-render of the SAME draft
|
|
1951
|
+
out2 = d / "final2.md"
|
|
1952
|
+
_capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out2), expand_pages=False))
|
|
1953
|
+
assert out2.read_text(encoding="utf-8") == text, "re-render of the same draft is not idempotent"
|
|
1954
|
+
# sec-index: doi: marker resolves to the promoted pmid twin
|
|
1955
|
+
doi_draft = d / "doi.draft.md"
|
|
1956
|
+
doi_draft.write_text("Only one [@doi:10.1056/nejmoa1103782].\n", encoding="utf-8")
|
|
1957
|
+
doi_out = d / "doi.md"
|
|
1958
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(doi_draft), out=str(doi_out), expand_pages=False))
|
|
1959
|
+
dtext = doi_out.read_text(encoding="utf-8")
|
|
1960
|
+
assert rc == 0 and "Only one [1]." in dtext and "PMID: 21639808." in dtext, f"doi twin resolution failed: {dtext}"
|
|
1961
|
+
assert len(re.findall(r"(?m)^\[\d+\]", dtext.split("## References", 1)[1])) == 1
|
|
1962
|
+
# unknown key: exit 1, output NOT written
|
|
1963
|
+
bad = d / "bad.draft.md"
|
|
1964
|
+
bad.write_text("Broken [@pmid:99999999].\n", encoding="utf-8")
|
|
1965
|
+
bad_out = d / "bad.md"
|
|
1966
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(bad), out=str(bad_out), expand_pages=False))
|
|
1967
|
+
assert rc != 0 and not bad_out.exists(), "unknown key must exit 1 without writing output"
|
|
1968
|
+
# [MISSING ...] entry: exit 1, output NOT written
|
|
1969
|
+
tonly = {"type": "trial", "title": "Title-only degraded trial", "provenance": [{"aspect": "a"}]}
|
|
1970
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(tonly), stdin=False, aspect=None))
|
|
1971
|
+
tkey = next(k for k in read_ledger(f).by_key if k.startswith("title:"))
|
|
1972
|
+
miss = d / "miss.draft.md"
|
|
1973
|
+
miss.write_text(f"Degraded [@{tkey}] cite.\n", encoding="utf-8")
|
|
1974
|
+
miss_out = d / "miss.md"
|
|
1975
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(miss), out=str(miss_out), expand_pages=False))
|
|
1976
|
+
assert rc != 0 and not miss_out.exists(), "MISSING-rendering record must exit 1 without writing output"
|
|
1977
|
+
# double-render guard: rendering an already-rendered report fails
|
|
1978
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(out), out=str(d / "double.md"), expand_pages=False))
|
|
1979
|
+
assert rc != 0, "double-render must fail (no markers, References present)"
|
|
1980
|
+
assert not (d / "double.md").exists()
|
|
1981
|
+
# fence-awareness: a fenced example References section / marker is
|
|
1982
|
+
# never stripped and never numbered
|
|
1983
|
+
fence_draft = d / "fence.draft.md"
|
|
1984
|
+
fence_draft.write_text(
|
|
1985
|
+
"# F\n\nReal cite [@pmid:21639808].\n\nExample (do not touch):\n\n```\n"
|
|
1986
|
+
"## References\n\n[1] example entry.\n\nCite like [@pmid:21639808].\n```\n\nTail kept.\n",
|
|
1987
|
+
encoding="utf-8",
|
|
1988
|
+
)
|
|
1989
|
+
fence_out = d / "fence.md"
|
|
1990
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(fence_draft), out=str(fence_out), expand_pages=False))
|
|
1991
|
+
ftext = fence_out.read_text(encoding="utf-8")
|
|
1992
|
+
assert rc == 0, "fenced content must not break render"
|
|
1993
|
+
assert "## References\n\n[1] example entry." in ftext, "fenced References example was stripped"
|
|
1994
|
+
assert "[@pmid:21639808]" in ftext, "marker inside a fence was rewritten"
|
|
1995
|
+
assert "Tail kept." in ftext, "content after a fenced References example was truncated"
|
|
1996
|
+
assert "Real cite [1]." in ftext and ftext.rstrip().endswith("[1] Chapman PB, Hauschild A, Robert C, et al. Improved survival with vemurafenib in melanoma with BRAF V600E mutation. N Engl J Med. 2011;364(26):2507-16. DOI: 10.1056/nejmoa1103782. PMID: 21639808."), \
|
|
1997
|
+
f"real marker/References wrong:\n{ftext}"
|
|
1998
|
+
# mixed citation group: at least one cite-key + a non-citation token
|
|
1999
|
+
# must hard-fail (never silently drop the citation)
|
|
2000
|
+
mix = d / "mix.draft.md"
|
|
2001
|
+
mix.write_text("Mixed [@pmid:21639808; see note] group.\n", encoding="utf-8")
|
|
2002
|
+
mix_out = d / "mix.md"
|
|
2003
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(mix), out=str(mix_out), expand_pages=False))
|
|
2004
|
+
assert rc != 0 and not mix_out.exists(), "mixed citation group must exit 1 without writing output"
|
|
2005
|
+
check("render", st_render)
|
|
2006
|
+
|
|
2007
|
+
# ---- check: worker validity gate -----------------------------------
|
|
2008
|
+
def st_check():
|
|
2009
|
+
f = d / "ck.jsonl"
|
|
2010
|
+
f.write_text("", encoding="utf-8")
|
|
2011
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f)))
|
|
2012
|
+
assert rc == 0 and "0 record(s)" in out and "quarantined 0" in out, f"empty ledger must pass: {out}"
|
|
2013
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
|
|
2014
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f)))
|
|
2015
|
+
assert rc == 0 and "1 record(s)" in out and "pmid:21639808" in out and "; OK" in out, out
|
|
2016
|
+
bad = d / "ck2.jsonl"
|
|
2017
|
+
bad.write_text(json.dumps(_fixture_article()) + "\nnot json\n", encoding="utf-8")
|
|
2018
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(bad)))
|
|
2019
|
+
assert rc == 1 and "quarantined 1" in out and "FAIL" in out, f"quarantined ledger must fail: {out}"
|
|
2020
|
+
check("check", st_check)
|
|
2021
|
+
|
|
2022
|
+
# ---- check --markers: worker marker cross-validation gate ------------
|
|
2023
|
+
def st_check_markers():
|
|
2024
|
+
f = d / "ckm.jsonl"
|
|
2025
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
|
|
2026
|
+
good = d / "good.md"
|
|
2027
|
+
good.write_text(
|
|
2028
|
+
"Real [@pmid:21639808] plus its doi twin [@doi:10.1056/NEJMOA1103782].\n"
|
|
2029
|
+
"Prose [@home] and shape-ish [@gene:BRAF] stay verbatim.\n"
|
|
2030
|
+
"CI text [95% CI 78-89] and a link [1](http://x) are not citations.\n"
|
|
2031
|
+
"Fenced example:\n\n```\n[@pmid:99999999]\n```\n",
|
|
2032
|
+
encoding="utf-8",
|
|
2033
|
+
)
|
|
2034
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(good)]))
|
|
2035
|
+
assert rc == 0, f"resolvable markers must pass:\n{out}"
|
|
2036
|
+
assert "markers[good.md]: 2 citation group(s), 2 marker(s) resolved" in out
|
|
2037
|
+
assert "markers: 0 problem(s) across 1 file(s); OK" in out
|
|
2038
|
+
# unresolved marker -> exit 1
|
|
2039
|
+
bad = d / "bad.md"
|
|
2040
|
+
bad.write_text("Broken [@pmid:99999999] cite.\n", encoding="utf-8")
|
|
2041
|
+
buf = io.StringIO()
|
|
2042
|
+
with contextlib.redirect_stderr(buf):
|
|
2043
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(bad)]))
|
|
2044
|
+
assert rc == 1 and "unresolved marker pmid:99999999" in buf.getvalue(), out + buf.getvalue()
|
|
2045
|
+
# mixed group (cite + plain token) -> exit 1, mirroring render
|
|
2046
|
+
mix = d / "mix.md"
|
|
2047
|
+
mix.write_text("Mixed [@pmid:21639808; see note] group.\n", encoding="utf-8")
|
|
2048
|
+
buf = io.StringIO()
|
|
2049
|
+
with contextlib.redirect_stderr(buf):
|
|
2050
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(mix)]))
|
|
2051
|
+
assert rc == 1 and "mixed citation group" in buf.getvalue(), out + buf.getvalue()
|
|
2052
|
+
# missing marker file -> exit 1 (never a vacuous pass)
|
|
2053
|
+
rc, out = _capture(cmd_check, argparse.Namespace(file=str(f), markers=[str(d / "nope.md")]))
|
|
2054
|
+
assert rc == 1 and "marker file not found" in out, out
|
|
2055
|
+
check("check-markers", st_check_markers)
|
|
2056
|
+
|
|
2057
|
+
# ---- render: hand-typed numeric bracket warning (non-fatal) ---------
|
|
2058
|
+
def st_render_handtyped_warning():
|
|
2059
|
+
f = d / "ht.jsonl"
|
|
2060
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(_fixture_article()), stdin=False, aspect=None))
|
|
2061
|
+
draft = d / "ht.draft.md"
|
|
2062
|
+
draft.write_text(
|
|
2063
|
+
"Clean [@pmid:21639808] marker.\n\nBut a hand-typed [1] leak and [2, 3] too.\n\n"
|
|
2064
|
+
"Not citations: [95% CI 78-89], link [4](http://x), wiki [[5, 6]], fence:\n\n```\n[7]\n```\n",
|
|
2065
|
+
encoding="utf-8",
|
|
2066
|
+
)
|
|
2067
|
+
out_path = d / "ht.md"
|
|
2068
|
+
buf_err = io.StringIO()
|
|
2069
|
+
with contextlib.redirect_stderr(buf_err):
|
|
2070
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(draft), out=str(out_path), expand_pages=False))
|
|
2071
|
+
err = buf_err.getvalue()
|
|
2072
|
+
assert rc == 0, "hand-typed brackets must NOT fail render"
|
|
2073
|
+
assert "hand-typed numeric citation bracket" in err and "line(s) [3]" in err, err
|
|
2074
|
+
assert "[7]" not in err.replace("line(s)", ""), "fenced [7] must not warn"
|
|
2075
|
+
# clean draft: no warning at all
|
|
2076
|
+
clean = d / "ht2.draft.md"
|
|
2077
|
+
clean.write_text("Only [@pmid:21639808] here.\n", encoding="utf-8")
|
|
2078
|
+
buf_err2 = io.StringIO()
|
|
2079
|
+
with contextlib.redirect_stderr(buf_err2):
|
|
2080
|
+
rc, _ = _capture(cmd_render, argparse.Namespace(ledger=str(f), draft=str(clean), out=str(d / "ht2.md"), expand_pages=False))
|
|
2081
|
+
assert rc == 0 and "hand-typed" not in buf_err2.getvalue(), buf_err2.getvalue()
|
|
2082
|
+
check("render-handtyped-warning", st_render_handtyped_warning)
|
|
2083
|
+
|
|
2084
|
+
# ---- pdb dataset support ---------------------------------------------
|
|
2085
|
+
def st_pdb_dataset():
|
|
2086
|
+
f = d / "pdb.jsonl"
|
|
2087
|
+
rec = {"type": "dataset", "ids": {"pdb": "6N65"}, "title": "KRAS G-quadruplex G16T mutant",
|
|
2088
|
+
"meta": {"method": "X-RAY DIFFRACTION", "resolution": "1.6 Å"},
|
|
2089
|
+
"provenance": [{"aspect": "pdb_aspect"}]}
|
|
2090
|
+
_capture(cmd_add, argparse.Namespace(file=str(f), record=json.dumps(rec), stdin=False, aspect=None))
|
|
2091
|
+
led = read_ledger(f)
|
|
2092
|
+
assert "pdb:6N65" in led.by_key, "pdb:6N65 key not derived"
|
|
2093
|
+
_, out = _capture(cmd_bib, argparse.Namespace(file=str(f), keys="pdb:6N65", expand_pages=False, offset=0))
|
|
2094
|
+
line = out.splitlines()[0]
|
|
2095
|
+
assert "PDB structure 6N65: KRAS G-quadruplex G16T mutant." in line
|
|
2096
|
+
assert "[X-RAY DIFFRACTION]" in line and "Resolution: 1.6 Å." in line
|
|
2097
|
+
assert "https://www.rcsb.org/structure/6N65" in line
|
|
2098
|
+
check("pdb-dataset", st_pdb_dataset)
|
|
2099
|
+
|
|
1348
2100
|
failed = [r for r in results if r[1] is not None]
|
|
1349
2101
|
for name, err in results:
|
|
1350
2102
|
print(f"{'PASS' if err is None else 'FAIL'} {name}" + (f": {err}" if err else ""))
|
|
@@ -1394,6 +2146,19 @@ def main() -> int:
|
|
|
1394
2146
|
p.add_argument("file")
|
|
1395
2147
|
p.set_defaults(fn=cmd_stats)
|
|
1396
2148
|
|
|
2149
|
+
p = sub.add_parser("render", help="Number cite-key markers in a draft and append the References section")
|
|
2150
|
+
p.add_argument("ledger", help="merged ledger file (evidence/sources.jsonl)")
|
|
2151
|
+
p.add_argument("draft", help="markdown draft authored with [@key] markers")
|
|
2152
|
+
p.add_argument("-o", "--out", required=True, help="output report path (never written on failure)")
|
|
2153
|
+
p.add_argument("--expand-pages", action="store_true", help="expand abbreviated page ranges (2507-16 -> 2507-2516)")
|
|
2154
|
+
p.set_defaults(fn=cmd_render)
|
|
2155
|
+
|
|
2156
|
+
p = sub.add_parser("check", help="Validate a ledger file (exit 1 on quarantined lines; with --markers also on unresolved/mixed markers)")
|
|
2157
|
+
p.add_argument("file")
|
|
2158
|
+
p.add_argument("--markers", nargs="+", metavar="MD",
|
|
2159
|
+
help="markdown file(s) whose [@key] markers must resolve in this ledger")
|
|
2160
|
+
p.set_defaults(fn=cmd_check)
|
|
2161
|
+
|
|
1397
2162
|
p = sub.add_parser("selftest", help="Hermetic feature-matrix selftest (no network)")
|
|
1398
2163
|
p.set_defaults(fn=lambda a: selftest())
|
|
1399
2164
|
|