medsci-skills 5.0.0 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/metadata/distribution_files.json +82 -22
  2. package/metadata/distribution_manifest.json +1 -1
  3. package/package.json +1 -1
  4. package/skills/fulltext-retrieval/SKILL.md +52 -5
  5. package/skills/fulltext-retrieval/fetch_oa.py +293 -74
  6. package/skills/fulltext-retrieval/fetch_oa_report_challenge/expected/projection.json +14 -0
  7. package/skills/fulltext-retrieval/fetch_oa_report_challenge/extracted_text.json +4 -0
  8. package/skills/fulltext-retrieval/fetch_oa_report_challenge/results.json +6 -0
  9. package/skills/fulltext-retrieval/fetch_oa_report_challenge/run_challenge.py +86 -0
  10. package/skills/fulltext-retrieval/fetch_oa_report_challenge/verify.sh +6 -0
  11. package/skills/fulltext-retrieval/fetch_oa_report_challenge/worklist.tsv +5 -0
  12. package/skills/fulltext-retrieval/references/find_available_pdf.js +82 -0
  13. package/skills/fulltext-retrieval/skill.yml +7 -2
  14. package/skills/lit-sync/SKILL.md +86 -6
  15. package/skills/lit-sync/skill.yml +3 -1
  16. package/skills/mllm-eval/SKILL.md +8 -0
  17. package/skills/mllm-eval/references/evaluation_axes.md +161 -0
  18. package/skills/model-evaluation/SKILL.md +12 -0
  19. package/skills/model-evaluation/references/metric_selection_grounding.md +139 -0
  20. package/skills/model-evaluation/scripts/check_metric_reporting.py +1 -1
  21. package/skills/model-evaluation/scripts/metric_reporting_challenge/fixture/det_good_wrapped.md +5 -0
  22. package/skills/model-evaluation/scripts/metric_reporting_challenge/fixture/det_no_iou.md +4 -0
  23. package/skills/model-evaluation/scripts/metric_reporting_challenge/verify.sh +6 -1
  24. package/skills/model-validation/SKILL.md +6 -0
  25. package/skills/model-validation/references/validation_design.md +150 -0
  26. package/skills/search-lit/SKILL.md +22 -67
@@ -2,20 +2,28 @@
2
2
  """
3
3
  Open-access full-text PDF batch retrieval.
4
4
 
5
- Pipeline: Unpaywall → PMC (Europe PMC REST / OA FTP / web)
6
- OpenAlex → Crossref → landing-page scrape.
5
+ Pipeline: arXiv (for 10.48550/arXiv.* DOIs) Unpaywall
6
+ PMC (Europe PMC REST / OA FTP / web) → OpenAlex → Crossref →
7
+ landing-page scrape.
7
8
 
8
9
  Usage:
9
10
  python fetch_oa.py dois.txt --output pdfs/ --email user@example.com
10
- python fetch_oa.py dois.txt -o pdfs/ -e user@example.com --verbose
11
+ python fetch_oa.py worklist.tsv -o pdfs/ -e user@example.com --verbose
12
+ python fetch_oa.py worklist.csv -o pdfs/ -e user@example.com --report pdfs/retrieval_report.json
13
+
14
+ Worklist formats: plain DOI-per-line, or TSV/CSV/Markdown-table with a DOI
15
+ column (optional PMID and Title columns). A Title column enables a best-effort
16
+ title cross-check (via `pdftotext` if installed) that flags mislabeled PDFs.
11
17
  """
12
18
 
13
19
  import argparse
20
+ import csv
21
+ import io
14
22
  import json
15
23
  import logging
16
- import os
17
24
  import re
18
- import sys
25
+ import shutil
26
+ import subprocess
19
27
  import time
20
28
  import urllib.error
21
29
  import urllib.parse
@@ -25,6 +33,9 @@ from pathlib import Path
25
33
 
26
34
  MIN_PDF_BYTES = 10 * 1024
27
35
  USER_AGENT = "medsci-skills/1.0"
36
+ REPORT_SCHEMA_VERSION = 1
37
+ TITLE_MATCH_THRESHOLD = 0.6
38
+ RETRIEVED_STATUSES = ("oa", "pmc", "arxiv", "skip")
28
39
 
29
40
  log = logging.getLogger("fetch_oa")
30
41
 
@@ -38,6 +49,11 @@ def _ua(email: str) -> str:
38
49
  return f"{USER_AGENT} (mailto:{email})"
39
50
 
40
51
 
52
+ def safe_doi_name(doi: str) -> str:
53
+ """Filesystem-safe filename stem for a DOI."""
54
+ return re.sub(r"[^\w\-.]", "_", doi)
55
+
56
+
41
57
  def is_valid_pdf(data: bytes) -> bool:
42
58
  return data.startswith(b"%PDF-") and len(data) >= MIN_PDF_BYTES
43
59
 
@@ -68,6 +84,84 @@ def existing_pdf_ok(path: Path) -> bool:
68
84
  return False
69
85
 
70
86
 
87
+ # ============================================================
88
+ # Title cross-check (pure, offline-testable)
89
+ # ============================================================
90
+
91
+ def normalize_title(text: str) -> str:
92
+ """Lowercase, strip punctuation, collapse whitespace."""
93
+ text = (text or "").lower()
94
+ text = re.sub(r"[^a-z0-9\s]", " ", text)
95
+ return " ".join(text.split())
96
+
97
+
98
+ def title_overlap(expected_title: str, extracted_text: str) -> float:
99
+ """Fraction of meaningful expected-title tokens present in extracted text.
100
+
101
+ Tokens of length <= 2 are dropped as near-stopwords. Returns 0.0 when the
102
+ expected title has no usable tokens.
103
+ """
104
+ expected = {t for t in normalize_title(expected_title).split() if len(t) > 2}
105
+ if not expected:
106
+ return 0.0
107
+ got = set(normalize_title(extracted_text).split())
108
+ return len(expected & got) / len(expected)
109
+
110
+
111
+ def classify_title_match(expected_title: str, extracted_text,
112
+ threshold: float = TITLE_MATCH_THRESHOLD) -> str:
113
+ """Tri-state title check: 'match' | 'mismatch' | 'unavailable'.
114
+
115
+ 'unavailable' when no expected title is supplied or no extracted text is
116
+ available (e.g. pdftotext absent). A low overlap is 'mismatch' — flagged,
117
+ never used to auto-reject a downloaded PDF.
118
+ """
119
+ if not expected_title or not extracted_text:
120
+ return "unavailable"
121
+ return "match" if title_overlap(expected_title, extracted_text) >= threshold else "mismatch"
122
+
123
+
124
+ def extract_pdf_text(path: Path, max_pages: int = 2) -> str | None:
125
+ """Best-effort first-page text via `pdftotext` (poppler). None if unavailable."""
126
+ if not shutil.which("pdftotext"):
127
+ return None
128
+ try:
129
+ out = subprocess.run(
130
+ ["pdftotext", "-f", "1", "-l", str(max_pages), str(path), "-"],
131
+ capture_output=True, timeout=20,
132
+ )
133
+ if out.returncode == 0:
134
+ return out.stdout.decode("utf-8", errors="ignore")
135
+ except (OSError, subprocess.SubprocessError):
136
+ pass
137
+ return None
138
+
139
+
140
+ # ============================================================
141
+ # arXiv (direct, for 10.48550/arXiv.* DOIs)
142
+ # ============================================================
143
+
144
+ _ARXIV_DOI_RE = re.compile(r"^10\.48550/arxiv\.(.+)$", re.IGNORECASE)
145
+ _ARXIV_ID_RE = re.compile(r"^arxiv:(.+)$", re.IGNORECASE)
146
+
147
+
148
+ def arxiv_id_from_doi(doi: str) -> str | None:
149
+ """Extract an arXiv ID from a DataCite arXiv DOI or a bare arXiv: id.
150
+
151
+ Handles new-style (2401.01234, 2401.01234v2) and old-style
152
+ (hep-th/9901001) identifiers; version suffix preserved when present.
153
+ """
154
+ s = (doi or "").strip()
155
+ m = _ARXIV_DOI_RE.match(s) or _ARXIV_ID_RE.match(s)
156
+ return m.group(1).strip() if m else None
157
+
158
+
159
+ def arxiv_pdf_url(doi: str) -> str | None:
160
+ """Direct arXiv PDF URL for an arXiv DOI/ID (None if not an arXiv id)."""
161
+ aid = arxiv_id_from_doi(doi)
162
+ return f"https://arxiv.org/pdf/{aid}" if aid else None
163
+
164
+
71
165
  # ============================================================
72
166
  # 1. Unpaywall
73
167
  # ============================================================
@@ -270,41 +364,34 @@ def download_pdf(url: str, outpath: Path, email: str) -> bool:
270
364
  # 5. Main pipeline
271
365
  # ============================================================
272
366
 
273
- def gather_candidates(doi: str, email: str) -> list[str]:
274
- """Collect OA PDF candidate URLs from multiple sources."""
275
- urls: list[str] = []
276
-
277
- def add(v: str | None):
278
- if v and v not in urls:
279
- urls.append(v)
280
-
281
- add(unpaywall_lookup(doi, email))
282
- for v in openalex_lookup(doi, email):
283
- add(v)
284
- for v in crossref_lookup(doi, email):
285
- add(v)
286
- add(f"https://doi.org/{doi}")
287
- return urls
288
-
289
-
290
367
  def process_doi(doi: str, outdir: Path, email: str,
291
- pmid: str = "") -> str:
292
- """Try to download a PDF for one DOI. Returns status string."""
293
- safe_name = re.sub(r"[^\w\-.]", "_", doi)
294
- outpath = outdir / f"{safe_name}.pdf"
368
+ pmid: str = "") -> tuple[str, str]:
369
+ """Try to download a PDF for one DOI.
370
+
371
+ Returns (status, source):
372
+ status ∈ {"arxiv", "oa", "pmc", "skip", "fail"}
373
+ source identifies the resolver that succeeded (e.g. "unpaywall", "pmc",
374
+ "openalex", "crossref", "landing", "arxiv", "existing", "").
375
+ """
376
+ outpath = outdir / f"{safe_doi_name(doi)}.pdf"
295
377
 
296
378
  if existing_pdf_ok(outpath):
297
- return "skip"
379
+ return ("skip", "existing")
298
380
 
299
381
  # Remove stale stub
300
382
  if outpath.exists():
301
383
  outpath.unlink(missing_ok=True)
302
384
 
385
+ # Step 0: arXiv direct (for 10.48550/arXiv.* DOIs)
386
+ ax_url = arxiv_pdf_url(doi)
387
+ if ax_url and download_pdf(ax_url, outpath, email):
388
+ return ("arxiv", "arxiv")
389
+
303
390
  # Step 1: Unpaywall direct PDF URL (fastest path)
304
391
  uw_url = unpaywall_lookup(doi, email)
305
392
  if uw_url and ".pdf" in uw_url.lower():
306
393
  if download_pdf(uw_url, outpath, email):
307
- return "oa"
394
+ return ("oa", "unpaywall")
308
395
  time.sleep(0.3)
309
396
 
310
397
  # Step 2: PMC (try before slow landing-page scraping)
@@ -312,59 +399,158 @@ def process_doi(doi: str, outdir: Path, email: str,
312
399
  if not pmcid:
313
400
  pmcid = id_to_pmcid(doi, email)
314
401
  if pmcid and download_pmc_pdf(pmcid, outpath, email):
315
- return "pmc"
402
+ return ("pmc", "pmc")
316
403
 
317
404
  # Step 3: OA candidates from OpenAlex, Crossref, landing pages
318
- candidates: list[str] = []
319
- if uw_url and uw_url not in candidates:
320
- candidates.append(uw_url)
405
+ candidates: list[tuple[str, str]] = []
406
+ seen: set[str] = set()
407
+
408
+ def add(source: str, url: str | None):
409
+ if url and url not in seen:
410
+ seen.add(url)
411
+ candidates.append((source, url))
412
+
413
+ add("unpaywall", uw_url)
321
414
  for v in openalex_lookup(doi, email):
322
- if v not in candidates:
323
- candidates.append(v)
415
+ add("openalex", v)
324
416
  for v in crossref_lookup(doi, email):
325
- if v not in candidates:
326
- candidates.append(v)
327
- candidates.append(f"https://doi.org/{doi}")
417
+ add("crossref", v)
418
+ add("landing", f"https://doi.org/{doi}")
328
419
 
329
- for url in candidates:
420
+ for source, url in candidates:
330
421
  if ".pdf" in url.lower():
331
422
  ok = download_pdf(url, outpath, email)
332
423
  else:
333
424
  ok = download_from_landing(url, outpath, email)
334
425
  if ok:
335
- return "oa"
426
+ return ("oa", source)
336
427
  time.sleep(0.3)
337
428
 
338
- return "fail"
429
+ return ("fail", "")
430
+
431
+
432
+ def build_report(records: list[dict], results: dict[str, tuple[str, str]],
433
+ outdir: Path, extracted_text_by_doi: dict[str, str] | None = None,
434
+ threshold: float = TITLE_MATCH_THRESHOLD) -> dict:
435
+ """Assemble a deterministic retrieval report (no network, no I/O writes).
436
+
437
+ records: list of {"doi", "pmid", "title"}.
438
+ results: doi -> (status, source) as returned by process_doi.
439
+ outdir: directory where PDFs were written (used for file/size lookup).
440
+ extracted_text_by_doi: optional doi -> first-page text for title cross-check.
441
+ """
442
+ extracted_text_by_doi = extracted_text_by_doi or {}
443
+ items = []
444
+ for rec in records:
445
+ doi = rec["doi"]
446
+ status, source = results.get(doi, ("fail", ""))
447
+ path = outdir / f"{safe_doi_name(doi)}.pdf"
448
+ have_file = status in RETRIEVED_STATUSES and path.exists()
449
+ size = path.stat().st_size if have_file else 0
450
+ if have_file:
451
+ title_match = classify_title_match(
452
+ rec.get("title", ""), extracted_text_by_doi.get(doi), threshold)
453
+ else:
454
+ title_match = "unavailable"
455
+ items.append({
456
+ "doi": doi,
457
+ "pmid": rec.get("pmid", ""),
458
+ "title": rec.get("title", ""),
459
+ "status": status,
460
+ "source": source,
461
+ "file": path.name if have_file else "",
462
+ "size_bytes": size,
463
+ "title_match": title_match,
464
+ })
465
+
466
+ retrieved = [i for i in items if i["status"] in RETRIEVED_STATUSES]
467
+ not_retrieved = [i for i in items if i["status"] == "fail"]
468
+ return {
469
+ "schema_version": REPORT_SCHEMA_VERSION,
470
+ "generated_by": "fetch_oa.py",
471
+ "counts": {
472
+ "total": len(items),
473
+ "retrieved": len(retrieved),
474
+ "not_retrieved": len(not_retrieved),
475
+ "title_mismatch": sum(1 for i in items if i["title_match"] == "mismatch"),
476
+ },
477
+ "items": items,
478
+ }
479
+
480
+
481
+ def _norm_key(key: str) -> str:
482
+ return (key or "").strip().lstrip("#").strip().lower()
483
+
484
+
485
+ def _records_from_dictrows(rows) -> list[dict]:
486
+ records = []
487
+ for row in rows:
488
+ rec = {"doi": "", "pmid": "", "title": ""}
489
+ for k, v in row.items():
490
+ nk = _norm_key(k)
491
+ if nk in rec:
492
+ rec[nk] = (v or "").strip()
493
+ if rec["doi"]:
494
+ records.append(rec)
495
+ return records
496
+
497
+
498
+ def _records_from_markdown(lines: list[str]) -> list[dict]:
499
+ pipe_rows = [ln for ln in lines if ln.strip().startswith("|")]
500
+ if not pipe_rows:
501
+ return []
502
+
503
+ def cells(line: str) -> list[str]:
504
+ return [c.strip() for c in line.strip().strip("|").split("|")]
505
+
506
+ header = [_norm_key(c) for c in cells(pipe_rows[0])]
507
+ records = []
508
+ for line in pipe_rows[1:]:
509
+ c = cells(line)
510
+ # Skip the |---|---| separator row
511
+ if c and all(set(x) <= set("-: ") for x in c):
512
+ continue
513
+ row = dict(zip(header, c))
514
+ doi = (row.get("doi") or "").strip()
515
+ if doi:
516
+ records.append({
517
+ "doi": doi,
518
+ "pmid": (row.get("pmid") or "").strip(),
519
+ "title": (row.get("title") or "").strip(),
520
+ })
521
+ return records
339
522
 
340
523
 
341
524
  def read_doi_file(path: Path) -> list[dict]:
342
- """Read DOI list. Supports plain DOIs or TSV with DOI/PMID columns."""
525
+ """Read a worklist of DOIs.
526
+
527
+ Supports: plain DOI-per-line; TSV/CSV with a DOI header (optional PMID,
528
+ Title columns); and a Markdown pipe table with a DOI column. Each record is
529
+ {"doi", "pmid", "title"}.
530
+ """
531
+ text = Path(path).read_text(encoding="utf-8")
532
+ lines = text.splitlines()
533
+ first = next((ln for ln in lines
534
+ if ln.strip() and not ln.strip().startswith("#")), "")
535
+ low = first.lower()
536
+
537
+ # Markdown pipe table with a DOI column
538
+ if first.strip().startswith("|") and "doi" in low:
539
+ return _records_from_markdown(lines)
540
+
541
+ # Delimited (TSV or CSV) with a DOI header
542
+ if "doi" in low and ("\t" in first or "," in first):
543
+ delimiter = "\t" if "\t" in first else ","
544
+ body = "\n".join(ln for ln in lines if not ln.strip().startswith("#"))
545
+ reader = csv.DictReader(io.StringIO(body), delimiter=delimiter)
546
+ return _records_from_dictrows(reader)
547
+
548
+ # Plain text: one DOI per line
343
549
  records = []
344
- with open(path, encoding="utf-8") as f:
345
- first_line = f.readline().strip()
346
- f.seek(0)
347
-
348
- # TSV with header containing DOI column
349
- if "\t" in first_line and "doi" in first_line.lower():
350
- import csv
351
- reader = csv.DictReader(f, delimiter="\t")
352
- for row in reader:
353
- doi = ""
354
- pmid = ""
355
- for k, v in row.items():
356
- if k.lower().strip() == "doi":
357
- doi = (v or "").strip()
358
- elif k.lower().strip() == "pmid":
359
- pmid = (v or "").strip()
360
- if doi:
361
- records.append({"doi": doi, "pmid": pmid})
362
- else:
363
- # Plain text: one DOI per line
364
- for line in f:
365
- line = line.strip()
366
- if line and not line.startswith("#"):
367
- records.append({"doi": line, "pmid": ""})
550
+ for line in lines:
551
+ line = line.strip()
552
+ if line and not line.startswith("#"):
553
+ records.append({"doi": line, "pmid": "", "title": ""})
368
554
  return records
369
555
 
370
556
 
@@ -372,11 +558,15 @@ def main():
372
558
  parser = argparse.ArgumentParser(
373
559
  description="Batch download open-access PDFs by DOI.")
374
560
  parser.add_argument("input", type=Path,
375
- help="File with DOIs (one per line, or TSV with DOI column)")
561
+ help="Worklist: DOIs (one per line) or TSV/CSV/Markdown "
562
+ "with a DOI column (optional PMID, Title)")
376
563
  parser.add_argument("-o", "--output", type=Path, default=Path("pdfs"),
377
564
  help="Output directory (default: pdfs/)")
378
565
  parser.add_argument("-e", "--email", required=True,
379
566
  help="Contact email (required by Unpaywall TOS)")
567
+ parser.add_argument("--report", type=Path, default=None,
568
+ help="Path for the JSON retrieval report "
569
+ "(default: <output>/retrieval_report.json)")
380
570
  parser.add_argument("-v", "--verbose", action="store_true",
381
571
  help="Show debug messages")
382
572
  args = parser.parse_args()
@@ -387,33 +577,62 @@ def main():
387
577
  )
388
578
 
389
579
  args.output.mkdir(parents=True, exist_ok=True)
580
+ report_path = args.report or (args.output / "retrieval_report.json")
390
581
  records = read_doi_file(args.input)
391
582
  print(f"Loaded {len(records)} DOIs from {args.input}")
392
583
 
393
- stats = {"oa": 0, "pmc": 0, "fail": 0, "skip": 0}
584
+ stats = {"arxiv": 0, "oa": 0, "pmc": 0, "fail": 0, "skip": 0}
585
+ results: dict[str, tuple[str, str]] = {}
394
586
 
395
587
  for i, rec in enumerate(records, 1):
396
588
  doi = rec["doi"]
397
589
  pmid = rec.get("pmid", "")
398
590
  print(f" [{i}/{len(records)}] {doi}", end=" … ", flush=True)
399
591
 
400
- status = process_doi(doi, args.output, args.email, pmid)
592
+ status, source = process_doi(doi, args.output, args.email, pmid)
593
+ results[doi] = (status, source)
401
594
  stats[status] += 1
402
595
 
403
- labels = {"oa": "OK (OA)", "pmc": "OK (PMC)",
596
+ labels = {"arxiv": "OK (arXiv)", "oa": "OK (OA)", "pmc": "OK (PMC)",
404
597
  "fail": "FAIL", "skip": "SKIP"}
405
598
  print(labels[status])
406
599
  time.sleep(0.5)
407
600
 
601
+ # Best-effort title cross-check on successful downloads (needs pdftotext).
602
+ extracted: dict[str, str] = {}
603
+ have_titles = any(r.get("title") for r in records)
604
+ if have_titles and shutil.which("pdftotext"):
605
+ for rec in records:
606
+ doi = rec["doi"]
607
+ if not rec.get("title"):
608
+ continue
609
+ status, _ = results.get(doi, ("fail", ""))
610
+ if status not in RETRIEVED_STATUSES:
611
+ continue
612
+ path = args.output / f"{safe_doi_name(doi)}.pdf"
613
+ if path.exists():
614
+ text = extract_pdf_text(path)
615
+ if text:
616
+ extracted[doi] = text
617
+
618
+ report = build_report(records, results, args.output, extracted)
619
+ report_path.parent.mkdir(parents=True, exist_ok=True)
620
+ report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
621
+
408
622
  print(f"\n--- Summary ---")
623
+ print(f" arXiv: {stats['arxiv']}")
409
624
  print(f" OA: {stats['oa']}")
410
625
  print(f" PMC: {stats['pmc']}")
411
626
  print(f" Failed: {stats['fail']}")
412
627
  print(f" Skipped: {stats['skip']}")
413
- total = stats["oa"] + stats["pmc"] + stats["fail"]
628
+ total = stats["arxiv"] + stats["oa"] + stats["pmc"] + stats["fail"]
414
629
  if total > 0:
415
- pct = (stats["oa"] + stats["pmc"]) / total * 100
630
+ pct = (stats["arxiv"] + stats["oa"] + stats["pmc"]) / total * 100
416
631
  print(f" Success: {pct:.0f}%")
632
+ mismatches = report["counts"]["title_mismatch"]
633
+ if mismatches:
634
+ print(f" Title mismatches flagged: {mismatches} (see report)")
635
+ print(f" Report: {report_path}")
417
636
 
418
637
  # Write failed DOIs for manual retrieval
419
638
  if stats["fail"] > 0:
@@ -422,10 +641,10 @@ def main():
422
641
  f.write("# DOIs needing manual retrieval\n")
423
642
  f.write("# Options: institutional access, ILL\n\n")
424
643
  for rec in records:
425
- safe = re.sub(r"[^\w\-.]", "_", rec["doi"])
426
- pdf = args.output / f"{safe}.pdf"
644
+ doi = rec["doi"]
645
+ pdf = args.output / f"{safe_doi_name(doi)}.pdf"
427
646
  if not existing_pdf_ok(pdf):
428
- f.write(f"{rec['doi']}\n")
647
+ f.write(f"{doi}\n")
429
648
  print(f" Manual list: {fail_path}")
430
649
 
431
650
 
@@ -0,0 +1,14 @@
1
+ {
2
+ "counts": {
3
+ "total": 4,
4
+ "retrieved": 3,
5
+ "not_retrieved": 1,
6
+ "title_mismatch": 1
7
+ },
8
+ "items": [
9
+ {"doi": "10.1111/valid.match", "status": "oa", "source": "unpaywall", "title_match": "match"},
10
+ {"doi": "10.2222/label.mismatch", "status": "oa", "source": "openalex", "title_match": "mismatch"},
11
+ {"doi": "10.3333/no.text", "status": "pmc", "source": "pmc", "title_match": "unavailable"},
12
+ {"doi": "10.9999/not.retrieved", "status": "fail", "source": "", "title_match": "unavailable"}
13
+ ]
14
+ }
@@ -0,0 +1,4 @@
1
+ {
2
+ "10.1111/valid.match": "Deep Learning for Pulmonary Nodule Detection on Chest CT\nAbstract: we present a convolutional neural network trained to detect pulmonary nodules.",
3
+ "10.2222/label.mismatch": "Genome-wide association study of type 2 diabetes in an Asian cohort\nIntroduction: we genotyped participants to identify susceptibility loci."
4
+ }
@@ -0,0 +1,6 @@
1
+ {
2
+ "10.1111/valid.match": ["oa", "unpaywall"],
3
+ "10.2222/label.mismatch": ["oa", "openalex"],
4
+ "10.3333/no.text": ["pmc", "pmc"],
5
+ "10.9999/not.retrieved": ["fail", ""]
6
+ }
@@ -0,0 +1,86 @@
1
+ #!/usr/bin/env python3
2
+ """Offline, network-free challenge for fetch_oa.build_report + title tri-state.
3
+
4
+ Loads committed fixtures (worklist.tsv, results.json, extracted_text.json),
5
+ fabricates stub PDFs in a temp dir for the "retrieved" DOIs, runs
6
+ fetch_oa.build_report, and asserts the resulting projection equals
7
+ expected/projection.json. Exercises read_doi_file (TSV+title), build_report,
8
+ and classify_title_match (match / mismatch / unavailable) without touching the
9
+ network or pdftotext. Stdlib-only.
10
+ """
11
+ import importlib.util
12
+ import json
13
+ import sys
14
+ import tempfile
15
+ from pathlib import Path
16
+
17
+ HERE = Path(__file__).resolve().parent
18
+ ENGINE = HERE.parent / "fetch_oa.py"
19
+
20
+
21
+ def load_engine():
22
+ spec = importlib.util.spec_from_file_location("fetch_oa", ENGINE)
23
+ mod = importlib.util.module_from_spec(spec)
24
+ spec.loader.exec_module(mod)
25
+ return mod
26
+
27
+
28
+ def projection(report: dict) -> dict:
29
+ items = sorted(
30
+ ({"doi": i["doi"], "status": i["status"], "source": i["source"],
31
+ "title_match": i["title_match"]} for i in report["items"]),
32
+ key=lambda x: x["doi"],
33
+ )
34
+ return {"counts": report["counts"], "items": items}
35
+
36
+
37
+ def main() -> int:
38
+ assert ENGINE.exists(), f"ENV-ERR: {ENGINE} missing"
39
+ m = load_engine()
40
+
41
+ records = m.read_doi_file(HERE / "worklist.tsv")
42
+ results = {k: tuple(v) for k, v in
43
+ json.loads((HERE / "results.json").read_text()).items()}
44
+ extracted = json.loads((HERE / "extracted_text.json").read_text())
45
+ expected = json.loads((HERE / "expected" / "projection.json").read_text())
46
+
47
+ fails = []
48
+
49
+ def check(label, cond):
50
+ print(f" {'PASS' if cond else 'FAIL'} {label}")
51
+ if not cond:
52
+ fails.append(label)
53
+
54
+ with tempfile.TemporaryDirectory() as tmp:
55
+ outdir = Path(tmp)
56
+ for rec in records:
57
+ status, _ = results.get(rec["doi"], ("fail", ""))
58
+ if status in m.RETRIEVED_STATUSES:
59
+ stub = outdir / f"{m.safe_doi_name(rec['doi'])}.pdf"
60
+ stub.write_bytes(b"%PDF-1.4\n" + b"0" * (11 * 1024))
61
+ report = m.build_report(records, results, outdir, extracted)
62
+ proj = projection(report)
63
+
64
+ by_doi = {i["doi"]: i for i in proj["items"]}
65
+ check("valid -> title_match 'match'",
66
+ by_doi["10.1111/valid.match"]["title_match"] == "match")
67
+ check("mislabel -> title_match 'mismatch'",
68
+ by_doi["10.2222/label.mismatch"]["title_match"] == "mismatch")
69
+ check("no-text -> title_match 'unavailable'",
70
+ by_doi["10.3333/no.text"]["title_match"] == "unavailable")
71
+ check("missing -> status 'fail'",
72
+ by_doi["10.9999/not.retrieved"]["status"] == "fail")
73
+ check("counts.retrieved == 3", proj["counts"]["retrieved"] == 3)
74
+ check("counts.not_retrieved == 1", proj["counts"]["not_retrieved"] == 1)
75
+ check("counts.title_mismatch == 1", proj["counts"]["title_mismatch"] == 1)
76
+ check("projection matches expected/projection.json", proj == expected)
77
+
78
+ if fails:
79
+ print(f"FAILURES: {len(fails)}")
80
+ return 1
81
+ print("ALL PASS")
82
+ return 0
83
+
84
+
85
+ if __name__ == "__main__":
86
+ sys.exit(main())
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env bash
2
+ # Network-free challenge: fetch_oa report builder + title-match tri-state.
3
+ # Mirrors CI usage: `bash skills/fulltext-retrieval/fetch_oa_report_challenge/verify.sh`
4
+ set -euo pipefail
5
+ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
6
+ python3 "$DIR/run_challenge.py"
@@ -0,0 +1,5 @@
1
+ DOI PMID Title
2
+ 10.1111/valid.match 1001 Deep learning for pulmonary nodule detection on chest CT
3
+ 10.2222/label.mismatch 1002 Automated segmentation of cardiac MRI using transformers
4
+ 10.3333/no.text 1003 Federated learning for multi-center radiology models
5
+ 10.9999/not.retrieved 1004 A paywalled study with no open-access copy