medsci-skills 5.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/metadata/distribution_files.json +82 -22
- package/metadata/distribution_manifest.json +1 -1
- package/package.json +1 -1
- package/skills/fulltext-retrieval/SKILL.md +52 -5
- package/skills/fulltext-retrieval/fetch_oa.py +293 -74
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/expected/projection.json +14 -0
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/extracted_text.json +4 -0
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/results.json +6 -0
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/run_challenge.py +86 -0
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/verify.sh +6 -0
- package/skills/fulltext-retrieval/fetch_oa_report_challenge/worklist.tsv +5 -0
- package/skills/fulltext-retrieval/references/find_available_pdf.js +82 -0
- package/skills/fulltext-retrieval/skill.yml +7 -2
- package/skills/lit-sync/SKILL.md +86 -6
- package/skills/lit-sync/skill.yml +3 -1
- package/skills/mllm-eval/SKILL.md +8 -0
- package/skills/mllm-eval/references/evaluation_axes.md +161 -0
- package/skills/model-evaluation/SKILL.md +12 -0
- package/skills/model-evaluation/references/metric_selection_grounding.md +139 -0
- package/skills/model-evaluation/scripts/check_metric_reporting.py +1 -1
- package/skills/model-evaluation/scripts/metric_reporting_challenge/fixture/det_good_wrapped.md +5 -0
- package/skills/model-evaluation/scripts/metric_reporting_challenge/fixture/det_no_iou.md +4 -0
- package/skills/model-evaluation/scripts/metric_reporting_challenge/verify.sh +6 -1
- package/skills/model-validation/SKILL.md +6 -0
- package/skills/model-validation/references/validation_design.md +150 -0
- package/skills/search-lit/SKILL.md +22 -67
|
@@ -2,20 +2,28 @@
|
|
|
2
2
|
"""
|
|
3
3
|
Open-access full-text PDF batch retrieval.
|
|
4
4
|
|
|
5
|
-
Pipeline:
|
|
6
|
-
OpenAlex → Crossref →
|
|
5
|
+
Pipeline: arXiv (for 10.48550/arXiv.* DOIs) → Unpaywall →
|
|
6
|
+
PMC (Europe PMC REST / OA FTP / web) → OpenAlex → Crossref →
|
|
7
|
+
landing-page scrape.
|
|
7
8
|
|
|
8
9
|
Usage:
|
|
9
10
|
python fetch_oa.py dois.txt --output pdfs/ --email user@example.com
|
|
10
|
-
python fetch_oa.py
|
|
11
|
+
python fetch_oa.py worklist.tsv -o pdfs/ -e user@example.com --verbose
|
|
12
|
+
python fetch_oa.py worklist.csv -o pdfs/ -e user@example.com --report pdfs/retrieval_report.json
|
|
13
|
+
|
|
14
|
+
Worklist formats: plain DOI-per-line, or TSV/CSV/Markdown-table with a DOI
|
|
15
|
+
column (optional PMID and Title columns). A Title column enables a best-effort
|
|
16
|
+
title cross-check (via `pdftotext` if installed) that flags mislabeled PDFs.
|
|
11
17
|
"""
|
|
12
18
|
|
|
13
19
|
import argparse
|
|
20
|
+
import csv
|
|
21
|
+
import io
|
|
14
22
|
import json
|
|
15
23
|
import logging
|
|
16
|
-
import os
|
|
17
24
|
import re
|
|
18
|
-
import
|
|
25
|
+
import shutil
|
|
26
|
+
import subprocess
|
|
19
27
|
import time
|
|
20
28
|
import urllib.error
|
|
21
29
|
import urllib.parse
|
|
@@ -25,6 +33,9 @@ from pathlib import Path
|
|
|
25
33
|
|
|
26
34
|
MIN_PDF_BYTES = 10 * 1024
|
|
27
35
|
USER_AGENT = "medsci-skills/1.0"
|
|
36
|
+
REPORT_SCHEMA_VERSION = 1
|
|
37
|
+
TITLE_MATCH_THRESHOLD = 0.6
|
|
38
|
+
RETRIEVED_STATUSES = ("oa", "pmc", "arxiv", "skip")
|
|
28
39
|
|
|
29
40
|
log = logging.getLogger("fetch_oa")
|
|
30
41
|
|
|
@@ -38,6 +49,11 @@ def _ua(email: str) -> str:
|
|
|
38
49
|
return f"{USER_AGENT} (mailto:{email})"
|
|
39
50
|
|
|
40
51
|
|
|
52
|
+
def safe_doi_name(doi: str) -> str:
|
|
53
|
+
"""Filesystem-safe filename stem for a DOI."""
|
|
54
|
+
return re.sub(r"[^\w\-.]", "_", doi)
|
|
55
|
+
|
|
56
|
+
|
|
41
57
|
def is_valid_pdf(data: bytes) -> bool:
|
|
42
58
|
return data.startswith(b"%PDF-") and len(data) >= MIN_PDF_BYTES
|
|
43
59
|
|
|
@@ -68,6 +84,84 @@ def existing_pdf_ok(path: Path) -> bool:
|
|
|
68
84
|
return False
|
|
69
85
|
|
|
70
86
|
|
|
87
|
+
# ============================================================
|
|
88
|
+
# Title cross-check (pure, offline-testable)
|
|
89
|
+
# ============================================================
|
|
90
|
+
|
|
91
|
+
def normalize_title(text: str) -> str:
|
|
92
|
+
"""Lowercase, strip punctuation, collapse whitespace."""
|
|
93
|
+
text = (text or "").lower()
|
|
94
|
+
text = re.sub(r"[^a-z0-9\s]", " ", text)
|
|
95
|
+
return " ".join(text.split())
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def title_overlap(expected_title: str, extracted_text: str) -> float:
|
|
99
|
+
"""Fraction of meaningful expected-title tokens present in extracted text.
|
|
100
|
+
|
|
101
|
+
Tokens of length <= 2 are dropped as near-stopwords. Returns 0.0 when the
|
|
102
|
+
expected title has no usable tokens.
|
|
103
|
+
"""
|
|
104
|
+
expected = {t for t in normalize_title(expected_title).split() if len(t) > 2}
|
|
105
|
+
if not expected:
|
|
106
|
+
return 0.0
|
|
107
|
+
got = set(normalize_title(extracted_text).split())
|
|
108
|
+
return len(expected & got) / len(expected)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def classify_title_match(expected_title: str, extracted_text,
|
|
112
|
+
threshold: float = TITLE_MATCH_THRESHOLD) -> str:
|
|
113
|
+
"""Tri-state title check: 'match' | 'mismatch' | 'unavailable'.
|
|
114
|
+
|
|
115
|
+
'unavailable' when no expected title is supplied or no extracted text is
|
|
116
|
+
available (e.g. pdftotext absent). A low overlap is 'mismatch' — flagged,
|
|
117
|
+
never used to auto-reject a downloaded PDF.
|
|
118
|
+
"""
|
|
119
|
+
if not expected_title or not extracted_text:
|
|
120
|
+
return "unavailable"
|
|
121
|
+
return "match" if title_overlap(expected_title, extracted_text) >= threshold else "mismatch"
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def extract_pdf_text(path: Path, max_pages: int = 2) -> str | None:
|
|
125
|
+
"""Best-effort first-page text via `pdftotext` (poppler). None if unavailable."""
|
|
126
|
+
if not shutil.which("pdftotext"):
|
|
127
|
+
return None
|
|
128
|
+
try:
|
|
129
|
+
out = subprocess.run(
|
|
130
|
+
["pdftotext", "-f", "1", "-l", str(max_pages), str(path), "-"],
|
|
131
|
+
capture_output=True, timeout=20,
|
|
132
|
+
)
|
|
133
|
+
if out.returncode == 0:
|
|
134
|
+
return out.stdout.decode("utf-8", errors="ignore")
|
|
135
|
+
except (OSError, subprocess.SubprocessError):
|
|
136
|
+
pass
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# ============================================================
|
|
141
|
+
# arXiv (direct, for 10.48550/arXiv.* DOIs)
|
|
142
|
+
# ============================================================
|
|
143
|
+
|
|
144
|
+
_ARXIV_DOI_RE = re.compile(r"^10\.48550/arxiv\.(.+)$", re.IGNORECASE)
|
|
145
|
+
_ARXIV_ID_RE = re.compile(r"^arxiv:(.+)$", re.IGNORECASE)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def arxiv_id_from_doi(doi: str) -> str | None:
|
|
149
|
+
"""Extract an arXiv ID from a DataCite arXiv DOI or a bare arXiv: id.
|
|
150
|
+
|
|
151
|
+
Handles new-style (2401.01234, 2401.01234v2) and old-style
|
|
152
|
+
(hep-th/9901001) identifiers; version suffix preserved when present.
|
|
153
|
+
"""
|
|
154
|
+
s = (doi or "").strip()
|
|
155
|
+
m = _ARXIV_DOI_RE.match(s) or _ARXIV_ID_RE.match(s)
|
|
156
|
+
return m.group(1).strip() if m else None
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def arxiv_pdf_url(doi: str) -> str | None:
|
|
160
|
+
"""Direct arXiv PDF URL for an arXiv DOI/ID (None if not an arXiv id)."""
|
|
161
|
+
aid = arxiv_id_from_doi(doi)
|
|
162
|
+
return f"https://arxiv.org/pdf/{aid}" if aid else None
|
|
163
|
+
|
|
164
|
+
|
|
71
165
|
# ============================================================
|
|
72
166
|
# 1. Unpaywall
|
|
73
167
|
# ============================================================
|
|
@@ -270,41 +364,34 @@ def download_pdf(url: str, outpath: Path, email: str) -> bool:
|
|
|
270
364
|
# 5. Main pipeline
|
|
271
365
|
# ============================================================
|
|
272
366
|
|
|
273
|
-
def gather_candidates(doi: str, email: str) -> list[str]:
|
|
274
|
-
"""Collect OA PDF candidate URLs from multiple sources."""
|
|
275
|
-
urls: list[str] = []
|
|
276
|
-
|
|
277
|
-
def add(v: str | None):
|
|
278
|
-
if v and v not in urls:
|
|
279
|
-
urls.append(v)
|
|
280
|
-
|
|
281
|
-
add(unpaywall_lookup(doi, email))
|
|
282
|
-
for v in openalex_lookup(doi, email):
|
|
283
|
-
add(v)
|
|
284
|
-
for v in crossref_lookup(doi, email):
|
|
285
|
-
add(v)
|
|
286
|
-
add(f"https://doi.org/{doi}")
|
|
287
|
-
return urls
|
|
288
|
-
|
|
289
|
-
|
|
290
367
|
def process_doi(doi: str, outdir: Path, email: str,
|
|
291
|
-
pmid: str = "") -> str:
|
|
292
|
-
"""Try to download a PDF for one DOI.
|
|
293
|
-
|
|
294
|
-
|
|
368
|
+
pmid: str = "") -> tuple[str, str]:
|
|
369
|
+
"""Try to download a PDF for one DOI.
|
|
370
|
+
|
|
371
|
+
Returns (status, source):
|
|
372
|
+
status ∈ {"arxiv", "oa", "pmc", "skip", "fail"}
|
|
373
|
+
source identifies the resolver that succeeded (e.g. "unpaywall", "pmc",
|
|
374
|
+
"openalex", "crossref", "landing", "arxiv", "existing", "").
|
|
375
|
+
"""
|
|
376
|
+
outpath = outdir / f"{safe_doi_name(doi)}.pdf"
|
|
295
377
|
|
|
296
378
|
if existing_pdf_ok(outpath):
|
|
297
|
-
return "skip"
|
|
379
|
+
return ("skip", "existing")
|
|
298
380
|
|
|
299
381
|
# Remove stale stub
|
|
300
382
|
if outpath.exists():
|
|
301
383
|
outpath.unlink(missing_ok=True)
|
|
302
384
|
|
|
385
|
+
# Step 0: arXiv direct (for 10.48550/arXiv.* DOIs)
|
|
386
|
+
ax_url = arxiv_pdf_url(doi)
|
|
387
|
+
if ax_url and download_pdf(ax_url, outpath, email):
|
|
388
|
+
return ("arxiv", "arxiv")
|
|
389
|
+
|
|
303
390
|
# Step 1: Unpaywall direct PDF URL (fastest path)
|
|
304
391
|
uw_url = unpaywall_lookup(doi, email)
|
|
305
392
|
if uw_url and ".pdf" in uw_url.lower():
|
|
306
393
|
if download_pdf(uw_url, outpath, email):
|
|
307
|
-
return "oa"
|
|
394
|
+
return ("oa", "unpaywall")
|
|
308
395
|
time.sleep(0.3)
|
|
309
396
|
|
|
310
397
|
# Step 2: PMC (try before slow landing-page scraping)
|
|
@@ -312,59 +399,158 @@ def process_doi(doi: str, outdir: Path, email: str,
|
|
|
312
399
|
if not pmcid:
|
|
313
400
|
pmcid = id_to_pmcid(doi, email)
|
|
314
401
|
if pmcid and download_pmc_pdf(pmcid, outpath, email):
|
|
315
|
-
return "pmc"
|
|
402
|
+
return ("pmc", "pmc")
|
|
316
403
|
|
|
317
404
|
# Step 3: OA candidates from OpenAlex, Crossref, landing pages
|
|
318
|
-
candidates: list[str] = []
|
|
319
|
-
|
|
320
|
-
|
|
405
|
+
candidates: list[tuple[str, str]] = []
|
|
406
|
+
seen: set[str] = set()
|
|
407
|
+
|
|
408
|
+
def add(source: str, url: str | None):
|
|
409
|
+
if url and url not in seen:
|
|
410
|
+
seen.add(url)
|
|
411
|
+
candidates.append((source, url))
|
|
412
|
+
|
|
413
|
+
add("unpaywall", uw_url)
|
|
321
414
|
for v in openalex_lookup(doi, email):
|
|
322
|
-
|
|
323
|
-
candidates.append(v)
|
|
415
|
+
add("openalex", v)
|
|
324
416
|
for v in crossref_lookup(doi, email):
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
candidates.append(f"https://doi.org/{doi}")
|
|
417
|
+
add("crossref", v)
|
|
418
|
+
add("landing", f"https://doi.org/{doi}")
|
|
328
419
|
|
|
329
|
-
for url in candidates:
|
|
420
|
+
for source, url in candidates:
|
|
330
421
|
if ".pdf" in url.lower():
|
|
331
422
|
ok = download_pdf(url, outpath, email)
|
|
332
423
|
else:
|
|
333
424
|
ok = download_from_landing(url, outpath, email)
|
|
334
425
|
if ok:
|
|
335
|
-
return "oa"
|
|
426
|
+
return ("oa", source)
|
|
336
427
|
time.sleep(0.3)
|
|
337
428
|
|
|
338
|
-
return "fail"
|
|
429
|
+
return ("fail", "")
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
def build_report(records: list[dict], results: dict[str, tuple[str, str]],
|
|
433
|
+
outdir: Path, extracted_text_by_doi: dict[str, str] | None = None,
|
|
434
|
+
threshold: float = TITLE_MATCH_THRESHOLD) -> dict:
|
|
435
|
+
"""Assemble a deterministic retrieval report (no network, no I/O writes).
|
|
436
|
+
|
|
437
|
+
records: list of {"doi", "pmid", "title"}.
|
|
438
|
+
results: doi -> (status, source) as returned by process_doi.
|
|
439
|
+
outdir: directory where PDFs were written (used for file/size lookup).
|
|
440
|
+
extracted_text_by_doi: optional doi -> first-page text for title cross-check.
|
|
441
|
+
"""
|
|
442
|
+
extracted_text_by_doi = extracted_text_by_doi or {}
|
|
443
|
+
items = []
|
|
444
|
+
for rec in records:
|
|
445
|
+
doi = rec["doi"]
|
|
446
|
+
status, source = results.get(doi, ("fail", ""))
|
|
447
|
+
path = outdir / f"{safe_doi_name(doi)}.pdf"
|
|
448
|
+
have_file = status in RETRIEVED_STATUSES and path.exists()
|
|
449
|
+
size = path.stat().st_size if have_file else 0
|
|
450
|
+
if have_file:
|
|
451
|
+
title_match = classify_title_match(
|
|
452
|
+
rec.get("title", ""), extracted_text_by_doi.get(doi), threshold)
|
|
453
|
+
else:
|
|
454
|
+
title_match = "unavailable"
|
|
455
|
+
items.append({
|
|
456
|
+
"doi": doi,
|
|
457
|
+
"pmid": rec.get("pmid", ""),
|
|
458
|
+
"title": rec.get("title", ""),
|
|
459
|
+
"status": status,
|
|
460
|
+
"source": source,
|
|
461
|
+
"file": path.name if have_file else "",
|
|
462
|
+
"size_bytes": size,
|
|
463
|
+
"title_match": title_match,
|
|
464
|
+
})
|
|
465
|
+
|
|
466
|
+
retrieved = [i for i in items if i["status"] in RETRIEVED_STATUSES]
|
|
467
|
+
not_retrieved = [i for i in items if i["status"] == "fail"]
|
|
468
|
+
return {
|
|
469
|
+
"schema_version": REPORT_SCHEMA_VERSION,
|
|
470
|
+
"generated_by": "fetch_oa.py",
|
|
471
|
+
"counts": {
|
|
472
|
+
"total": len(items),
|
|
473
|
+
"retrieved": len(retrieved),
|
|
474
|
+
"not_retrieved": len(not_retrieved),
|
|
475
|
+
"title_mismatch": sum(1 for i in items if i["title_match"] == "mismatch"),
|
|
476
|
+
},
|
|
477
|
+
"items": items,
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _norm_key(key: str) -> str:
|
|
482
|
+
return (key or "").strip().lstrip("#").strip().lower()
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _records_from_dictrows(rows) -> list[dict]:
|
|
486
|
+
records = []
|
|
487
|
+
for row in rows:
|
|
488
|
+
rec = {"doi": "", "pmid": "", "title": ""}
|
|
489
|
+
for k, v in row.items():
|
|
490
|
+
nk = _norm_key(k)
|
|
491
|
+
if nk in rec:
|
|
492
|
+
rec[nk] = (v or "").strip()
|
|
493
|
+
if rec["doi"]:
|
|
494
|
+
records.append(rec)
|
|
495
|
+
return records
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _records_from_markdown(lines: list[str]) -> list[dict]:
|
|
499
|
+
pipe_rows = [ln for ln in lines if ln.strip().startswith("|")]
|
|
500
|
+
if not pipe_rows:
|
|
501
|
+
return []
|
|
502
|
+
|
|
503
|
+
def cells(line: str) -> list[str]:
|
|
504
|
+
return [c.strip() for c in line.strip().strip("|").split("|")]
|
|
505
|
+
|
|
506
|
+
header = [_norm_key(c) for c in cells(pipe_rows[0])]
|
|
507
|
+
records = []
|
|
508
|
+
for line in pipe_rows[1:]:
|
|
509
|
+
c = cells(line)
|
|
510
|
+
# Skip the |---|---| separator row
|
|
511
|
+
if c and all(set(x) <= set("-: ") for x in c):
|
|
512
|
+
continue
|
|
513
|
+
row = dict(zip(header, c))
|
|
514
|
+
doi = (row.get("doi") or "").strip()
|
|
515
|
+
if doi:
|
|
516
|
+
records.append({
|
|
517
|
+
"doi": doi,
|
|
518
|
+
"pmid": (row.get("pmid") or "").strip(),
|
|
519
|
+
"title": (row.get("title") or "").strip(),
|
|
520
|
+
})
|
|
521
|
+
return records
|
|
339
522
|
|
|
340
523
|
|
|
341
524
|
def read_doi_file(path: Path) -> list[dict]:
|
|
342
|
-
"""Read
|
|
525
|
+
"""Read a worklist of DOIs.
|
|
526
|
+
|
|
527
|
+
Supports: plain DOI-per-line; TSV/CSV with a DOI header (optional PMID,
|
|
528
|
+
Title columns); and a Markdown pipe table with a DOI column. Each record is
|
|
529
|
+
{"doi", "pmid", "title"}.
|
|
530
|
+
"""
|
|
531
|
+
text = Path(path).read_text(encoding="utf-8")
|
|
532
|
+
lines = text.splitlines()
|
|
533
|
+
first = next((ln for ln in lines
|
|
534
|
+
if ln.strip() and not ln.strip().startswith("#")), "")
|
|
535
|
+
low = first.lower()
|
|
536
|
+
|
|
537
|
+
# Markdown pipe table with a DOI column
|
|
538
|
+
if first.strip().startswith("|") and "doi" in low:
|
|
539
|
+
return _records_from_markdown(lines)
|
|
540
|
+
|
|
541
|
+
# Delimited (TSV or CSV) with a DOI header
|
|
542
|
+
if "doi" in low and ("\t" in first or "," in first):
|
|
543
|
+
delimiter = "\t" if "\t" in first else ","
|
|
544
|
+
body = "\n".join(ln for ln in lines if not ln.strip().startswith("#"))
|
|
545
|
+
reader = csv.DictReader(io.StringIO(body), delimiter=delimiter)
|
|
546
|
+
return _records_from_dictrows(reader)
|
|
547
|
+
|
|
548
|
+
# Plain text: one DOI per line
|
|
343
549
|
records = []
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
# TSV with header containing DOI column
|
|
349
|
-
if "\t" in first_line and "doi" in first_line.lower():
|
|
350
|
-
import csv
|
|
351
|
-
reader = csv.DictReader(f, delimiter="\t")
|
|
352
|
-
for row in reader:
|
|
353
|
-
doi = ""
|
|
354
|
-
pmid = ""
|
|
355
|
-
for k, v in row.items():
|
|
356
|
-
if k.lower().strip() == "doi":
|
|
357
|
-
doi = (v or "").strip()
|
|
358
|
-
elif k.lower().strip() == "pmid":
|
|
359
|
-
pmid = (v or "").strip()
|
|
360
|
-
if doi:
|
|
361
|
-
records.append({"doi": doi, "pmid": pmid})
|
|
362
|
-
else:
|
|
363
|
-
# Plain text: one DOI per line
|
|
364
|
-
for line in f:
|
|
365
|
-
line = line.strip()
|
|
366
|
-
if line and not line.startswith("#"):
|
|
367
|
-
records.append({"doi": line, "pmid": ""})
|
|
550
|
+
for line in lines:
|
|
551
|
+
line = line.strip()
|
|
552
|
+
if line and not line.startswith("#"):
|
|
553
|
+
records.append({"doi": line, "pmid": "", "title": ""})
|
|
368
554
|
return records
|
|
369
555
|
|
|
370
556
|
|
|
@@ -372,11 +558,15 @@ def main():
|
|
|
372
558
|
parser = argparse.ArgumentParser(
|
|
373
559
|
description="Batch download open-access PDFs by DOI.")
|
|
374
560
|
parser.add_argument("input", type=Path,
|
|
375
|
-
help="
|
|
561
|
+
help="Worklist: DOIs (one per line) or TSV/CSV/Markdown "
|
|
562
|
+
"with a DOI column (optional PMID, Title)")
|
|
376
563
|
parser.add_argument("-o", "--output", type=Path, default=Path("pdfs"),
|
|
377
564
|
help="Output directory (default: pdfs/)")
|
|
378
565
|
parser.add_argument("-e", "--email", required=True,
|
|
379
566
|
help="Contact email (required by Unpaywall TOS)")
|
|
567
|
+
parser.add_argument("--report", type=Path, default=None,
|
|
568
|
+
help="Path for the JSON retrieval report "
|
|
569
|
+
"(default: <output>/retrieval_report.json)")
|
|
380
570
|
parser.add_argument("-v", "--verbose", action="store_true",
|
|
381
571
|
help="Show debug messages")
|
|
382
572
|
args = parser.parse_args()
|
|
@@ -387,33 +577,62 @@ def main():
|
|
|
387
577
|
)
|
|
388
578
|
|
|
389
579
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
580
|
+
report_path = args.report or (args.output / "retrieval_report.json")
|
|
390
581
|
records = read_doi_file(args.input)
|
|
391
582
|
print(f"Loaded {len(records)} DOIs from {args.input}")
|
|
392
583
|
|
|
393
|
-
stats = {"oa": 0, "pmc": 0, "fail": 0, "skip": 0}
|
|
584
|
+
stats = {"arxiv": 0, "oa": 0, "pmc": 0, "fail": 0, "skip": 0}
|
|
585
|
+
results: dict[str, tuple[str, str]] = {}
|
|
394
586
|
|
|
395
587
|
for i, rec in enumerate(records, 1):
|
|
396
588
|
doi = rec["doi"]
|
|
397
589
|
pmid = rec.get("pmid", "")
|
|
398
590
|
print(f" [{i}/{len(records)}] {doi}", end=" … ", flush=True)
|
|
399
591
|
|
|
400
|
-
status = process_doi(doi, args.output, args.email, pmid)
|
|
592
|
+
status, source = process_doi(doi, args.output, args.email, pmid)
|
|
593
|
+
results[doi] = (status, source)
|
|
401
594
|
stats[status] += 1
|
|
402
595
|
|
|
403
|
-
labels = {"oa": "OK (OA)", "pmc": "OK (PMC)",
|
|
596
|
+
labels = {"arxiv": "OK (arXiv)", "oa": "OK (OA)", "pmc": "OK (PMC)",
|
|
404
597
|
"fail": "FAIL", "skip": "SKIP"}
|
|
405
598
|
print(labels[status])
|
|
406
599
|
time.sleep(0.5)
|
|
407
600
|
|
|
601
|
+
# Best-effort title cross-check on successful downloads (needs pdftotext).
|
|
602
|
+
extracted: dict[str, str] = {}
|
|
603
|
+
have_titles = any(r.get("title") for r in records)
|
|
604
|
+
if have_titles and shutil.which("pdftotext"):
|
|
605
|
+
for rec in records:
|
|
606
|
+
doi = rec["doi"]
|
|
607
|
+
if not rec.get("title"):
|
|
608
|
+
continue
|
|
609
|
+
status, _ = results.get(doi, ("fail", ""))
|
|
610
|
+
if status not in RETRIEVED_STATUSES:
|
|
611
|
+
continue
|
|
612
|
+
path = args.output / f"{safe_doi_name(doi)}.pdf"
|
|
613
|
+
if path.exists():
|
|
614
|
+
text = extract_pdf_text(path)
|
|
615
|
+
if text:
|
|
616
|
+
extracted[doi] = text
|
|
617
|
+
|
|
618
|
+
report = build_report(records, results, args.output, extracted)
|
|
619
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
620
|
+
report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
|
621
|
+
|
|
408
622
|
print(f"\n--- Summary ---")
|
|
623
|
+
print(f" arXiv: {stats['arxiv']}")
|
|
409
624
|
print(f" OA: {stats['oa']}")
|
|
410
625
|
print(f" PMC: {stats['pmc']}")
|
|
411
626
|
print(f" Failed: {stats['fail']}")
|
|
412
627
|
print(f" Skipped: {stats['skip']}")
|
|
413
|
-
total = stats["oa"] + stats["pmc"] + stats["fail"]
|
|
628
|
+
total = stats["arxiv"] + stats["oa"] + stats["pmc"] + stats["fail"]
|
|
414
629
|
if total > 0:
|
|
415
|
-
pct = (stats["oa"] + stats["pmc"]) / total * 100
|
|
630
|
+
pct = (stats["arxiv"] + stats["oa"] + stats["pmc"]) / total * 100
|
|
416
631
|
print(f" Success: {pct:.0f}%")
|
|
632
|
+
mismatches = report["counts"]["title_mismatch"]
|
|
633
|
+
if mismatches:
|
|
634
|
+
print(f" Title mismatches flagged: {mismatches} (see report)")
|
|
635
|
+
print(f" Report: {report_path}")
|
|
417
636
|
|
|
418
637
|
# Write failed DOIs for manual retrieval
|
|
419
638
|
if stats["fail"] > 0:
|
|
@@ -422,10 +641,10 @@ def main():
|
|
|
422
641
|
f.write("# DOIs needing manual retrieval\n")
|
|
423
642
|
f.write("# Options: institutional access, ILL\n\n")
|
|
424
643
|
for rec in records:
|
|
425
|
-
|
|
426
|
-
pdf = args.output / f"{
|
|
644
|
+
doi = rec["doi"]
|
|
645
|
+
pdf = args.output / f"{safe_doi_name(doi)}.pdf"
|
|
427
646
|
if not existing_pdf_ok(pdf):
|
|
428
|
-
f.write(f"{
|
|
647
|
+
f.write(f"{doi}\n")
|
|
429
648
|
print(f" Manual list: {fail_path}")
|
|
430
649
|
|
|
431
650
|
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"counts": {
|
|
3
|
+
"total": 4,
|
|
4
|
+
"retrieved": 3,
|
|
5
|
+
"not_retrieved": 1,
|
|
6
|
+
"title_mismatch": 1
|
|
7
|
+
},
|
|
8
|
+
"items": [
|
|
9
|
+
{"doi": "10.1111/valid.match", "status": "oa", "source": "unpaywall", "title_match": "match"},
|
|
10
|
+
{"doi": "10.2222/label.mismatch", "status": "oa", "source": "openalex", "title_match": "mismatch"},
|
|
11
|
+
{"doi": "10.3333/no.text", "status": "pmc", "source": "pmc", "title_match": "unavailable"},
|
|
12
|
+
{"doi": "10.9999/not.retrieved", "status": "fail", "source": "", "title_match": "unavailable"}
|
|
13
|
+
]
|
|
14
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
{
|
|
2
|
+
"10.1111/valid.match": "Deep Learning for Pulmonary Nodule Detection on Chest CT\nAbstract: we present a convolutional neural network trained to detect pulmonary nodules.",
|
|
3
|
+
"10.2222/label.mismatch": "Genome-wide association study of type 2 diabetes in an Asian cohort\nIntroduction: we genotyped participants to identify susceptibility loci."
|
|
4
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Offline, network-free challenge for fetch_oa.build_report + title tri-state.
|
|
3
|
+
|
|
4
|
+
Loads committed fixtures (worklist.tsv, results.json, extracted_text.json),
|
|
5
|
+
fabricates stub PDFs in a temp dir for the "retrieved" DOIs, runs
|
|
6
|
+
fetch_oa.build_report, and asserts the resulting projection equals
|
|
7
|
+
expected/projection.json. Exercises read_doi_file (TSV+title), build_report,
|
|
8
|
+
and classify_title_match (match / mismatch / unavailable) without touching the
|
|
9
|
+
network or pdftotext. Stdlib-only.
|
|
10
|
+
"""
|
|
11
|
+
import importlib.util
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
HERE = Path(__file__).resolve().parent
|
|
18
|
+
ENGINE = HERE.parent / "fetch_oa.py"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def load_engine():
|
|
22
|
+
spec = importlib.util.spec_from_file_location("fetch_oa", ENGINE)
|
|
23
|
+
mod = importlib.util.module_from_spec(spec)
|
|
24
|
+
spec.loader.exec_module(mod)
|
|
25
|
+
return mod
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def projection(report: dict) -> dict:
|
|
29
|
+
items = sorted(
|
|
30
|
+
({"doi": i["doi"], "status": i["status"], "source": i["source"],
|
|
31
|
+
"title_match": i["title_match"]} for i in report["items"]),
|
|
32
|
+
key=lambda x: x["doi"],
|
|
33
|
+
)
|
|
34
|
+
return {"counts": report["counts"], "items": items}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
assert ENGINE.exists(), f"ENV-ERR: {ENGINE} missing"
|
|
39
|
+
m = load_engine()
|
|
40
|
+
|
|
41
|
+
records = m.read_doi_file(HERE / "worklist.tsv")
|
|
42
|
+
results = {k: tuple(v) for k, v in
|
|
43
|
+
json.loads((HERE / "results.json").read_text()).items()}
|
|
44
|
+
extracted = json.loads((HERE / "extracted_text.json").read_text())
|
|
45
|
+
expected = json.loads((HERE / "expected" / "projection.json").read_text())
|
|
46
|
+
|
|
47
|
+
fails = []
|
|
48
|
+
|
|
49
|
+
def check(label, cond):
|
|
50
|
+
print(f" {'PASS' if cond else 'FAIL'} {label}")
|
|
51
|
+
if not cond:
|
|
52
|
+
fails.append(label)
|
|
53
|
+
|
|
54
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
55
|
+
outdir = Path(tmp)
|
|
56
|
+
for rec in records:
|
|
57
|
+
status, _ = results.get(rec["doi"], ("fail", ""))
|
|
58
|
+
if status in m.RETRIEVED_STATUSES:
|
|
59
|
+
stub = outdir / f"{m.safe_doi_name(rec['doi'])}.pdf"
|
|
60
|
+
stub.write_bytes(b"%PDF-1.4\n" + b"0" * (11 * 1024))
|
|
61
|
+
report = m.build_report(records, results, outdir, extracted)
|
|
62
|
+
proj = projection(report)
|
|
63
|
+
|
|
64
|
+
by_doi = {i["doi"]: i for i in proj["items"]}
|
|
65
|
+
check("valid -> title_match 'match'",
|
|
66
|
+
by_doi["10.1111/valid.match"]["title_match"] == "match")
|
|
67
|
+
check("mislabel -> title_match 'mismatch'",
|
|
68
|
+
by_doi["10.2222/label.mismatch"]["title_match"] == "mismatch")
|
|
69
|
+
check("no-text -> title_match 'unavailable'",
|
|
70
|
+
by_doi["10.3333/no.text"]["title_match"] == "unavailable")
|
|
71
|
+
check("missing -> status 'fail'",
|
|
72
|
+
by_doi["10.9999/not.retrieved"]["status"] == "fail")
|
|
73
|
+
check("counts.retrieved == 3", proj["counts"]["retrieved"] == 3)
|
|
74
|
+
check("counts.not_retrieved == 1", proj["counts"]["not_retrieved"] == 1)
|
|
75
|
+
check("counts.title_mismatch == 1", proj["counts"]["title_mismatch"] == 1)
|
|
76
|
+
check("projection matches expected/projection.json", proj == expected)
|
|
77
|
+
|
|
78
|
+
if fails:
|
|
79
|
+
print(f"FAILURES: {len(fails)}")
|
|
80
|
+
return 1
|
|
81
|
+
print("ALL PASS")
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
sys.exit(main())
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Network-free challenge: fetch_oa report builder + title-match tri-state.
|
|
3
|
+
# Mirrors CI usage: `bash skills/fulltext-retrieval/fetch_oa_report_challenge/verify.sh`
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
6
|
+
python3 "$DIR/run_challenge.py"
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
DOI PMID Title
|
|
2
|
+
10.1111/valid.match 1001 Deep learning for pulmonary nodule detection on chest CT
|
|
3
|
+
10.2222/label.mismatch 1002 Automated segmentation of cardiac MRI using transformers
|
|
4
|
+
10.3333/no.text 1003 Federated learning for multi-center radiology models
|
|
5
|
+
10.9999/not.retrieved 1004 A paywalled study with no open-access copy
|