@cyanheads/pubmed-mcp-server 2.10.18 → 2.10.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +16 -15
- package/CLAUDE.md +16 -15
- package/Dockerfile +86 -36
- package/README.md +86 -109
- package/changelog/2.10.x/2.10.19.md +42 -0
- package/changelog/2.10.x/2.10.20.md +63 -0
- package/dist/mcp-server/resources/definitions/database-info.resource.d.ts +43 -1
- package/dist/mcp-server/resources/definitions/database-info.resource.d.ts.map +1 -1
- package/dist/mcp-server/resources/definitions/database-info.resource.js +2 -0
- package/dist/mcp-server/resources/definitions/database-info.resource.js.map +1 -1
- package/dist/mcp-server/tools/definitions/_text.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/_text.js +10 -5
- package/dist/mcp-server/tools/definitions/_text.js.map +1 -1
- package/dist/mcp-server/tools/definitions/_visible-text.d.ts +19 -0
- package/dist/mcp-server/tools/definitions/_visible-text.d.ts.map +1 -0
- package/dist/mcp-server/tools/definitions/_visible-text.js +22 -0
- package/dist/mcp-server/tools/definitions/_visible-text.js.map +1 -0
- package/dist/mcp-server/tools/definitions/convert-ids.tool.d.ts +11 -4
- package/dist/mcp-server/tools/definitions/convert-ids.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/convert-ids.tool.js +1 -1
- package/dist/mcp-server/tools/definitions/convert-ids.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.d.ts +20 -7
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.js +68 -15
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts +37 -12
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js +590 -481
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts +5 -5
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.js +31 -13
- package/dist/mcp-server/tools/definitions/find-related.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/format-citations.tool.d.ts +11 -4
- package/dist/mcp-server/tools/definitions/format-citations.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.d.ts +26 -21
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.js +109 -55
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts +15 -7
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js +19 -11
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.d.ts +20 -2
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.js +98 -14
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.d.ts +21 -6
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.js +42 -10
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts +42 -18
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.js +220 -48
- package/dist/mcp-server/tools/definitions/search-articles.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts +15 -7
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.js +13 -9
- package/dist/mcp-server/tools/definitions/spell-check.tool.js.map +1 -1
- package/dist/services/error-contracts.d.ts +43 -72
- package/dist/services/error-contracts.d.ts.map +1 -1
- package/dist/services/error-contracts.js +44 -76
- package/dist/services/error-contracts.js.map +1 -1
- package/dist/services/europe-pmc/api-client.d.ts +8 -2
- package/dist/services/europe-pmc/api-client.d.ts.map +1 -1
- package/dist/services/europe-pmc/api-client.js +13 -8
- package/dist/services/europe-pmc/api-client.js.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.d.ts +25 -2
- package/dist/services/europe-pmc/europe-pmc-service.d.ts.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.js +90 -25
- package/dist/services/europe-pmc/europe-pmc-service.js.map +1 -1
- package/dist/services/europe-pmc/types.d.ts +6 -1
- package/dist/services/europe-pmc/types.d.ts.map +1 -1
- package/dist/services/ncbi/api-client.d.ts.map +1 -1
- package/dist/services/ncbi/api-client.js +2 -3
- package/dist/services/ncbi/api-client.js.map +1 -1
- package/dist/services/ncbi/formatting/citation-formatter.d.ts +24 -0
- package/dist/services/ncbi/formatting/citation-formatter.d.ts.map +1 -1
- package/dist/services/ncbi/formatting/citation-formatter.js +147 -46
- package/dist/services/ncbi/formatting/citation-formatter.js.map +1 -1
- package/dist/services/ncbi/ncbi-service.d.ts +18 -2
- package/dist/services/ncbi/ncbi-service.d.ts.map +1 -1
- package/dist/services/ncbi/ncbi-service.js +182 -90
- package/dist/services/ncbi/ncbi-service.js.map +1 -1
- package/dist/services/ncbi/parsing/article-parser.d.ts +15 -1
- package/dist/services/ncbi/parsing/article-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/article-parser.js +67 -29
- package/dist/services/ncbi/parsing/article-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/esummary-parser.d.ts +9 -0
- package/dist/services/ncbi/parsing/esummary-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/esummary-parser.js +26 -2
- package/dist/services/ncbi/parsing/esummary-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/pmc-article-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-article-parser.js +395 -147
- package/dist/services/ncbi/parsing/pmc-article-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts +73 -5
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js +223 -27
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js.map +1 -1
- package/dist/services/ncbi/parsing/text-helpers.d.ts +8 -6
- package/dist/services/ncbi/parsing/text-helpers.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/text-helpers.js +95 -19
- package/dist/services/ncbi/parsing/text-helpers.js.map +1 -1
- package/dist/services/ncbi/response-handler.d.ts +20 -12
- package/dist/services/ncbi/response-handler.d.ts.map +1 -1
- package/dist/services/ncbi/response-handler.js +134 -38
- package/dist/services/ncbi/response-handler.js.map +1 -1
- package/dist/services/ncbi/types.d.ts +35 -2
- package/dist/services/ncbi/types.d.ts.map +1 -1
- package/dist/services/openalex/api-client.d.ts.map +1 -1
- package/dist/services/openalex/api-client.js +6 -7
- package/dist/services/openalex/api-client.js.map +1 -1
- package/dist/services/openalex/openalex-service.d.ts +10 -6
- package/dist/services/openalex/openalex-service.d.ts.map +1 -1
- package/dist/services/openalex/openalex-service.js +14 -12
- package/dist/services/openalex/openalex-service.js.map +1 -1
- package/dist/services/unpaywall/unpaywall-service.d.ts.map +1 -1
- package/dist/services/unpaywall/unpaywall-service.js +1 -3
- package/dist/services/unpaywall/unpaywall-service.js.map +1 -1
- package/package.json +16 -13
- package/server.json +10 -24
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Full-text fetch tool. Resolves full-text articles through a
|
|
3
3
|
* three-stage chain: NCBI PMC EFetch → Europe PMC `fullTextXML` → Unpaywall.
|
|
4
|
-
* Accepts three
|
|
4
|
+
* Accepts three input fields, alone or together, naming at most 10 distinct
|
|
5
|
+
* identifiers between them:
|
|
5
6
|
*
|
|
6
7
|
* - `pmcids` — fetch directly by PMC ID, once per record however it is
|
|
7
8
|
* spelled (`PMC123`, `pmc123`, `123`, zero-padded `PMC0123`), and reported
|
|
@@ -16,6 +17,13 @@
|
|
|
16
17
|
* DOIs are case-insensitive: every casing of one DOI runs the chain once,
|
|
17
18
|
* and `unavailable[]` reports each casing as the caller wrote it.
|
|
18
19
|
*
|
|
20
|
+
* Routing runs `pmcids`, then `pmids`, then `dois`. Every tier keys its work on
|
|
21
|
+
* the record — PMC on the PMCID, Europe PMC on the search hit's `source` + `id`,
|
|
22
|
+
* Unpaywall on the lowercased DOI — and the first id to reach a record owns it:
|
|
23
|
+
* a later id naming the same article joins the owner's chain as another caller
|
|
24
|
+
* id, so the article is fetched and returned once. Each `unavailable[]` entry
|
|
25
|
+
* and deferred id carries the field its id was sent in.
|
|
26
|
+
*
|
|
19
27
|
* Output uses a discriminated union on `source` (`pmc` | `unpaywall`) with an
|
|
20
28
|
* extra `viaSource` discriminator that records which layer produced the
|
|
21
29
|
* content. EPMC's JATS reuses the `pmc` schema shape because it's the same
|
|
@@ -30,6 +38,7 @@
|
|
|
30
38
|
* @module src/mcp-server/tools/definitions/fetch-fulltext.tool
|
|
31
39
|
*/
|
|
32
40
|
import { tool, z } from '@cyanheads/mcp-ts-core';
|
|
41
|
+
import { JsonRpcErrorCode, serializationError } from '@cyanheads/mcp-ts-core/errors';
|
|
33
42
|
import { htmlExtractor, pdfParser } from '@cyanheads/mcp-ts-core/utils';
|
|
34
43
|
import { getServerConfig } from '../../../config/server-config.js';
|
|
35
44
|
import { NCBI_SERVICE_ERRORS } from '../../../services/error-contracts.js';
|
|
@@ -45,6 +54,7 @@ import { fitWholeItems } from './_budget.js';
|
|
|
45
54
|
import { conceptMeta, EDAM_DATA_RETRIEVAL, SCHEMA_SCHOLARLY_ARTICLE } from './_concepts.js';
|
|
46
55
|
import { doiStringSchema, normalizePmid, pmcidStringSchema, pmidStringSchema } from './_schemas.js';
|
|
47
56
|
import { escapeMarkdownInline, escapeMarkdownTableCell, sliceAtWordBoundary } from './_text.js';
|
|
57
|
+
import { hasVisibleText } from './_visible-text.js';
|
|
48
58
|
/**
|
|
49
59
|
* Canonical digits of a PMC ID {@link pmcidStringSchema} accepted: the `PMC`
|
|
50
60
|
* prefix dropped in any case, and leading zeros stripped the way
|
|
@@ -67,6 +77,31 @@ function matchesSectionFilter(title, lowerFilter) {
|
|
|
67
77
|
function lowerCase(s) {
|
|
68
78
|
return s.toLowerCase();
|
|
69
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Split a `sections` filter into the terms it filters on and the paths of its
|
|
82
|
+
* blank elements. An exactly-empty element is skipped — form clients send one
|
|
83
|
+
* for a field left untouched — so `[""]` filters on nothing and `["Methods", ""]`
|
|
84
|
+
* on Methods alone. An element with no visible character (`hasVisibleText`)
|
|
85
|
+
* names no heading — as a substring, a single space matches every multi-word
|
|
86
|
+
* title and a zero-width space usually none — so it is reported by its index
|
|
87
|
+
* (`sections.1`) instead. A kept term is trimmed of surrounding whitespace, so
|
|
88
|
+
* `[" Methods"]` matches what `["Methods"]` does — as a substring, a padded
|
|
89
|
+
* term misses every heading the padding does not happen to fit. Whitespace
|
|
90
|
+
* inside a term and every other character are matched literally. (#186)
|
|
91
|
+
*/
|
|
92
|
+
function splitSectionFilter(sections = []) {
|
|
93
|
+
const blank = [];
|
|
94
|
+
const terms = [];
|
|
95
|
+
for (const [index, term] of sections.entries()) {
|
|
96
|
+
if (term === '')
|
|
97
|
+
continue;
|
|
98
|
+
if (hasVisibleText(term))
|
|
99
|
+
terms.push(term.trim());
|
|
100
|
+
else
|
|
101
|
+
blank.push(`sections.${index}`);
|
|
102
|
+
}
|
|
103
|
+
return { blank, terms };
|
|
104
|
+
}
|
|
70
105
|
/**
|
|
71
106
|
* Prune a section tree to the branches a `sections` filter selects, matching
|
|
72
107
|
* titles at every nesting depth rather than the top level alone. (#126)
|
|
@@ -437,8 +472,11 @@ const TableSchema = z
|
|
|
437
472
|
.describe('How many leading `rows` entries are header rows — a `<thead>` block, or leading rows made entirely of `<th>`. 0 when the table declares none. Several header rows stack: read one column top to bottom for its full header path.'),
|
|
438
473
|
rows: z
|
|
439
474
|
.array(z.array(z.string()).describe('One row, as cell text by grid column'))
|
|
440
|
-
.describe('Cell text by row, in document order, one entry per grid column. `colspan` and `rowspan` are expanded, so a cell covering several columns or rows repeats its text across each cell it covers and a well-formed table is rectangular — align on position from the left, and read a repeated value as one spanning cell rather than several measurements. Empty when `unextractableReason` is set.'),
|
|
441
|
-
footnotes: z
|
|
475
|
+
.describe('Cell text by row, in document order, one entry per grid column. `colspan` and `rowspan` are expanded, so a cell covering several columns or rows repeats its text across each cell it covers and a well-formed table is rectangular — align on position from the left, and read a repeated value as one spanning cell rather than several measurements. A line break in a cell — a `<break/>`, a paragraph, or a list item in the source — is kept as `\\n`, so `1.08\\n2.44` is two values; superscripts and other inline markup are never spaced (`0.108a`). Empty when `unextractableReason` is set.'),
|
|
476
|
+
footnotes: z
|
|
477
|
+
.string()
|
|
478
|
+
.optional()
|
|
479
|
+
.describe('`<table-wrap-foot>` text, one line per footnote or paragraph, joined by `\\n`. A footnote marker deposited as a `<label>` leads its line, followed by a space (`* No patient …`).'),
|
|
442
480
|
unextractableReason: z
|
|
443
481
|
.enum(['cals-tgroup', 'graphic-only', 'no-rows'])
|
|
444
482
|
.optional()
|
|
@@ -551,11 +589,11 @@ const UnpaywallArticleSchema = z
|
|
|
551
589
|
pmcId: z
|
|
552
590
|
.string()
|
|
553
591
|
.optional()
|
|
554
|
-
.describe('PMC ID this article was requested under, in `PMC<digits>` form — present
|
|
592
|
+
.describe('PMC ID this article was requested under, in `PMC<digits>` form — present when a `pmcids` entry named it, absent when it was requested under a PMID or DOI. When several identifiers name one article, it is requested under the first to reach it: `pmcids`, then `pmids`, then `dois`. Ties the article back to the requested identifier, which `unavailable[]` keys on for the ids that found nothing.'),
|
|
555
593
|
pmid: z
|
|
556
594
|
.string()
|
|
557
595
|
.optional()
|
|
558
|
-
.describe('PubMed ID when
|
|
596
|
+
.describe('PubMed ID this article was requested under — present when it was requested under a `pmids` entry, absent when under a PMC ID or DOI (see `pmcId` for which identifier an article several of them name is requested under)'),
|
|
559
597
|
pubmedUrl: z.string().optional().describe('PubMed URL — present when `pmid` is set'),
|
|
560
598
|
doi: z.string().describe('DOI used to locate the open-access copy'),
|
|
561
599
|
sourceUrl: z.string().describe('URL the content was fetched from'),
|
|
@@ -637,7 +675,9 @@ const UnavailableSchema = z
|
|
|
637
675
|
id: z
|
|
638
676
|
.string()
|
|
639
677
|
.describe('Identifier the chain could not resolve — PMID, PMCID, or DOI per `idType`'),
|
|
640
|
-
idType: z
|
|
678
|
+
idType: z
|
|
679
|
+
.enum(['pmid', 'pmcid', 'doi'])
|
|
680
|
+
.describe('Which input field the id was sent in — `pmids`, `pmcids`, or `dois` respectively'),
|
|
641
681
|
reason: UnavailableReasonSchema,
|
|
642
682
|
triedTiers: z
|
|
643
683
|
.array(TriedTierSchema)
|
|
@@ -778,10 +818,23 @@ const DeferredSchema = z
|
|
|
778
818
|
.describe('Articles the chain resolved but withheld to stay under the ceiling'),
|
|
779
819
|
idType: z
|
|
780
820
|
.enum(['pmid', 'pmcid', 'doi'])
|
|
781
|
-
.
|
|
821
|
+
.optional()
|
|
822
|
+
.describe('The input field every deferred id was sent in — re-submit them as `pmids`, `pmcids`, or `dois` respectively. Matches the `idType` on `unavailable` entries. Absent when the deferred ids came from more than one field; `pmcids` / `pmids` / `dois` then group them.'),
|
|
782
823
|
ids: z
|
|
783
824
|
.array(z.string())
|
|
784
|
-
.describe('Identifiers of the deferred articles, in response order, keyed as they were requested (PMC IDs in `PMC<digits>` form). Re-call `pubmed_fetch_fulltext` with these under the `idType`
|
|
825
|
+
.describe('Identifiers of the deferred articles, in response order, keyed as they were requested (PMC IDs in `PMC<digits>` form). Re-call `pubmed_fetch_fulltext` with these under the `idType` field — or, when `idType` is absent, with the `pmcids` / `pmids` / `dois` lists below — and the same other inputs. Never contains an id from `unavailable`.'),
|
|
826
|
+
pmcids: z
|
|
827
|
+
.array(z.string())
|
|
828
|
+
.optional()
|
|
829
|
+
.describe('The deferred ids sent as PMC IDs, in response order — re-submit them as `pmcids`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a PMC ID.'),
|
|
830
|
+
pmids: z
|
|
831
|
+
.array(z.string())
|
|
832
|
+
.optional()
|
|
833
|
+
.describe('The deferred ids sent as PMIDs, in response order — re-submit them as `pmids`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a PMID.'),
|
|
834
|
+
dois: z
|
|
835
|
+
.array(z.string())
|
|
836
|
+
.optional()
|
|
837
|
+
.describe('The deferred ids sent as DOIs, in response order — re-submit them as `dois`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a DOI.'),
|
|
785
838
|
nextDeferredCharacters: z
|
|
786
839
|
.number()
|
|
787
840
|
.describe('Serialized size of the next deferred article — the first entry in `ids`, where the response stopped. Raise `maxResponseCharacters` to at least this to make progress; a smaller article further down `ids` cannot be reached until this one fits.'),
|
|
@@ -1179,9 +1232,24 @@ function buildTruncationNotice(truncation) {
|
|
|
1179
1232
|
*/
|
|
1180
1233
|
function buildDeferralNotice(deferred) {
|
|
1181
1234
|
const spent = deferred.returnedCharacters === 0
|
|
1182
|
-
? `
|
|
1235
|
+
? `The first article alone exceeds the requested maxResponseCharacters of ${deferred.maxResponseCharacters}, so none were returned.`
|
|
1183
1236
|
: `Response character budget reached: ${deferred.returnedCharacters} of ${deferred.maxResponseCharacters} characters returned.`;
|
|
1184
|
-
|
|
1237
|
+
const resend = deferred.idType
|
|
1238
|
+
? `Re-call pubmed_fetch_fulltext with those ids under \`${deferred.idType}s\``
|
|
1239
|
+
: `Re-call pubmed_fetch_fulltext with each id under its own field (${deferredFieldLists(deferred)})`;
|
|
1240
|
+
return `${spent} ${deferred.deferredCount} resolved article(s) were deferred whole: ${deferred.ids.join(', ')}. ${resend} to retrieve them, or raise maxResponseCharacters to at least ${deferred.nextDeferredCharacters} — the size of the next deferred article.`;
|
|
1241
|
+
}
|
|
1242
|
+
/**
|
|
1243
|
+
* The per-field lists of a deferral that spans fields, in field order —
|
|
1244
|
+
* `` `pmcids`: PMC1; `dois`: 10.1/x `` — or `''` when `idType` covers them all.
|
|
1245
|
+
*/
|
|
1246
|
+
function deferredFieldLists(deferred) {
|
|
1247
|
+
return ['pmcids', 'pmids', 'dois']
|
|
1248
|
+
.flatMap((field) => {
|
|
1249
|
+
const ids = deferred[field];
|
|
1250
|
+
return ids?.length ? [`\`${field}\`: ${ids.join(', ')}`] : [];
|
|
1251
|
+
})
|
|
1252
|
+
.join('; ');
|
|
1185
1253
|
}
|
|
1186
1254
|
// ─── Tool Definition ─────────────────────────────────────────────────────────
|
|
1187
1255
|
/**
|
|
@@ -1216,11 +1284,13 @@ export function buildFulltextDescription(tiers) {
|
|
|
1216
1284
|
: tiers.unpaywall
|
|
1217
1285
|
? '; DOIs with no PMC copy recover via Unpaywall open access'
|
|
1218
1286
|
: '';
|
|
1219
|
-
const input = `
|
|
1287
|
+
const input = `Name articles by \`pmcids\` (PMC IDs directly), \`pmids\` (PubMed IDs, auto-resolved), \`dois\` (DOIs, auto-resolved to PMC via the ID Converter${doiTail}), or any mix of the three — up to 10 distinct identifiers per call; an article several of them name is fetched and returned once.`;
|
|
1220
1288
|
const budget = 'Two independent character controls: `maxCharacters` caps body text per article, `maxResponseCharacters` caps the whole response and defers articles past the ceiling whole, listing them in `deferred.ids` for a follow-up call.';
|
|
1221
1289
|
return `${base} ${fallback} ${input} ${budget}`;
|
|
1222
1290
|
}
|
|
1223
1291
|
const serverConfig = getServerConfig();
|
|
1292
|
+
/** Distinct identifiers one call may name across `pmcids`, `pmids`, and `dois`. (#192) */
|
|
1293
|
+
const MAX_IDENTIFIERS = 10;
|
|
1224
1294
|
export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
1225
1295
|
description: buildFulltextDescription({
|
|
1226
1296
|
europePmc: serverConfig.europepmcEnabled,
|
|
@@ -1232,8 +1302,18 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1232
1302
|
// Only the ID routing's `idConvert` calls run unwrapped. Every Europe PMC and
|
|
1233
1303
|
// Unpaywall call sits behind a catch that folds the failure into
|
|
1234
1304
|
// `unavailable[].triedTiers`, so their service reasons never reach a caller
|
|
1235
|
-
// and are not declared here. (#168)
|
|
1236
|
-
|
|
1305
|
+
// and are not declared here. (#168) `blank_filter` is the handler's own input
|
|
1306
|
+
// check, raised before any upstream request. (#186)
|
|
1307
|
+
errors: [
|
|
1308
|
+
...NCBI_SERVICE_ERRORS,
|
|
1309
|
+
{
|
|
1310
|
+
reason: 'blank_filter',
|
|
1311
|
+
code: JsonRpcErrorCode.ValidationError,
|
|
1312
|
+
when: 'A `sections` element holds no visible character — only whitespace, or invisible characters such as a zero-width space — so it names no heading. An exactly-empty element is not blank here; it is skipped.',
|
|
1313
|
+
recovery: 'Give each named `sections` element a heading term such as Methods, or make it an empty string, which is skipped; omit `sections` for no filter. The same blank element will be rejected again.',
|
|
1314
|
+
retryable: false,
|
|
1315
|
+
},
|
|
1316
|
+
],
|
|
1237
1317
|
input: z
|
|
1238
1318
|
.object({
|
|
1239
1319
|
pmcids: z
|
|
@@ -1241,19 +1321,19 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1241
1321
|
.min(1)
|
|
1242
1322
|
.max(10)
|
|
1243
1323
|
.optional()
|
|
1244
|
-
.describe('PMC IDs to fetch (e.g. ["PMC9575052"]).
|
|
1324
|
+
.describe('PMC IDs to fetch (e.g. ["PMC9575052"]). Combinable with `pmids` and `dois` — at most 10 distinct identifiers across the three. PMC IDs with no retrievable full text fall through to Europe PMC, then to Unpaywall on the DOI the chain resolves for them.'),
|
|
1245
1325
|
pmids: z
|
|
1246
1326
|
.array(pmidStringSchema)
|
|
1247
1327
|
.min(1)
|
|
1248
1328
|
.max(10)
|
|
1249
1329
|
.optional()
|
|
1250
|
-
.describe('PubMed IDs.
|
|
1330
|
+
.describe('PubMed IDs. Combinable with `pmcids` and `dois` — at most 10 distinct identifiers across the three. Articles in PMC are returned as structured JATS; articles not in PMC fall through to Europe PMC (when EPMC has a `fullTextXML`), then to Unpaywall when `UNPAYWALL_EMAIL` is set and a DOI is available.'),
|
|
1251
1331
|
dois: z
|
|
1252
1332
|
.array(doiStringSchema)
|
|
1253
1333
|
.min(1)
|
|
1254
1334
|
.max(10)
|
|
1255
1335
|
.optional()
|
|
1256
|
-
.describe('DOIs to resolve (e.g. ["10.21203/rs.3.rs-9010375/v1"]), one per element.
|
|
1336
|
+
.describe('DOIs to resolve (e.g. ["10.21203/rs.3.rs-9010375/v1"]), one per element. Combinable with `pmcids` and `pmids` — at most 10 distinct identifiers across the three. Resolved to a PMCID via the PMC ID Converter and returned as structured JATS when the article is in PMC; DOIs with no PMC counterpart (preprints, EPMC-only OA) fall through to Europe PMC, then Unpaywall, when those layers are enabled.'),
|
|
1257
1337
|
includeReferences: z
|
|
1258
1338
|
.boolean()
|
|
1259
1339
|
.default(false)
|
|
@@ -1276,7 +1356,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1276
1356
|
sections: z
|
|
1277
1357
|
.array(z.string())
|
|
1278
1358
|
.optional()
|
|
1279
|
-
.describe('Filter to specific sections by title (e.g. ["Introduction", "Methods", "Results", "Discussion"]). A term matches a section or subsection title at any nesting depth, case-insensitively, as a substring — "resul" matches "Results". A section whose own title matches is returned whole; one kept only because a nested subsection matched keeps its heading as a breadcrumb, with its own text cleared and only the matching branch beneath it. Tables and assets narrow with the filter: one whose section did not survive, or that names no section, is dropped. Applies to `source=pmc` results only.'),
|
|
1359
|
+
.describe('Filter to specific sections by title (e.g. ["Introduction", "Methods", "Results", "Discussion"]). A term matches a section or subsection title at any nesting depth, case-insensitively, as a substring — "resul" matches "Results". A section whose own title matches is returned whole; one kept only because a nested subsection matched keeps its heading as a breadcrumb, with its own text cleared and only the matching branch beneath it. Tables and assets narrow with the filter: one whose section did not survive, or that names no section, is dropped. Empty strings are skipped, so `[""]` applies no filter; an element of only whitespace or invisible characters, such as a zero-width space, is rejected. Applies to `source=pmc` results only.'),
|
|
1280
1360
|
maxCharacters: z
|
|
1281
1361
|
.number()
|
|
1282
1362
|
.int()
|
|
@@ -1303,8 +1383,27 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1303
1383
|
.default('truncate')
|
|
1304
1384
|
.describe('How to spend `maxCharacters` across an article that exceeds it. truncate: fill sections in document order, so early sections stay whole, the section the budget runs out in is cut, and every section or subsection past that point is dropped (counted in `truncation.omittedSections`). outline: split the budget evenly so every section and subsection keeps its heading, and an excerpt as far as the budget reaches — a heading the budget left empty is marked as such in the rendered text. Use it to survey what an article contains before requesting specific `sections`. Ignored when no budget is set, and identical for `source=unpaywall` bodies, which have no headings to preserve.'),
|
|
1305
1385
|
})
|
|
1306
|
-
|
|
1307
|
-
|
|
1386
|
+
// Counted in each field's canonical form, the one its chain keys on, so the
|
|
1387
|
+
// cap is settled before any upstream request and holds every fallback tier
|
|
1388
|
+
// to at most 10 candidates. (#192)
|
|
1389
|
+
.superRefine((v, refinement) => {
|
|
1390
|
+
const distinct = new Set([
|
|
1391
|
+
...(v.pmcids ?? []).map((id) => `pmcid:${normalizePmcId(id)}`),
|
|
1392
|
+
...(v.pmids ?? []).map((id) => `pmid:${normalizePmid(id)}`),
|
|
1393
|
+
...(v.dois ?? []).map((id) => `doi:${id.toLowerCase()}`),
|
|
1394
|
+
]);
|
|
1395
|
+
if (distinct.size === 0) {
|
|
1396
|
+
refinement.addIssue({
|
|
1397
|
+
code: 'custom',
|
|
1398
|
+
message: 'Provide at least one of `pmcids`, `pmids`, or `dois`.',
|
|
1399
|
+
});
|
|
1400
|
+
}
|
|
1401
|
+
else if (distinct.size > MAX_IDENTIFIERS) {
|
|
1402
|
+
refinement.addIssue({
|
|
1403
|
+
code: 'custom',
|
|
1404
|
+
message: `\`pmcids\`, \`pmids\`, and \`dois\` together name ${distinct.size} distinct identifiers; a call accepts at most ${MAX_IDENTIFIERS} — split them across calls.`,
|
|
1405
|
+
});
|
|
1406
|
+
}
|
|
1308
1407
|
}),
|
|
1309
1408
|
output: z.object({
|
|
1310
1409
|
articles: z.array(ArticleSchema).describe('Full-text articles'),
|
|
@@ -1314,7 +1413,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1314
1413
|
unavailable: z
|
|
1315
1414
|
.array(UnavailableSchema)
|
|
1316
1415
|
.optional()
|
|
1317
|
-
.describe('Per-identifier explanations for any requested PMIDs, PMCIDs, or DOIs with no returnable full text. `idType`
|
|
1416
|
+
.describe('Per-identifier explanations for any requested PMIDs, PMCIDs, or DOIs with no returnable full text. `idType` names the field each id was sent in. Distinct from `deferred`: nothing here is retrievable by re-calling, and an id never appears in both.'),
|
|
1318
1417
|
truncation: TruncationSchema.optional(),
|
|
1319
1418
|
deferred: DeferredSchema.optional(),
|
|
1320
1419
|
}),
|
|
@@ -1336,17 +1435,27 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1336
1435
|
.describe('True when a character budget shortened at least one returned body, or withheld a whole article. Absent when every resolved article is present with its full post-filter body. The per-article body accounting is in `truncation`; the withheld ids are in `deferred`.'),
|
|
1337
1436
|
},
|
|
1338
1437
|
async handler(input, ctx) {
|
|
1438
|
+
const idCount = (input.pmcids?.length ?? 0) + (input.pmids?.length ?? 0) + (input.dois?.length ?? 0);
|
|
1339
1439
|
ctx.log.info('Executing pubmed_fetch_fulltext', {
|
|
1340
1440
|
hasPmcids: !!input.pmcids,
|
|
1341
1441
|
hasPmids: !!input.pmids,
|
|
1342
1442
|
hasDois: !!input.dois,
|
|
1343
|
-
idCount
|
|
1443
|
+
idCount,
|
|
1344
1444
|
});
|
|
1445
|
+
// A blank `sections` element fails the call before any upstream request;
|
|
1446
|
+
// every stage filters on the nonblank terms alone. (#186)
|
|
1447
|
+
const sectionFilter = splitSectionFilter(input.sections);
|
|
1448
|
+
if (sectionFilter.blank.length > 0) {
|
|
1449
|
+
const one = sectionFilter.blank.length === 1;
|
|
1450
|
+
throw ctx.fail('blank_filter', `${sectionFilter.blank.map((path) => `\`${path}\``).join(', ')} ${one ? 'holds' : 'hold'} only whitespace or invisible characters, so ${one ? 'it names' : 'they name'} no section heading.`, { fields: sectionFilter.blank });
|
|
1451
|
+
}
|
|
1452
|
+
const filters = { ...input, sections: sectionFilter.terms };
|
|
1345
1453
|
// ── Chain tracking ──────────────────────────────────────────────────────
|
|
1346
1454
|
// Per-input-id tier history (the `triedTiers` array on unavailable entries).
|
|
1347
|
-
// Keys:
|
|
1348
|
-
//
|
|
1349
|
-
//
|
|
1455
|
+
// Keys: canonical PMID for a `pmids` entry, prefixed PMCID for a `pmcids`
|
|
1456
|
+
// entry, first-submitted casing for a `dois` entry — disjoint shapes, so
|
|
1457
|
+
// one map holds all three. `recoveredIds` collects ids the chain produced an
|
|
1458
|
+
// article for, so we can skip them when building `unavailable[]`.
|
|
1350
1459
|
const chainByInput = new Map();
|
|
1351
1460
|
const recoveredIds = new Set();
|
|
1352
1461
|
// Per-input-id set of tiers this deployment has not configured AND that
|
|
@@ -1360,13 +1469,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1360
1469
|
tiers.add(tier);
|
|
1361
1470
|
unqueriedByInput.set(inputId, tiers);
|
|
1362
1471
|
};
|
|
1363
|
-
// Back-map from a
|
|
1364
|
-
//
|
|
1365
|
-
//
|
|
1472
|
+
// Back-map from a prefixed PMCID to the input id that owns its record — the
|
|
1473
|
+
// PMC ID itself, or the PMID or DOI the converter placed on it — so the PMC
|
|
1474
|
+
// stage attributes recoveries and misses to the original input id.
|
|
1366
1475
|
const pmcidToInputId = new Map();
|
|
1367
|
-
// DOI
|
|
1368
|
-
//
|
|
1369
|
-
|
|
1476
|
+
// A DOI the routing learned for a PMC record — from the converter, or from a
|
|
1477
|
+
// `dois` entry that named it — so a PMC miss can still reach Unpaywall
|
|
1478
|
+
// without another lookup, whichever field owns the record.
|
|
1479
|
+
const doiByPmcRecord = new Map();
|
|
1370
1480
|
// Ids of returned articles whose `sections` filter removed every body
|
|
1371
1481
|
// section — collected across the PMC and EPMC stages to drive one recovery
|
|
1372
1482
|
// notice via ctx.enrich.notice (#80).
|
|
@@ -1385,19 +1495,46 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1385
1495
|
// caller can re-submit rather than whatever id the article happens to
|
|
1386
1496
|
// carry — a `pmids` request recovers articles keyed by PMCID. (#100)
|
|
1387
1497
|
const inputIdByArticle = new Map();
|
|
1388
|
-
// The caller's own spellings of each chain key,
|
|
1389
|
-
// `deferred
|
|
1390
|
-
// PMID's chain runs on its canonical form
|
|
1391
|
-
//
|
|
1392
|
-
//
|
|
1393
|
-
//
|
|
1498
|
+
// The caller's own spellings of each chain key, each with the field it was
|
|
1499
|
+
// sent in, so `unavailable[]` and `deferred` report what was submitted and
|
|
1500
|
+
// where. A PMID's chain runs on its canonical form and a DOI's once however
|
|
1501
|
+
// many casings name it, so those keys gather several spellings; a PMC ID is
|
|
1502
|
+
// reported in its `PMC<digits>` key form. The first entry is the key's own,
|
|
1503
|
+
// the id that opened the chain; an id that reaches a record another one
|
|
1504
|
+
// already owns is folded in after it (see `joinChain`). (#161, #166, #192)
|
|
1394
1505
|
const callerIds = new Map();
|
|
1395
|
-
const addCallerId = (key,
|
|
1506
|
+
const addCallerId = (key, callerId) => {
|
|
1396
1507
|
const spellings = callerIds.get(key) ?? [];
|
|
1397
|
-
if (!spellings.
|
|
1398
|
-
spellings.push(
|
|
1508
|
+
if (!spellings.some((s) => s.id === callerId.id && s.idType === callerId.idType)) {
|
|
1509
|
+
spellings.push(callerId);
|
|
1510
|
+
}
|
|
1399
1511
|
callerIds.set(key, spellings);
|
|
1400
1512
|
};
|
|
1513
|
+
// A joined spelling's own tiers from before its join, and the length of the
|
|
1514
|
+
// owner's chain at that point: it reports those tiers, then only what the
|
|
1515
|
+
// owner's chain gained afterwards, never a step another id took. (#192)
|
|
1516
|
+
const ownChainOf = new Map();
|
|
1517
|
+
/**
|
|
1518
|
+
* Fold `joiner` into `owner` once both are known to name one record: the
|
|
1519
|
+
* joiner's spellings become caller ids of the owner's chain, so the record is
|
|
1520
|
+
* worked once and every spelling is recovered with it, or reported
|
|
1521
|
+
* unavailable with its own tiers up to the join and the owner's after. (#192)
|
|
1522
|
+
*/
|
|
1523
|
+
const joinChain = (joiner, owner) => {
|
|
1524
|
+
const joinerChain = chainByInput.get(joiner) ?? [];
|
|
1525
|
+
const from = chainByInput.get(owner)?.length ?? 0;
|
|
1526
|
+
for (const callerId of callerIds.get(joiner) ?? []) {
|
|
1527
|
+
const own = ownChainOf.get(callerId);
|
|
1528
|
+
const tried = [...(own?.tried ?? []), ...joinerChain.slice(own?.from ?? 0)];
|
|
1529
|
+
ownChainOf.set(callerId, { tried, from });
|
|
1530
|
+
addCallerId(owner, callerId);
|
|
1531
|
+
}
|
|
1532
|
+
callerIds.delete(joiner);
|
|
1533
|
+
chainByInput.delete(joiner);
|
|
1534
|
+
unqueriedByInput.delete(joiner);
|
|
1535
|
+
if (bodylessInputIds.delete(joiner))
|
|
1536
|
+
bodylessInputIds.add(owner);
|
|
1537
|
+
};
|
|
1401
1538
|
const budget = {
|
|
1402
1539
|
overflowMode: input.overflowMode,
|
|
1403
1540
|
...(input.maxCharacters !== undefined && { maxCharacters: input.maxCharacters }),
|
|
@@ -1405,18 +1542,19 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1405
1542
|
maxCharactersPerSection: input.maxCharactersPerSection,
|
|
1406
1543
|
}),
|
|
1407
1544
|
};
|
|
1408
|
-
|
|
1409
|
-
//
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
//
|
|
1416
|
-
//
|
|
1417
|
-
//
|
|
1418
|
-
//
|
|
1419
|
-
//
|
|
1545
|
+
// ── Routing → produce the buckets the staged chain consumes ─────────────
|
|
1546
|
+
// `pmcids`, then `pmids`, then `dois`: the first id to reach a record owns
|
|
1547
|
+
// it, so the PMC IDs claim their records before any converter answer is
|
|
1548
|
+
// placed. Fallback candidates are bucketed by the field that owns them and
|
|
1549
|
+
// run in that order, which is how a later tier picks a record's owner too.
|
|
1550
|
+
const pmcIds = [];
|
|
1551
|
+
const pending = { pmcid: [], pmid: [], doi: [] };
|
|
1552
|
+
// Send a PMCID to PMC EFetch under the input id that named it. A second
|
|
1553
|
+
// input id that reaches the same PMC record — two DOIs for one article, or
|
|
1554
|
+
// a PMID for a PMC ID already given — joins the first one's chain as
|
|
1555
|
+
// another spelling of it: the record is fetched once, and both ids are
|
|
1556
|
+
// recovered, or reported unavailable with that chain, rather than the later
|
|
1557
|
+
// id taking the PMCID over and leaving the earlier one with an empty chain.
|
|
1420
1558
|
const routeToPmc = (inputId, pmcid) => {
|
|
1421
1559
|
const normalized = normalizePmcId(pmcid);
|
|
1422
1560
|
const prefixed = withPmcPrefix(normalized);
|
|
@@ -1426,135 +1564,126 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1426
1564
|
pmcidToInputId.set(prefixed, inputId);
|
|
1427
1565
|
return;
|
|
1428
1566
|
}
|
|
1429
|
-
if (owner
|
|
1430
|
-
|
|
1431
|
-
for (const spelling of callerIds.get(inputId) ?? [inputId])
|
|
1432
|
-
addCallerId(owner, spelling);
|
|
1433
|
-
callerIds.delete(inputId);
|
|
1434
|
-
chainByInput.delete(inputId);
|
|
1567
|
+
if (owner !== inputId)
|
|
1568
|
+
joinChain(inputId, owner);
|
|
1435
1569
|
};
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
const
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1570
|
+
const noteRecordDoi = (pmcid, doi) => {
|
|
1571
|
+
const prefixed = withPmcPrefix(normalizePmcId(pmcid));
|
|
1572
|
+
if (doi && !doiByPmcRecord.has(prefixed))
|
|
1573
|
+
doiByPmcRecord.set(prefixed, doi);
|
|
1574
|
+
};
|
|
1575
|
+
// `PMC123`, `pmc123`, `123` and `PMC0123` name one record; it runs the
|
|
1576
|
+
// chain once, keyed and reported in `PMC<digits>` form. (#170)
|
|
1577
|
+
for (const id of input.pmcids ?? []) {
|
|
1578
|
+
const normalized = normalizePmcId(id);
|
|
1579
|
+
const key = withPmcPrefix(normalized);
|
|
1580
|
+
if (!chainByInput.has(key))
|
|
1581
|
+
chainByInput.set(key, []);
|
|
1582
|
+
addCallerId(key, { id: key, idType: 'pmcid' });
|
|
1583
|
+
routeToPmc(key, normalized);
|
|
1584
|
+
}
|
|
1585
|
+
// The ID Converter parses `00000001` as PMID 1 yet reports it "not found
|
|
1586
|
+
// in PMC", and every later stage answers with NCBI's own PMID, so the chain
|
|
1587
|
+
// runs once per distinct PMID in its canonical form. (#161)
|
|
1588
|
+
for (const id of input.pmids ?? [])
|
|
1589
|
+
addCallerId(normalizePmid(id), { id, idType: 'pmid' });
|
|
1590
|
+
const pmids = [...new Set((input.pmids ?? []).map(normalizePmid))];
|
|
1591
|
+
// DOIs resolve through the converter too, so PMC-indexed DOIs reach PMC
|
|
1592
|
+
// EFetch instead of going straight to the EPMC/Unpaywall fallback (which
|
|
1593
|
+
// misses articles whose only OA copy is the PMC JATS).
|
|
1594
|
+
//
|
|
1595
|
+
// DOIs are case-insensitive, and the converter echoes one casing in
|
|
1596
|
+
// `requested-id` for DOIs that differ only in case, so every casing of a
|
|
1597
|
+
// DOI shares one chain, keyed by the first spelling submitted, and
|
|
1598
|
+
// converter records are matched to it case-insensitively. (#166)
|
|
1599
|
+
const chainKeyByDoi = new Map();
|
|
1600
|
+
for (const doi of input.dois ?? []) {
|
|
1601
|
+
const key = chainKeyByDoi.get(doi.toLowerCase()) ?? doi;
|
|
1602
|
+
chainKeyByDoi.set(doi.toLowerCase(), key);
|
|
1603
|
+
addCallerId(key, { id: doi, idType: 'doi' });
|
|
1604
|
+
}
|
|
1605
|
+
const dois = [...new Set(chainKeyByDoi.values())];
|
|
1606
|
+
for (const key of [...pmids, ...dois])
|
|
1607
|
+
chainByInput.set(key, []);
|
|
1608
|
+
// The two lookups are independent; their answers are placed in field order.
|
|
1609
|
+
const convertOptions = ctx.signal ? { signal: ctx.signal } : undefined;
|
|
1610
|
+
const [pmidRecords, doiRecords] = await Promise.all([
|
|
1611
|
+
pmids.length > 0 ? getNcbiService().idConvert(pmids, 'pmid', convertOptions) : [],
|
|
1612
|
+
dois.length > 0 ? getNcbiService().idConvert(dois, 'doi', convertOptions) : [],
|
|
1613
|
+
]);
|
|
1614
|
+
const seenPmids = new Set();
|
|
1615
|
+
for (const r of pmidRecords) {
|
|
1616
|
+
if (r.pmid === undefined)
|
|
1617
|
+
continue;
|
|
1618
|
+
const pmid = String(r.pmid);
|
|
1619
|
+
seenPmids.add(pmid);
|
|
1620
|
+
if (r.pmcid) {
|
|
1621
|
+
noteRecordDoi(String(r.pmcid), r.doi);
|
|
1622
|
+
routeToPmc(pmid, String(r.pmcid));
|
|
1464
1623
|
}
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
pmidFallbackCandidates.push({ pmid: requested });
|
|
1473
|
-
}
|
|
1624
|
+
else {
|
|
1625
|
+
chainByInput.get(pmid)?.push({
|
|
1626
|
+
tier: 'pmc',
|
|
1627
|
+
outcome: 'not-attempted',
|
|
1628
|
+
detail: 'PMID has no PMC counterpart',
|
|
1629
|
+
});
|
|
1630
|
+
pending.pmid.push({ idType: 'pmid', id: pmid, ...(r.doi && { doi: r.doi }) });
|
|
1474
1631
|
}
|
|
1475
1632
|
}
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1633
|
+
for (const requested of pmids) {
|
|
1634
|
+
if (!seenPmids.has(requested)) {
|
|
1635
|
+
chainByInput.get(requested)?.push({
|
|
1636
|
+
tier: 'pmc',
|
|
1637
|
+
outcome: 'not-attempted',
|
|
1638
|
+
detail: 'ID Converter returned no record for this PMID',
|
|
1639
|
+
});
|
|
1640
|
+
pending.pmid.push({ idType: 'pmid', id: requested });
|
|
1641
|
+
}
|
|
1482
1642
|
}
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
//
|
|
1486
|
-
//
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
for (const doi of input.dois) {
|
|
1495
|
-
const key = chainKeyByDoi.get(doi.toLowerCase()) ?? doi;
|
|
1496
|
-
chainKeyByDoi.set(doi.toLowerCase(), key);
|
|
1497
|
-
addCallerId(key, doi);
|
|
1643
|
+
const seenDois = new Set();
|
|
1644
|
+
for (const r of doiRecords) {
|
|
1645
|
+
// Match on the echoed `requested-id`, not `r.doi`: the record's own DOI
|
|
1646
|
+
// can be cased differently from anything the caller sent.
|
|
1647
|
+
const doi = chainKeyByDoi.get(String(r['requested-id']).toLowerCase());
|
|
1648
|
+
if (doi === undefined)
|
|
1649
|
+
continue;
|
|
1650
|
+
seenDois.add(doi);
|
|
1651
|
+
if (r.pmcid) {
|
|
1652
|
+
noteRecordDoi(String(r.pmcid), doi);
|
|
1653
|
+
routeToPmc(doi, String(r.pmcid));
|
|
1498
1654
|
}
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
// can be cased differently from anything the caller sent.
|
|
1507
|
-
const doi = chainKeyByDoi.get(String(r['requested-id']).toLowerCase());
|
|
1508
|
-
if (doi === undefined)
|
|
1509
|
-
continue;
|
|
1510
|
-
seen.add(doi);
|
|
1511
|
-
if (r.pmcid) {
|
|
1512
|
-
routeToPmc(doi, String(r.pmcid));
|
|
1513
|
-
}
|
|
1514
|
-
else {
|
|
1515
|
-
chainByInput.get(doi)?.push({
|
|
1516
|
-
tier: 'pmc',
|
|
1517
|
-
outcome: 'not-attempted',
|
|
1518
|
-
detail: 'DOI has no PMC counterpart',
|
|
1519
|
-
});
|
|
1520
|
-
doiCandidates.push({ doi });
|
|
1521
|
-
}
|
|
1655
|
+
else {
|
|
1656
|
+
chainByInput.get(doi)?.push({
|
|
1657
|
+
tier: 'pmc',
|
|
1658
|
+
outcome: 'not-attempted',
|
|
1659
|
+
detail: 'DOI has no PMC counterpart',
|
|
1660
|
+
});
|
|
1661
|
+
pending.doi.push({ idType: 'doi', id: doi, doi });
|
|
1522
1662
|
}
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
}
|
|
1663
|
+
}
|
|
1664
|
+
for (const requested of dois) {
|
|
1665
|
+
if (!seenDois.has(requested)) {
|
|
1666
|
+
chainByInput.get(requested)?.push({
|
|
1667
|
+
tier: 'pmc',
|
|
1668
|
+
outcome: 'not-attempted',
|
|
1669
|
+
detail: 'ID Converter returned no record for this DOI',
|
|
1670
|
+
});
|
|
1671
|
+
pending.doi.push({ idType: 'doi', id: requested, doi: requested });
|
|
1532
1672
|
}
|
|
1533
1673
|
}
|
|
1534
|
-
// Route PMC-missed prefixed PMCIDs
|
|
1535
|
-
//
|
|
1536
|
-
//
|
|
1537
|
-
//
|
|
1538
|
-
//
|
|
1674
|
+
// Route PMC-missed prefixed PMCIDs on to the remaining tiers, each as a
|
|
1675
|
+
// candidate of the field that owns its record — which decides how Europe
|
|
1676
|
+
// PMC searches for it and which id an Unpaywall article is stamped with. A
|
|
1677
|
+
// DOI the routing learned for the record rides along, sparing Unpaywall a
|
|
1678
|
+
// lookup; a DOI owner carries its own.
|
|
1539
1679
|
const routePmcMissesToFallback = (missingPrefixed) => {
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
const pmid = pmcidToInputId.get(prefixed);
|
|
1548
|
-
if (pmid)
|
|
1549
|
-
pmidFallbackCandidates.push(pmidContext.get(pmid) ?? { pmid });
|
|
1550
|
-
}
|
|
1551
|
-
}
|
|
1552
|
-
else if (input.dois) {
|
|
1553
|
-
for (const prefixed of missingPrefixed) {
|
|
1554
|
-
const doi = pmcidToInputId.get(prefixed);
|
|
1555
|
-
if (doi)
|
|
1556
|
-
doiCandidates.push({ doi });
|
|
1557
|
-
}
|
|
1680
|
+
for (const prefixed of missingPrefixed) {
|
|
1681
|
+
const owner = pmcidToInputId.get(prefixed) ?? prefixed;
|
|
1682
|
+
const idType = callerIds.get(owner)?.[0]?.idType;
|
|
1683
|
+
if (idType === undefined)
|
|
1684
|
+
continue;
|
|
1685
|
+
const doi = idType === 'doi' ? owner : doiByPmcRecord.get(prefixed);
|
|
1686
|
+
pending[idType].push({ idType, id: owner, ...(doi && { doi }) });
|
|
1558
1687
|
}
|
|
1559
1688
|
};
|
|
1560
1689
|
// ── Stage 1: PMC EFetch ─────────────────────────────────────────────────
|
|
@@ -1571,7 +1700,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1571
1700
|
});
|
|
1572
1701
|
const articleSet = findOne(xmlData, 'pmc-articleset');
|
|
1573
1702
|
if (!articleSet) {
|
|
1574
|
-
throw
|
|
1703
|
+
throw serializationError('PMC EFetch response missing pmc-articleset wrapper');
|
|
1575
1704
|
}
|
|
1576
1705
|
// A parsed article with no body sections is front matter only — PMC
|
|
1577
1706
|
// returns one whenever the publisher blocks full-text XML download. It
|
|
@@ -1586,8 +1715,8 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1586
1715
|
bodylessPmcIds.add(before.pmcId);
|
|
1587
1716
|
continue;
|
|
1588
1717
|
}
|
|
1589
|
-
const after = applyPmcFilters(before,
|
|
1590
|
-
if (isSectionFilterMiss(before, after,
|
|
1718
|
+
const after = applyPmcFilters(before, filters);
|
|
1719
|
+
if (isSectionFilterMiss(before, after, filters.sections)) {
|
|
1591
1720
|
sectionFilterMisses.push(articleDisplayId(after));
|
|
1592
1721
|
}
|
|
1593
1722
|
const budgeted = applyPmcBudget(after, budget);
|
|
@@ -1634,6 +1763,8 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1634
1763
|
routePmcMissesToFallback(missing);
|
|
1635
1764
|
}
|
|
1636
1765
|
catch (error) {
|
|
1766
|
+
if (ctx.signal.aborted)
|
|
1767
|
+
throw error;
|
|
1637
1768
|
const detail = error instanceof Error ? error.message : String(error);
|
|
1638
1769
|
ctx.log.warning('PMC EFetch failed; chain continues with next layer', {
|
|
1639
1770
|
pmcIdCount: pmcIds.length,
|
|
@@ -1647,85 +1778,72 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1647
1778
|
routePmcMissesToFallback(allPrefixed);
|
|
1648
1779
|
}
|
|
1649
1780
|
}
|
|
1781
|
+
// The records PMC EFetch served, by PMCID, under the id that owns each: the
|
|
1782
|
+
// ID Converter can miss a PMC record a Europe PMC hit links, and that hit
|
|
1783
|
+
// joins the owner rather than fetching the article again. (#192)
|
|
1784
|
+
const servedByPmc = new Map();
|
|
1785
|
+
for (const a of pmcArticles) {
|
|
1786
|
+
if (a.pmcId)
|
|
1787
|
+
servedByPmc.set(a.pmcId, inputIdByArticle.get(a) ?? a.pmcId);
|
|
1788
|
+
}
|
|
1650
1789
|
// ── Stage 2: Europe PMC fullTextXML ─────────────────────────────────────
|
|
1790
|
+
// Each candidate's `id` is its own chain key from here on: a PMC miss is
|
|
1791
|
+
// routed under the id that owns the record.
|
|
1792
|
+
let candidates = [...pending.pmcid, ...pending.pmid, ...pending.doi];
|
|
1651
1793
|
const epmc = getEuropePmcService();
|
|
1652
1794
|
const epmcOutcomes = epmc
|
|
1653
|
-
? await runEpmcStage(epmc, {
|
|
1654
|
-
pmidFallbackCandidates,
|
|
1655
|
-
pmcidFallbackCandidates,
|
|
1656
|
-
doiCandidates,
|
|
1657
|
-
input,
|
|
1658
|
-
budget,
|
|
1659
|
-
ctx,
|
|
1660
|
-
})
|
|
1795
|
+
? await runEpmcStage(epmc, { candidates, servedByPmc, input: filters, budget, ctx })
|
|
1661
1796
|
: {
|
|
1662
1797
|
articles: [],
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
pmcidOutcomes: new Map(),
|
|
1668
|
-
doiOutcomes: new Map(),
|
|
1798
|
+
articleInputIds: new Map(),
|
|
1799
|
+
joins: [],
|
|
1800
|
+
outcomes: new Map(),
|
|
1801
|
+
remaining: candidates,
|
|
1669
1802
|
sectionFilterMisses: [],
|
|
1670
1803
|
truncatedArticles: [],
|
|
1671
1804
|
omittedSections: 0,
|
|
1672
|
-
articleInputIds: new Map(),
|
|
1673
1805
|
};
|
|
1674
1806
|
pmcArticles = pmcArticles.concat(epmcOutcomes.articles);
|
|
1675
1807
|
truncatedArticles.push(...epmcOutcomes.truncatedArticles);
|
|
1676
1808
|
omittedSections += epmcOutcomes.omittedSections;
|
|
1677
1809
|
for (const [article, candidateId] of epmcOutcomes.articleInputIds) {
|
|
1678
|
-
inputIdByArticle.set(article,
|
|
1810
|
+
inputIdByArticle.set(article, candidateId);
|
|
1679
1811
|
}
|
|
1680
1812
|
// Fold EPMC outcomes into each id's chain. EPMC-served articles count as
|
|
1681
1813
|
// recovered, so their ids are added to `recoveredIds` here.
|
|
1682
1814
|
if (!epmc) {
|
|
1683
|
-
const epmcDisabledEntry = {
|
|
1684
|
-
tier: 'europepmc',
|
|
1685
|
-
outcome: 'not-attempted',
|
|
1686
|
-
detail: 'EUROPEPMC_ENABLED=false',
|
|
1687
|
-
};
|
|
1688
1815
|
// EPMC searches by PMID, PMCID, and DOI alike, so it could have served
|
|
1689
1816
|
// every candidate that reached this stage — no applicability test.
|
|
1690
|
-
const
|
|
1691
|
-
chainByInput.get(
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
const prefixed = withPmcPrefix(c.pmcid);
|
|
1698
|
-
skipEpmc(pmcidToInputId.get(prefixed) ?? prefixed);
|
|
1817
|
+
for (const c of candidates) {
|
|
1818
|
+
chainByInput.get(c.id)?.push({
|
|
1819
|
+
tier: 'europepmc',
|
|
1820
|
+
outcome: 'not-attempted',
|
|
1821
|
+
detail: 'EUROPEPMC_ENABLED=false',
|
|
1822
|
+
});
|
|
1823
|
+
markUnqueried(c.id, 'europepmc');
|
|
1699
1824
|
}
|
|
1700
|
-
for (const c of doiCandidates)
|
|
1701
|
-
skipEpmc(c.doi);
|
|
1702
1825
|
}
|
|
1703
1826
|
else {
|
|
1704
|
-
|
|
1827
|
+
// A candidate whose search reached a record an earlier one already owns
|
|
1828
|
+
// joins that chain before the owner's outcome lands on it. (#192)
|
|
1829
|
+
for (const { id, owner } of epmcOutcomes.joins)
|
|
1830
|
+
joinChain(id, owner);
|
|
1831
|
+
for (const [inputId, outcome] of epmcOutcomes.outcomes) {
|
|
1705
1832
|
if (outcome.kind === 'hit') {
|
|
1706
1833
|
recoveredIds.add(inputId);
|
|
1707
|
-
|
|
1834
|
+
continue;
|
|
1708
1835
|
}
|
|
1709
1836
|
if (outcome.kind === 'no-body')
|
|
1710
1837
|
bodylessInputIds.add(inputId);
|
|
1711
1838
|
chainByInput.get(inputId)?.push(epmcTierFromOutcome(outcome));
|
|
1712
|
-
};
|
|
1713
|
-
for (const [pmid, outcome] of epmcOutcomes.pmidOutcomes)
|
|
1714
|
-
foldEpmcOutcome(pmid, outcome);
|
|
1715
|
-
for (const [prefixed, outcome] of epmcOutcomes.pmcidOutcomes) {
|
|
1716
|
-
foldEpmcOutcome(pmcidToInputId.get(prefixed) ?? prefixed, outcome);
|
|
1717
1839
|
}
|
|
1718
|
-
for (const [doi, outcome] of epmcOutcomes.doiOutcomes)
|
|
1719
|
-
foldEpmcOutcome(doi, outcome);
|
|
1720
1840
|
}
|
|
1721
|
-
|
|
1722
|
-
pmcidFallbackCandidates = epmcOutcomes.remainingPmcid;
|
|
1723
|
-
doiCandidates = epmcOutcomes.remainingDoi;
|
|
1841
|
+
candidates = epmcOutcomes.remaining;
|
|
1724
1842
|
sectionFilterMisses.push(...epmcOutcomes.sectionFilterMisses);
|
|
1725
1843
|
// ── Stage 3: Unpaywall fallback ─────────────────────────────────────────
|
|
1726
1844
|
const unpaywall = getUnpaywallService();
|
|
1727
1845
|
const fallbackArticles = [];
|
|
1728
|
-
// Detail of a DOI-backfill lookup that threw, per
|
|
1846
|
+
// Detail of a DOI-backfill lookup that threw, per field. A candidate still
|
|
1729
1847
|
// DOI-less after one of these is an unknown, not a settled absence: it
|
|
1730
1848
|
// reports `doi-lookup-failed` rather than `no-doi`. (#119)
|
|
1731
1849
|
let pmcidDoiLookupFailure;
|
|
@@ -1736,196 +1854,129 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1736
1854
|
outcome: failure ? 'doi-lookup-failed' : 'no-doi',
|
|
1737
1855
|
...(failure && { detail: failure }),
|
|
1738
1856
|
});
|
|
1739
|
-
//
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
1742
|
-
// Converter, which returns DOIs for
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
detail: 'UNPAYWALL_EMAIL is not set',
|
|
1756
|
-
});
|
|
1757
|
-
markUnqueried(inputId, 'unpaywall');
|
|
1758
|
-
}
|
|
1759
|
-
}
|
|
1760
|
-
else {
|
|
1761
|
-
const needDoi = pmcidFallbackCandidates
|
|
1762
|
-
.filter((c) => !c.doi)
|
|
1763
|
-
.map((c) => withPmcPrefix(c.pmcid));
|
|
1764
|
-
if (needDoi.length > 0) {
|
|
1765
|
-
try {
|
|
1766
|
-
const records = await getNcbiService().idConvert(needDoi, 'pmcid', ctx.signal ? { signal: ctx.signal } : undefined);
|
|
1767
|
-
const doiByPmcid = new Map();
|
|
1768
|
-
for (const r of records) {
|
|
1769
|
-
if (r.pmcid && r.doi) {
|
|
1770
|
-
doiByPmcid.set(withPmcPrefix(normalizePmcId(String(r.pmcid))), String(r.doi));
|
|
1771
|
-
}
|
|
1772
|
-
}
|
|
1773
|
-
pmcidFallbackCandidates = pmcidFallbackCandidates.map((c) => {
|
|
1774
|
-
if (c.doi)
|
|
1775
|
-
return c;
|
|
1776
|
-
const doi = doiByPmcid.get(withPmcPrefix(c.pmcid));
|
|
1777
|
-
return doi ? { ...c, doi } : c;
|
|
1778
|
-
});
|
|
1779
|
-
}
|
|
1780
|
-
catch (error) {
|
|
1781
|
-
pmcidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1782
|
-
ctx.log.warning('Failed to resolve PMCID → DOI for the Unpaywall fallback', {
|
|
1783
|
-
error: pmcidDoiLookupFailure,
|
|
1784
|
-
pmcidCount: needDoi.length,
|
|
1785
|
-
});
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
1788
|
-
const outcomes = await Promise.all(pmcidFallbackCandidates.map(async (candidate) => {
|
|
1789
|
-
// The prefixed PMCID is the id `unavailable[]` keys on, so stamping
|
|
1790
|
-
// it on the article makes a partially-recovered batch report its
|
|
1791
|
-
// successes and its failures under the same identifier. (#92)
|
|
1792
|
-
const pmcId = withPmcPrefix(candidate.pmcid);
|
|
1793
|
-
return {
|
|
1794
|
-
pmcId,
|
|
1795
|
-
result: candidate.doi
|
|
1796
|
-
? await resolveUnpaywall({ pmcId, doi: candidate.doi, epmcTitle: candidate.title, budget }, unpaywall, ctx)
|
|
1797
|
-
: undefined,
|
|
1798
|
-
};
|
|
1799
|
-
}));
|
|
1800
|
-
for (const { pmcId, result } of outcomes) {
|
|
1801
|
-
const inputId = pmcidToInputId.get(pmcId) ?? pmcId;
|
|
1802
|
-
if (result === undefined) {
|
|
1803
|
-
chainByInput.get(inputId)?.push(doilessTierEntry(pmcidDoiLookupFailure));
|
|
1804
|
-
}
|
|
1805
|
-
else if ('article' in result) {
|
|
1806
|
-
fallbackArticles.push(result.article);
|
|
1807
|
-
inputIdByArticle.set(result.article, inputId);
|
|
1808
|
-
if (result.truncation)
|
|
1809
|
-
truncatedArticles.push(result.truncation);
|
|
1810
|
-
recoveredIds.add(inputId);
|
|
1811
|
-
}
|
|
1812
|
-
else {
|
|
1813
|
-
const u = result.unavailable;
|
|
1814
|
-
chainByInput.get(inputId)?.push({
|
|
1815
|
-
tier: 'unpaywall',
|
|
1816
|
-
outcome: unpaywallReasonToTierOutcome(u.reason),
|
|
1817
|
-
...(u.detail && { detail: u.detail }),
|
|
1818
|
-
});
|
|
1857
|
+
// Candidates reach Unpaywall on the DOI the chain already holds — a `dois`
|
|
1858
|
+
// entry's own, one the routing learned for a PMC record, or one a Europe
|
|
1859
|
+
// PMC hit carried, captured on non-hit outcomes too (#88). The rest are
|
|
1860
|
+
// backfilled: a PMC ID through the PMC ID Converter, which returns DOIs for
|
|
1861
|
+
// PMC-indexed records, and only when Unpaywall is configured; a PMID from
|
|
1862
|
+
// PubMed metadata (db=pubmed), since the converter returns DOIs only for
|
|
1863
|
+
// articles it has in PMC. The two lookups are independent.
|
|
1864
|
+
const backfillPmcidDois = async (pmcids) => {
|
|
1865
|
+
const found = new Map();
|
|
1866
|
+
if (pmcids.length === 0)
|
|
1867
|
+
return found;
|
|
1868
|
+
try {
|
|
1869
|
+
const records = await getNcbiService().idConvert(pmcids, 'pmcid', ctx.signal ? { signal: ctx.signal } : undefined);
|
|
1870
|
+
for (const r of records) {
|
|
1871
|
+
if (r.pmcid && r.doi) {
|
|
1872
|
+
found.set(withPmcPrefix(normalizePmcId(String(r.pmcid))), String(r.doi));
|
|
1819
1873
|
}
|
|
1820
1874
|
}
|
|
1821
1875
|
}
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
const doiMap = await fetchPubmedDois(needDoi, ctx.signal);
|
|
1831
|
-
pmidFallbackCandidates = pmidFallbackCandidates.map((c) => {
|
|
1832
|
-
if (c.doi)
|
|
1833
|
-
return c;
|
|
1834
|
-
const doi = doiMap.get(c.pmid);
|
|
1835
|
-
return doi ? { ...c, doi } : c;
|
|
1836
|
-
});
|
|
1837
|
-
}
|
|
1838
|
-
catch (error) {
|
|
1839
|
-
pmidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1840
|
-
ctx.log.warning('Failed to batch-fetch DOIs from PubMed for Unpaywall fallback', {
|
|
1841
|
-
error: pmidDoiLookupFailure,
|
|
1842
|
-
pmidCount: needDoi.length,
|
|
1843
|
-
});
|
|
1844
|
-
}
|
|
1876
|
+
catch (error) {
|
|
1877
|
+
if (ctx.signal.aborted)
|
|
1878
|
+
throw error;
|
|
1879
|
+
pmcidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1880
|
+
ctx.log.warning('Failed to resolve PMCID → DOI for the Unpaywall fallback', {
|
|
1881
|
+
error: pmcidDoiLookupFailure,
|
|
1882
|
+
pmcidCount: pmcids.length,
|
|
1883
|
+
});
|
|
1845
1884
|
}
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
chainByInput.get(c.pmid)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
1854
|
-
continue;
|
|
1855
|
-
}
|
|
1856
|
-
chainByInput.get(c.pmid)?.push({
|
|
1857
|
-
tier: 'unpaywall',
|
|
1858
|
-
outcome: 'not-attempted',
|
|
1859
|
-
detail: 'UNPAYWALL_EMAIL is not set',
|
|
1860
|
-
});
|
|
1861
|
-
markUnqueried(c.pmid, 'unpaywall');
|
|
1862
|
-
}
|
|
1885
|
+
return found;
|
|
1886
|
+
};
|
|
1887
|
+
const backfillPmidDois = async (pmidsNeedingDoi) => {
|
|
1888
|
+
if (pmidsNeedingDoi.length === 0)
|
|
1889
|
+
return new Map();
|
|
1890
|
+
try {
|
|
1891
|
+
return await fetchPubmedDois(pmidsNeedingDoi, ctx.signal);
|
|
1863
1892
|
}
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1893
|
+
catch (error) {
|
|
1894
|
+
if (ctx.signal.aborted)
|
|
1895
|
+
throw error;
|
|
1896
|
+
pmidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1897
|
+
ctx.log.warning('Failed to batch-fetch DOIs from PubMed for Unpaywall fallback', {
|
|
1898
|
+
error: pmidDoiLookupFailure,
|
|
1899
|
+
pmidCount: pmidsNeedingDoi.length,
|
|
1900
|
+
});
|
|
1901
|
+
return new Map();
|
|
1902
|
+
}
|
|
1903
|
+
};
|
|
1904
|
+
const needingDoi = (idType) => candidates.filter((c) => c.idType === idType && !c.doi).map((c) => c.id);
|
|
1905
|
+
const [pmcidDois, pmidDois] = await Promise.all([
|
|
1906
|
+
unpaywall ? backfillPmcidDois(needingDoi('pmcid')) : new Map(),
|
|
1907
|
+
backfillPmidDois(needingDoi('pmid')),
|
|
1908
|
+
]);
|
|
1909
|
+
const backfilled = { pmcid: pmcidDois, pmid: pmidDois, doi: new Map() };
|
|
1910
|
+
candidates = candidates.map((c) => {
|
|
1911
|
+
const doi = c.doi ?? backfilled[c.idType].get(c.id);
|
|
1912
|
+
return doi ? { ...c, doi } : c;
|
|
1913
|
+
});
|
|
1914
|
+
if (!unpaywall) {
|
|
1915
|
+
for (const c of candidates) {
|
|
1916
|
+
// `fetchPubmedDois` has already run, so a PMID with no DOI could not
|
|
1917
|
+
// have reached Unpaywall configured or not — that is `no-doi`, a real
|
|
1918
|
+
// answer, not an incomplete search, unless the lookup itself threw. A
|
|
1919
|
+
// PMC ID's DOI is looked up only for Unpaywall, so its absence means
|
|
1920
|
+
// "never looked up", and the tier stays a genuine unknown and is marked.
|
|
1921
|
+
if (!c.doi && c.idType === 'pmid') {
|
|
1922
|
+
chainByInput.get(c.id)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
1923
|
+
continue;
|
|
1890
1924
|
}
|
|
1925
|
+
chainByInput.get(c.id)?.push({
|
|
1926
|
+
tier: 'unpaywall',
|
|
1927
|
+
outcome: 'not-attempted',
|
|
1928
|
+
detail: 'UNPAYWALL_EMAIL is not set',
|
|
1929
|
+
});
|
|
1930
|
+
markUnqueried(c.id, 'unpaywall');
|
|
1891
1931
|
}
|
|
1892
1932
|
}
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
markUnqueried(c.doi, 'unpaywall');
|
|
1933
|
+
else {
|
|
1934
|
+
// One resolve per DOI, case-insensitively: a later candidate carrying a
|
|
1935
|
+
// DOI an earlier one holds joins that chain, and the article is stamped
|
|
1936
|
+
// with the owner's identifier. (#192)
|
|
1937
|
+
const ownerByDoi = new Map();
|
|
1938
|
+
for (const c of candidates) {
|
|
1939
|
+
if (!c.doi) {
|
|
1940
|
+
const failure = c.idType === 'pmcid' ? pmcidDoiLookupFailure : pmidDoiLookupFailure;
|
|
1941
|
+
chainByInput.get(c.id)?.push(doilessTierEntry(failure));
|
|
1942
|
+
continue;
|
|
1904
1943
|
}
|
|
1944
|
+
const owner = ownerByDoi.get(c.doi.toLowerCase());
|
|
1945
|
+
if (owner)
|
|
1946
|
+
joinChain(c.id, owner.id);
|
|
1947
|
+
else
|
|
1948
|
+
ownerByDoi.set(c.doi.toLowerCase(), { ...c, doi: c.doi });
|
|
1905
1949
|
}
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1950
|
+
// `resolveUnpaywall` catches its own failures so this Promise.all
|
|
1951
|
+
// doesn't reject under normal operation.
|
|
1952
|
+
const outcomes = await Promise.all([...ownerByDoi.values()].map(async (c) => ({
|
|
1953
|
+
inputId: c.id,
|
|
1954
|
+
// A PMC ID or PMID is stamped on the article so a partially-recovered
|
|
1955
|
+
// batch reports its successes and its failures under the same
|
|
1956
|
+
// identifier; Unpaywall itself only knows the DOI. (#92)
|
|
1957
|
+
result: await resolveUnpaywall({
|
|
1958
|
+
...(c.idType === 'pmcid' && { pmcId: c.id }),
|
|
1959
|
+
...(c.idType === 'pmid' && { pmid: c.id }),
|
|
1910
1960
|
doi: c.doi,
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1961
|
+
epmcTitle: c.title,
|
|
1962
|
+
budget,
|
|
1963
|
+
}, unpaywall, ctx),
|
|
1964
|
+
})));
|
|
1965
|
+
for (const { inputId, result } of outcomes) {
|
|
1966
|
+
if ('article' in result) {
|
|
1967
|
+
fallbackArticles.push(result.article);
|
|
1968
|
+
inputIdByArticle.set(result.article, inputId);
|
|
1969
|
+
if (result.truncation)
|
|
1970
|
+
truncatedArticles.push(result.truncation);
|
|
1971
|
+
recoveredIds.add(inputId);
|
|
1972
|
+
}
|
|
1973
|
+
else {
|
|
1974
|
+
const u = result.unavailable;
|
|
1975
|
+
chainByInput.get(inputId)?.push({
|
|
1976
|
+
tier: 'unpaywall',
|
|
1977
|
+
outcome: unpaywallReasonToTierOutcome(u.reason),
|
|
1978
|
+
...(u.detail && { detail: u.detail }),
|
|
1979
|
+
});
|
|
1929
1980
|
}
|
|
1930
1981
|
}
|
|
1931
1982
|
}
|
|
@@ -1935,12 +1986,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1935
1986
|
if (recoveredIds.has(id))
|
|
1936
1987
|
continue;
|
|
1937
1988
|
const unqueried = unqueriedByInput.get(id);
|
|
1938
|
-
for (const callerId of callerIds.get(id) ?? [
|
|
1989
|
+
for (const callerId of callerIds.get(id) ?? []) {
|
|
1990
|
+
const own = ownChainOf.get(callerId);
|
|
1991
|
+
const triedTiers = own ? [...own.tried, ...chain.slice(own.from)] : chain;
|
|
1939
1992
|
unavailable.push({
|
|
1940
|
-
id: callerId,
|
|
1941
|
-
idType,
|
|
1942
|
-
reason: reasonFromChain(
|
|
1943
|
-
triedTiers
|
|
1993
|
+
id: callerId.id,
|
|
1994
|
+
idType: callerId.idType,
|
|
1995
|
+
reason: reasonFromChain(triedTiers),
|
|
1996
|
+
triedTiers,
|
|
1944
1997
|
...(unqueried?.size && { unqueriedTiers: [...unqueried] }),
|
|
1945
1998
|
});
|
|
1946
1999
|
}
|
|
@@ -1955,17 +2008,32 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1955
2008
|
const fit = ceiling === undefined ? undefined : fitWholeItems(resolved, ceiling);
|
|
1956
2009
|
const articles = fit?.kept ?? resolved;
|
|
1957
2010
|
const nextDeferredCharacters = fit?.nextDeferredCharacters;
|
|
2011
|
+
// Each deferred article is handed back under the first spelling of the id
|
|
2012
|
+
// that owns it, in the field that id was sent in. Every recovery site
|
|
2013
|
+
// records the owner; a PMC record that came back without a PMC ID has none,
|
|
2014
|
+
// and is reported under the id it carries — the total-function fallback,
|
|
2015
|
+
// not an expected path.
|
|
2016
|
+
const deferredIds = (fit?.deferred ?? []).map((a) => {
|
|
2017
|
+
const key = inputIdByArticle.get(a);
|
|
2018
|
+
const owner = key === undefined ? undefined : callerIds.get(key)?.[0];
|
|
2019
|
+
return (owner ?? { id: articleDisplayId(a), idType: a.pmcId ? 'pmcid' : a.pmid ? 'pmid' : 'doi' });
|
|
2020
|
+
});
|
|
2021
|
+
const deferredFields = new Set(deferredIds.map((d) => d.idType));
|
|
2022
|
+
const [onlyField] = deferredFields;
|
|
2023
|
+
const deferredUnder = (idType) => deferredIds.filter((d) => d.idType === idType).map((d) => d.id);
|
|
1958
2024
|
const deferred = ceiling !== undefined && nextDeferredCharacters !== undefined && fit
|
|
1959
2025
|
? {
|
|
1960
2026
|
maxResponseCharacters: ceiling,
|
|
1961
2027
|
returnedCharacters: fit.keptCharacters,
|
|
1962
2028
|
deferredCount: fit.deferred.length,
|
|
1963
|
-
|
|
1964
|
-
//
|
|
1965
|
-
|
|
1966
|
-
ids:
|
|
1967
|
-
|
|
1968
|
-
|
|
2029
|
+
// One field names them all — always so for a single-field call — or
|
|
2030
|
+
// each field's list says where to resend its ids. (#192)
|
|
2031
|
+
...(deferredFields.size === 1 && onlyField && { idType: onlyField }),
|
|
2032
|
+
ids: deferredIds.map((d) => d.id),
|
|
2033
|
+
...(deferredFields.size > 1 && {
|
|
2034
|
+
...(deferredFields.has('pmcid') && { pmcids: deferredUnder('pmcid') }),
|
|
2035
|
+
...(deferredFields.has('pmid') && { pmids: deferredUnder('pmid') }),
|
|
2036
|
+
...(deferredFields.has('doi') && { dois: deferredUnder('doi') }),
|
|
1969
2037
|
}),
|
|
1970
2038
|
nextDeferredCharacters,
|
|
1971
2039
|
}
|
|
@@ -1985,7 +2053,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1985
2053
|
}
|
|
1986
2054
|
}
|
|
1987
2055
|
ctx.log.info('pubmed_fetch_fulltext completed', {
|
|
1988
|
-
requested:
|
|
2056
|
+
requested: idCount,
|
|
1989
2057
|
returned: articles.length,
|
|
1990
2058
|
pmcHits: pmcArticles.filter((a) => a.viaSource === 'pmc').length,
|
|
1991
2059
|
epmcHits: pmcArticles.filter((a) => a.viaSource === 'europepmc').length,
|
|
@@ -2019,8 +2087,8 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2019
2087
|
// Only the last ctx.enrich.notice survives, so the applicable fragments are
|
|
2020
2088
|
// collected and emitted once.
|
|
2021
2089
|
const notices = [];
|
|
2022
|
-
if (
|
|
2023
|
-
notices.push(buildSectionFilterMissNotice(sectionFilterMisses,
|
|
2090
|
+
if (sectionFilterMisses.length > 0) {
|
|
2091
|
+
notices.push(buildSectionFilterMissNotice(sectionFilterMisses, sectionFilter.terms));
|
|
2024
2092
|
}
|
|
2025
2093
|
const unrecoveredBodyless = [...bodylessInputIds].filter((id) => !recoveredIds.has(id));
|
|
2026
2094
|
if (unrecoveredBodyless.length > 0)
|
|
@@ -2075,7 +2143,16 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2075
2143
|
}
|
|
2076
2144
|
if (result.deferred) {
|
|
2077
2145
|
const d = result.deferred;
|
|
2078
|
-
lines.push(`\n**Deferred by the response budget:** ${d.deferredCount} article(s) — ${d.returnedCharacters} of ${d.maxResponseCharacters} budgeted characters returned; next deferred article ${d.nextDeferredCharacters} characters
|
|
2146
|
+
lines.push(`\n**Deferred by the response budget:** ${d.deferredCount} article(s) — ${d.returnedCharacters} of ${d.maxResponseCharacters} budgeted characters returned; next deferred article ${d.nextDeferredCharacters} characters`);
|
|
2147
|
+
// Rendered on presence, each independently: a response carries `idType`
|
|
2148
|
+
// or the per-field lists, never both.
|
|
2149
|
+
if (d.idType) {
|
|
2150
|
+
lines.push(`Re-call \`pubmed_fetch_fulltext\` with these ${d.idType} ids as \`${d.idType}s\`: ${d.ids.join(', ')}`);
|
|
2151
|
+
}
|
|
2152
|
+
const fieldLists = deferredFieldLists(d);
|
|
2153
|
+
if (fieldLists) {
|
|
2154
|
+
lines.push(`Re-call \`pubmed_fetch_fulltext\` with these ids, each under its own field — ${fieldLists}`);
|
|
2155
|
+
}
|
|
2079
2156
|
}
|
|
2080
2157
|
// An empty response under a budget is a deferral, not an absence — the
|
|
2081
2158
|
// articles resolved and the ids above retrieve them.
|
|
@@ -2098,7 +2175,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2098
2175
|
});
|
|
2099
2176
|
/**
|
|
2100
2177
|
* Merge what a Europe PMC hit carried onto a candidate bound for Unpaywall. A
|
|
2101
|
-
* DOI the candidate already holds wins — for `dois`
|
|
2178
|
+
* DOI the candidate already holds wins — for a `dois` entry it is the caller's
|
|
2102
2179
|
* own identifier.
|
|
2103
2180
|
*/
|
|
2104
2181
|
function carryEpmcHit(candidate, hit) {
|
|
@@ -2114,18 +2191,63 @@ function nonEmptyDisplayText(raw) {
|
|
|
2114
2191
|
}
|
|
2115
2192
|
/**
|
|
2116
2193
|
* Run the Europe PMC step against everything that fell through PMC EFetch
|
|
2117
|
-
* plus any
|
|
2118
|
-
*
|
|
2119
|
-
*
|
|
2194
|
+
* plus any DOI the converter could not place. Each candidate is searched by its
|
|
2195
|
+
* own id; each record the searches reach is then fetched as fullTextXML once,
|
|
2196
|
+
* for the first candidate in field order that reached it, and any later one
|
|
2197
|
+
* joins that candidate's chain. Hits become `source: 'pmc'` articles with
|
|
2198
|
+
* `viaSource: 'europepmc'`; misses flow through to the Unpaywall stage
|
|
2199
|
+
* unchanged.
|
|
2120
2200
|
*
|
|
2121
|
-
*
|
|
2122
|
-
* stays polite without serializing. Errors are caught and
|
|
2123
|
-
* helpers; a transient EPMC failure must not block the
|
|
2124
|
-
* fallback.
|
|
2201
|
+
* Searches and fetches run in parallel — the EPMC request queue caps
|
|
2202
|
+
* concurrency so this stays polite without serializing. Errors are caught and
|
|
2203
|
+
* logged inside the helpers; a transient EPMC failure must not block the
|
|
2204
|
+
* downstream Unpaywall fallback.
|
|
2125
2205
|
*/
|
|
2126
2206
|
async function runEpmcStage(epmc, args) {
|
|
2127
|
-
|
|
2128
|
-
|
|
2207
|
+
/**
|
|
2208
|
+
* Query shapes are load-bearing and not interchangeable with their quoted
|
|
2209
|
+
* variants. Europe PMC matches zero records for `EXT_ID:"<pmid>" AND SRC:MED`
|
|
2210
|
+
* and `PMCID:"PMC<digits>"` — the quotes only survive as long as no `AND SRC:`
|
|
2211
|
+
* clause follows. `SRC:PMC` is likewise wrong for a PMCID lookup: EPMC's
|
|
2212
|
+
* canonical record for a PMC-indexed article has `source: MED` and carries the
|
|
2213
|
+
* PMCID as a field, so the filter excludes the very record being sought. DOIs
|
|
2214
|
+
* keep their quotes — they carry slashes and dots that need them. (#85)
|
|
2215
|
+
*/
|
|
2216
|
+
const queryFor = (c) => {
|
|
2217
|
+
switch (c.idType) {
|
|
2218
|
+
case 'pmcid':
|
|
2219
|
+
return `PMCID:${c.id}`;
|
|
2220
|
+
case 'pmid':
|
|
2221
|
+
return `EXT_ID:${c.id} AND SRC:MED`;
|
|
2222
|
+
case 'doi':
|
|
2223
|
+
return `DOI:"${c.id}"`;
|
|
2224
|
+
}
|
|
2225
|
+
};
|
|
2226
|
+
const searched = await Promise.all(args.candidates.map(async (c) => ({
|
|
2227
|
+
c,
|
|
2228
|
+
search: await searchEpmcSafe(epmc, queryFor(c), args.ctx),
|
|
2229
|
+
})));
|
|
2230
|
+
// Every tier keys on the record: the first candidate whose search reached a
|
|
2231
|
+
// record owns it, and a later one joins its chain rather than fetching and
|
|
2232
|
+
// returning the same article again — as does one whose hit names a PMC ID
|
|
2233
|
+
// PMC EFetch already served, joining that article's owner. (#192)
|
|
2234
|
+
const joins = [];
|
|
2235
|
+
const ownerByRecord = new Map();
|
|
2236
|
+
const owned = searched.filter(({ c, search }) => {
|
|
2237
|
+
if (search.kind !== 'hit')
|
|
2238
|
+
return true;
|
|
2239
|
+
const record = `${search.hit.source}:${search.hit.id}`;
|
|
2240
|
+
const pmcid = search.hit.pmcid ?? (search.hit.source === 'PMC' ? search.hit.id : undefined);
|
|
2241
|
+
const servedOwner = pmcid === undefined ? undefined : args.servedByPmc.get(withPmcPrefix(normalizePmcId(pmcid)));
|
|
2242
|
+
const owner = servedOwner ?? ownerByRecord.get(record);
|
|
2243
|
+
if (owner === undefined) {
|
|
2244
|
+
ownerByRecord.set(record, c.id);
|
|
2245
|
+
return true;
|
|
2246
|
+
}
|
|
2247
|
+
joins.push({ id: c.id, owner });
|
|
2248
|
+
return false;
|
|
2249
|
+
});
|
|
2250
|
+
const runOne = async (c, search) => {
|
|
2129
2251
|
if (search.kind === 'error') {
|
|
2130
2252
|
return { c, outcome: { kind: 'service-error', detail: search.detail } };
|
|
2131
2253
|
}
|
|
@@ -2136,6 +2258,7 @@ async function runEpmcStage(epmc, args) {
|
|
|
2136
2258
|
...(search.hit.doi && { doi: search.hit.doi }),
|
|
2137
2259
|
...(title && { title }),
|
|
2138
2260
|
};
|
|
2261
|
+
const contextPmid = c.idType === 'pmid' ? c.id : undefined;
|
|
2139
2262
|
const fetched = await fetchEpmcArticle(epmc, search.hit, args, contextPmid);
|
|
2140
2263
|
if (fetched.kind === 'error') {
|
|
2141
2264
|
return { c, ...hit, outcome: { kind: 'service-error', detail: fetched.detail } };
|
|
@@ -2160,34 +2283,11 @@ async function runEpmcStage(epmc, args) {
|
|
|
2160
2283
|
...(fetched.truncation && { truncation: fetched.truncation }),
|
|
2161
2284
|
};
|
|
2162
2285
|
};
|
|
2163
|
-
|
|
2164
|
-
* Query shapes are load-bearing and not interchangeable with their quoted
|
|
2165
|
-
* variants. Europe PMC matches zero records for `EXT_ID:"<pmid>" AND SRC:MED`
|
|
2166
|
-
* and `PMCID:"PMC<digits>"` — the quotes only survive as long as no `AND SRC:`
|
|
2167
|
-
* clause follows. `SRC:PMC` is likewise wrong for a PMCID lookup: EPMC's
|
|
2168
|
-
* canonical record for a PMC-indexed article has `source: MED` and carries the
|
|
2169
|
-
* PMCID as a field, so the filter excludes the very record being sought. DOIs
|
|
2170
|
-
* keep their quotes — they carry slashes and dots that need them. (#85)
|
|
2171
|
-
*/
|
|
2172
|
-
const fetchForPmid = (c) => runOne(c, `EXT_ID:${c.pmid} AND SRC:MED`, c.pmid);
|
|
2173
|
-
const fetchForPmcid = (c) => {
|
|
2174
|
-
const normalized = withPmcPrefix(c.pmcid);
|
|
2175
|
-
return runOne({ c, normalized }, `PMCID:${normalized}`, undefined);
|
|
2176
|
-
};
|
|
2177
|
-
const fetchForDoi = (c) => runOne(c, `DOI:"${c.doi}"`, undefined);
|
|
2178
|
-
const [pmidResults, pmcidResults, doiResults] = await Promise.all([
|
|
2179
|
-
Promise.all(args.pmidFallbackCandidates.map(fetchForPmid)),
|
|
2180
|
-
Promise.all(args.pmcidFallbackCandidates.map(fetchForPmcid)),
|
|
2181
|
-
Promise.all(args.doiCandidates.map(fetchForDoi)),
|
|
2182
|
-
]);
|
|
2286
|
+
const runs = await Promise.all(owned.map(({ c, search }) => runOne(c, search)));
|
|
2183
2287
|
const articles = [];
|
|
2184
2288
|
const articleInputIds = new Map();
|
|
2185
|
-
const
|
|
2186
|
-
const
|
|
2187
|
-
const remainingDoi = [];
|
|
2188
|
-
const pmidOutcomes = new Map();
|
|
2189
|
-
const pmcidOutcomes = new Map();
|
|
2190
|
-
const doiOutcomes = new Map();
|
|
2289
|
+
const remaining = [];
|
|
2290
|
+
const outcomes = new Map();
|
|
2191
2291
|
const sectionFilterMisses = [];
|
|
2192
2292
|
const truncatedArticles = [];
|
|
2193
2293
|
let omittedSections = 0;
|
|
@@ -2200,39 +2300,22 @@ async function runEpmcStage(epmc, args) {
|
|
|
2200
2300
|
truncatedArticles.push(run.truncation);
|
|
2201
2301
|
omittedSections += run.omittedSections ?? 0;
|
|
2202
2302
|
};
|
|
2203
|
-
for (const run of
|
|
2204
|
-
|
|
2303
|
+
for (const run of runs) {
|
|
2304
|
+
outcomes.set(run.c.id, run.outcome);
|
|
2205
2305
|
if (run.article)
|
|
2206
|
-
collectHit(run.c.
|
|
2306
|
+
collectHit(run.c.id, { ...run, article: run.article });
|
|
2207
2307
|
// What the EPMC hit carried is evidence the next stage needs, whatever the
|
|
2208
|
-
// fetch outcome was: its DOI spares Unpaywall a
|
|
2209
|
-
//
|
|
2308
|
+
// fetch outcome was: its DOI spares Unpaywall a DOI lookup (#88, #119), and
|
|
2309
|
+
// its title backs up Unpaywall's own record (#144).
|
|
2210
2310
|
else
|
|
2211
|
-
|
|
2212
|
-
}
|
|
2213
|
-
for (const run of pmcidResults) {
|
|
2214
|
-
pmcidOutcomes.set(run.c.normalized, run.outcome);
|
|
2215
|
-
if (run.article)
|
|
2216
|
-
collectHit(run.c.normalized, { ...run, article: run.article });
|
|
2217
|
-
else
|
|
2218
|
-
remainingPmcid.push(carryEpmcHit(run.c.c, run));
|
|
2219
|
-
}
|
|
2220
|
-
for (const run of doiResults) {
|
|
2221
|
-
doiOutcomes.set(run.c.doi, run.outcome);
|
|
2222
|
-
if (run.article)
|
|
2223
|
-
collectHit(run.c.doi, { ...run, article: run.article });
|
|
2224
|
-
else
|
|
2225
|
-
remainingDoi.push(carryEpmcHit(run.c, run));
|
|
2311
|
+
remaining.push(carryEpmcHit(run.c, run));
|
|
2226
2312
|
}
|
|
2227
2313
|
return {
|
|
2228
2314
|
articles,
|
|
2229
2315
|
articleInputIds,
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
pmidOutcomes,
|
|
2234
|
-
pmcidOutcomes,
|
|
2235
|
-
doiOutcomes,
|
|
2316
|
+
joins,
|
|
2317
|
+
outcomes,
|
|
2318
|
+
remaining,
|
|
2236
2319
|
sectionFilterMisses,
|
|
2237
2320
|
truncatedArticles,
|
|
2238
2321
|
omittedSections,
|
|
@@ -2255,6 +2338,8 @@ async function searchEpmcSafe(epmc, query, ctx) {
|
|
|
2255
2338
|
return result.hits[0] ? { kind: 'hit', hit: result.hits[0] } : { kind: 'miss' };
|
|
2256
2339
|
}
|
|
2257
2340
|
catch (error) {
|
|
2341
|
+
if (ctx.signal.aborted)
|
|
2342
|
+
throw error;
|
|
2258
2343
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2259
2344
|
ctx.log.warning('Europe PMC search failed; chain continues with next layer', {
|
|
2260
2345
|
query,
|
|
@@ -2332,6 +2417,8 @@ async function fetchEpmcArticle(epmc, hit, args, contextPmid) {
|
|
|
2332
2417
|
};
|
|
2333
2418
|
}
|
|
2334
2419
|
catch (error) {
|
|
2420
|
+
if (args.ctx.signal.aborted)
|
|
2421
|
+
throw error;
|
|
2335
2422
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2336
2423
|
args.ctx.log.warning('Europe PMC fullTextXML failed; chain continues with next layer', {
|
|
2337
2424
|
epmcId: hit.id,
|
|
@@ -2370,8 +2457,8 @@ async function fetchPubmedDois(pmids, signal) {
|
|
|
2370
2457
|
}
|
|
2371
2458
|
/**
|
|
2372
2459
|
* Resolve a DOI to an open-access article via Unpaywall. `pmcId` and `pmid`,
|
|
2373
|
-
* when set, are stamped onto the resulting article so the
|
|
2374
|
-
*
|
|
2460
|
+
* when set, are stamped onto the resulting article so the id that requested it
|
|
2461
|
+
* carries its identifier through — Unpaywall itself only knows the DOI.
|
|
2375
2462
|
*
|
|
2376
2463
|
* The article's title is the first one found in Unpaywall's own record, then
|
|
2377
2464
|
* `epmcTitle` — the Europe PMC record the chain searched for this id — then,
|
|
@@ -2402,6 +2489,8 @@ async function resolveUnpaywall(args, service, ctx) {
|
|
|
2402
2489
|
resolution = await service.resolve(doi, ctx.signal);
|
|
2403
2490
|
}
|
|
2404
2491
|
catch (error) {
|
|
2492
|
+
if (ctx.signal.aborted)
|
|
2493
|
+
throw error;
|
|
2405
2494
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2406
2495
|
ctx.log.warning('Unpaywall DOI resolve failed', { doi, error: detail });
|
|
2407
2496
|
return { unavailable: { reason: 'service-error', detail } };
|
|
@@ -2414,6 +2503,8 @@ async function resolveUnpaywall(args, service, ctx) {
|
|
|
2414
2503
|
content = await service.fetchContent(resolution.location, ctx.signal);
|
|
2415
2504
|
}
|
|
2416
2505
|
catch (error) {
|
|
2506
|
+
if (ctx.signal.aborted)
|
|
2507
|
+
throw error;
|
|
2417
2508
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2418
2509
|
ctx.log.warning('Unpaywall content fetch failed', { doi, error: detail });
|
|
2419
2510
|
return { unavailable: { reason: 'fetch-failed', detail } };
|
|
@@ -2466,6 +2557,8 @@ async function resolveUnpaywall(args, service, ctx) {
|
|
|
2466
2557
|
}), text);
|
|
2467
2558
|
}
|
|
2468
2559
|
catch (error) {
|
|
2560
|
+
if (ctx.signal.aborted)
|
|
2561
|
+
throw error;
|
|
2469
2562
|
const detail = error instanceof Error ? error.message : String(error);
|
|
2470
2563
|
ctx.log.warning('Unpaywall content extraction failed', { pmid, doi, detail });
|
|
2471
2564
|
return { unavailable: { reason: 'parse-failed', detail } };
|
|
@@ -2692,8 +2785,8 @@ function formatPmcArticle(a, lines, truncation, mode) {
|
|
|
2692
2785
|
}
|
|
2693
2786
|
if (a.affiliations?.length) {
|
|
2694
2787
|
lines.push(`\n**Affiliations:**`);
|
|
2695
|
-
for (const
|
|
2696
|
-
lines.push(
|
|
2788
|
+
for (const aff of a.affiliations)
|
|
2789
|
+
lines.push(`- ${aff}`);
|
|
2697
2790
|
}
|
|
2698
2791
|
if (a.journal) {
|
|
2699
2792
|
const parts = [];
|
|
@@ -2764,6 +2857,9 @@ function formatPmcArticle(a, lines, truncation, mode) {
|
|
|
2764
2857
|
* `rowspan`, so a well-formed table arrives rectangular and every value renders
|
|
2765
2858
|
* under the header it belongs to; a row still short of the widest is padded on
|
|
2766
2859
|
* the right with empty cells only, never a neighbour's value.
|
|
2860
|
+
*
|
|
2861
|
+
* A grid row and the Footnotes line are single lines, so the `\n` a cell or the
|
|
2862
|
+
* footer keeps between its lines renders as {@link inlineLines} says. (#185)
|
|
2767
2863
|
*/
|
|
2768
2864
|
function formatTables(tables, lines) {
|
|
2769
2865
|
lines.push(`\n#### Tables (${tables.length})`);
|
|
@@ -2782,10 +2878,23 @@ function formatTables(tables, lines) {
|
|
|
2782
2878
|
}
|
|
2783
2879
|
if (table.rows.length > 0)
|
|
2784
2880
|
lines.push(...renderTableGrid(table.rows, table.headerRowCount));
|
|
2785
|
-
if (table.footnotes)
|
|
2786
|
-
lines.push(`\nFootnotes: ${escapeMarkdownInline(table.footnotes)}`);
|
|
2881
|
+
if (table.footnotes) {
|
|
2882
|
+
lines.push(`\nFootnotes: ${escapeMarkdownInline(inlineLines(table.footnotes))}`);
|
|
2883
|
+
}
|
|
2787
2884
|
}
|
|
2788
2885
|
}
|
|
2886
|
+
/**
|
|
2887
|
+
* Lay a multi-line cell or footer value onto one Markdown line, its lines joined
|
|
2888
|
+
* by ` · ` — the joiner the header fold and the meta lines already use. A bare
|
|
2889
|
+
* space would run one footnote into the next, and a line per footnote would put
|
|
2890
|
+
* markers such as `*`, `#` and `1.` at the start of a line, where they become
|
|
2891
|
+
* Markdown structure. Callers escape the joined result, so emphasis pairing is
|
|
2892
|
+
* judged across the whole rendered line rather than one source line at a time.
|
|
2893
|
+
* (#185)
|
|
2894
|
+
*/
|
|
2895
|
+
function inlineLines(text) {
|
|
2896
|
+
return text.replaceAll('\n', ' · ');
|
|
2897
|
+
}
|
|
2789
2898
|
/**
|
|
2790
2899
|
* Render every figure and supplementary item, so a `content[]` reader gets the
|
|
2791
2900
|
* same caption, pointer and placement `structuredContent` carries rather than a
|
|
@@ -2851,7 +2960,7 @@ function foldHeaderColumn(headerRows, column) {
|
|
|
2851
2960
|
/** One table's rows as Markdown grid lines, preceded by a blank line. */
|
|
2852
2961
|
function renderTableGrid(rows, headerRowCount) {
|
|
2853
2962
|
const columns = rows.reduce((widest, row) => Math.max(widest, row.length), 0);
|
|
2854
|
-
const renderRow = (cells) => `| ${Array.from({ length: columns }, (_, i) => escapeMarkdownTableCell(cells[i] ?? '')).join(' | ')} |`;
|
|
2963
|
+
const renderRow = (cells) => `| ${Array.from({ length: columns }, (_, i) => escapeMarkdownTableCell(inlineLines(cells[i] ?? ''))).join(' | ')} |`;
|
|
2855
2964
|
const headerRows = rows.slice(0, headerRowCount);
|
|
2856
2965
|
const header = Array.from({ length: columns }, (_, i) => foldHeaderColumn(headerRows, i));
|
|
2857
2966
|
return [
|