@cyanheads/pubmed-mcp-server 2.10.19 → 2.10.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +8 -5
- package/CLAUDE.md +8 -5
- package/Dockerfile +12 -3
- package/README.md +12 -11
- package/changelog/2.10.x/2.10.20.md +63 -0
- package/changelog/2.10.x/2.10.21.md +42 -0
- package/dist/mcp-server/tools/definitions/_text.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/_text.js +10 -5
- package/dist/mcp-server/tools/definitions/_text.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.d.ts +8 -0
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.js +64 -14
- package/dist/mcp-server/tools/definitions/fetch-articles.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts +44 -9
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js +815 -563
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts +5 -5
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.js +12 -8
- package/dist/mcp-server/tools/definitions/find-related.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/format-citations.tool.d.ts +12 -0
- package/dist/mcp-server/tools/definitions/format-citations.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/format-citations.tool.js +53 -7
- package/dist/mcp-server/tools/definitions/format-citations.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.d.ts +11 -7
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.js +72 -34
- package/dist/mcp-server/tools/definitions/lookup-citation.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts +2 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js +11 -3
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.d.ts +18 -0
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.js +96 -13
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-fetch.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.d.ts +4 -3
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.js +7 -5
- package/dist/mcp-server/tools/definitions/pubmed-europepmc-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts +6 -4
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.js +33 -11
- package/dist/mcp-server/tools/definitions/search-articles.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts +2 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.js +5 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.js.map +1 -1
- package/dist/services/error-contracts.d.ts +1 -1
- package/dist/services/error-contracts.d.ts.map +1 -1
- package/dist/services/error-contracts.js +1 -1
- package/dist/services/error-contracts.js.map +1 -1
- package/dist/services/europe-pmc/api-client.d.ts.map +1 -1
- package/dist/services/europe-pmc/api-client.js +4 -3
- package/dist/services/europe-pmc/api-client.js.map +1 -1
- package/dist/services/fetch-redaction.d.ts +23 -0
- package/dist/services/fetch-redaction.d.ts.map +1 -0
- package/dist/services/fetch-redaction.js +62 -0
- package/dist/services/fetch-redaction.js.map +1 -0
- package/dist/services/ncbi/api-client.d.ts +3 -0
- package/dist/services/ncbi/api-client.d.ts.map +1 -1
- package/dist/services/ncbi/api-client.js +21 -8
- package/dist/services/ncbi/api-client.js.map +1 -1
- package/dist/services/ncbi/formatting/citation-formatter.d.ts +42 -2
- package/dist/services/ncbi/formatting/citation-formatter.d.ts.map +1 -1
- package/dist/services/ncbi/formatting/citation-formatter.js +370 -91
- package/dist/services/ncbi/formatting/citation-formatter.js.map +1 -1
- package/dist/services/ncbi/ncbi-service.d.ts +16 -1
- package/dist/services/ncbi/ncbi-service.d.ts.map +1 -1
- package/dist/services/ncbi/ncbi-service.js +117 -71
- package/dist/services/ncbi/ncbi-service.js.map +1 -1
- package/dist/services/ncbi/parsing/article-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/article-parser.js +116 -61
- package/dist/services/ncbi/parsing/article-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/esummary-parser.d.ts +9 -0
- package/dist/services/ncbi/parsing/esummary-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/esummary-parser.js +26 -2
- package/dist/services/ncbi/parsing/esummary-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/pmc-article-parser.d.ts +8 -7
- package/dist/services/ncbi/parsing/pmc-article-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-article-parser.js +633 -185
- package/dist/services/ncbi/parsing/pmc-article-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts +102 -5
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js +389 -28
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js.map +1 -1
- package/dist/services/ncbi/parsing/text-helpers.d.ts +8 -6
- package/dist/services/ncbi/parsing/text-helpers.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/text-helpers.js +95 -19
- package/dist/services/ncbi/parsing/text-helpers.js.map +1 -1
- package/dist/services/ncbi/response-handler.d.ts +20 -12
- package/dist/services/ncbi/response-handler.d.ts.map +1 -1
- package/dist/services/ncbi/response-handler.js +115 -19
- package/dist/services/ncbi/response-handler.js.map +1 -1
- package/dist/services/ncbi/types.d.ts +44 -4
- package/dist/services/ncbi/types.d.ts.map +1 -1
- package/dist/services/openalex/api-client.d.ts.map +1 -1
- package/dist/services/openalex/api-client.js +4 -3
- package/dist/services/openalex/api-client.js.map +1 -1
- package/dist/services/openalex/openalex-service.d.ts +10 -6
- package/dist/services/openalex/openalex-service.d.ts.map +1 -1
- package/dist/services/openalex/openalex-service.js +13 -9
- package/dist/services/openalex/openalex-service.js.map +1 -1
- package/dist/services/unpaywall/unpaywall-service.d.ts +3 -0
- package/dist/services/unpaywall/unpaywall-service.d.ts.map +1 -1
- package/dist/services/unpaywall/unpaywall-service.js +18 -8
- package/dist/services/unpaywall/unpaywall-service.js.map +1 -1
- package/package.json +14 -12
- package/server.json +10 -24
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview Full-text fetch tool. Resolves full-text articles through a
|
|
3
3
|
* three-stage chain: NCBI PMC EFetch → Europe PMC `fullTextXML` → Unpaywall.
|
|
4
|
-
* Accepts three
|
|
4
|
+
* Accepts three input fields, alone or together, naming at most 50 distinct
|
|
5
|
+
* identifiers between them. The first 10 in routing order are fetched; the rest
|
|
6
|
+
* are returned unfetched in `overLimit`, before any upstream request:
|
|
5
7
|
*
|
|
6
8
|
* - `pmcids` — fetch directly by PMC ID, once per record however it is
|
|
7
9
|
* spelled (`PMC123`, `pmc123`, `123`, zero-padded `PMC0123`), and reported
|
|
@@ -16,6 +18,17 @@
|
|
|
16
18
|
* DOIs are case-insensitive: every casing of one DOI runs the chain once,
|
|
17
19
|
* and `unavailable[]` reports each casing as the caller wrote it.
|
|
18
20
|
*
|
|
21
|
+
* The singular `pmcid`, `pmcId`, `pmid`, and `doi` are unadvertised aliases of
|
|
22
|
+
* these fields, and a lone string under any of them reads as a one-element
|
|
23
|
+
* array. (#221)
|
|
24
|
+
*
|
|
25
|
+
* Routing runs `pmcids`, then `pmids`, then `dois`. Every tier keys its work on
|
|
26
|
+
* the record — PMC on the PMCID, Europe PMC on the search hit's `source` + `id`,
|
|
27
|
+
* Unpaywall on the lowercased DOI — and the first id to reach a record owns it:
|
|
28
|
+
* a later id naming the same article joins the owner's chain as another caller
|
|
29
|
+
* id, so the article is fetched and returned once. Each `unavailable[]` entry,
|
|
30
|
+
* deferred id, and over-limit id carries the field its id was sent in.
|
|
31
|
+
*
|
|
19
32
|
* Output uses a discriminated union on `source` (`pmc` | `unpaywall`) with an
|
|
20
33
|
* extra `viaSource` discriminator that records which layer produced the
|
|
21
34
|
* content. EPMC's JATS reuses the `pmc` schema shape because it's the same
|
|
@@ -30,7 +43,7 @@
|
|
|
30
43
|
* @module src/mcp-server/tools/definitions/fetch-fulltext.tool
|
|
31
44
|
*/
|
|
32
45
|
import { tool, z } from '@cyanheads/mcp-ts-core';
|
|
33
|
-
import { serializationError } from '@cyanheads/mcp-ts-core/errors';
|
|
46
|
+
import { JsonRpcErrorCode, serializationError } from '@cyanheads/mcp-ts-core/errors';
|
|
34
47
|
import { htmlExtractor, pdfParser } from '@cyanheads/mcp-ts-core/utils';
|
|
35
48
|
import { getServerConfig } from '../../../config/server-config.js';
|
|
36
49
|
import { NCBI_SERVICE_ERRORS } from '../../../services/error-contracts.js';
|
|
@@ -46,6 +59,7 @@ import { fitWholeItems } from './_budget.js';
|
|
|
46
59
|
import { conceptMeta, EDAM_DATA_RETRIEVAL, SCHEMA_SCHOLARLY_ARTICLE } from './_concepts.js';
|
|
47
60
|
import { doiStringSchema, normalizePmid, pmcidStringSchema, pmidStringSchema } from './_schemas.js';
|
|
48
61
|
import { escapeMarkdownInline, escapeMarkdownTableCell, sliceAtWordBoundary } from './_text.js';
|
|
62
|
+
import { hasVisibleText } from './_visible-text.js';
|
|
49
63
|
/**
|
|
50
64
|
* Canonical digits of a PMC ID {@link pmcidStringSchema} accepted: the `PMC`
|
|
51
65
|
* prefix dropped in any case, and leading zeros stripped the way
|
|
@@ -68,6 +82,31 @@ function matchesSectionFilter(title, lowerFilter) {
|
|
|
68
82
|
function lowerCase(s) {
|
|
69
83
|
return s.toLowerCase();
|
|
70
84
|
}
|
|
85
|
+
/**
|
|
86
|
+
* Split a `sections` filter into the terms it filters on and the paths of its
|
|
87
|
+
* blank elements. An exactly-empty element is skipped — form clients send one
|
|
88
|
+
* for a field left untouched — so `[""]` filters on nothing and `["Methods", ""]`
|
|
89
|
+
* on Methods alone. An element with no visible character (`hasVisibleText`)
|
|
90
|
+
* names no heading — as a substring, a single space matches every multi-word
|
|
91
|
+
* title and a zero-width space usually none — so it is reported by its index
|
|
92
|
+
* (`sections.1`) instead. A kept term is trimmed of surrounding whitespace, so
|
|
93
|
+
* `[" Methods"]` matches what `["Methods"]` does — as a substring, a padded
|
|
94
|
+
* term misses every heading the padding does not happen to fit. Whitespace
|
|
95
|
+
* inside a term and every other character are matched literally. (#186)
|
|
96
|
+
*/
|
|
97
|
+
function splitSectionFilter(sections = []) {
|
|
98
|
+
const blank = [];
|
|
99
|
+
const terms = [];
|
|
100
|
+
for (const [index, term] of sections.entries()) {
|
|
101
|
+
if (term === '')
|
|
102
|
+
continue;
|
|
103
|
+
if (hasVisibleText(term))
|
|
104
|
+
terms.push(term.trim());
|
|
105
|
+
else
|
|
106
|
+
blank.push(`sections.${index}`);
|
|
107
|
+
}
|
|
108
|
+
return { blank, terms };
|
|
109
|
+
}
|
|
71
110
|
/**
|
|
72
111
|
* Prune a section tree to the branches a `sections` filter selects, matching
|
|
73
112
|
* titles at every nesting depth rather than the top level alone. (#126)
|
|
@@ -252,6 +291,28 @@ function applyAssetFilters(article, filters) {
|
|
|
252
291
|
const surviving = matchedSectionTitles(article.sections, filters.sections.map(lowerCase));
|
|
253
292
|
return withAssets(article, assets.filter((a) => a.sectionTitle !== undefined && surviving.has(a.sectionTitle)));
|
|
254
293
|
}
|
|
294
|
+
/**
|
|
295
|
+
* The top-level sections the `sections` filter and the `maxSections` slice
|
|
296
|
+
* select, in order. Both act on each top-level section independently or cut a
|
|
297
|
+
* prefix, so applied to the body alone this returns exactly the body prefix of
|
|
298
|
+
* what it returns for body plus back matter — which is how {@link
|
|
299
|
+
* bodySectionCount} finds where the back matter starts.
|
|
300
|
+
*/
|
|
301
|
+
function selectSections(sections, filters) {
|
|
302
|
+
const pruned = filters.sections?.length
|
|
303
|
+
? pruneSections(sections, filters.sections.map(lowerCase))
|
|
304
|
+
: sections;
|
|
305
|
+
return filters.maxSections === undefined ? pruned : pruned.slice(0, filters.maxSections);
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* How many leading entries of the filtered `sections` are body sections, the
|
|
309
|
+
* rest being back matter: the filter applied to the parsed body alone. The depth
|
|
310
|
+
* clamp that follows the filter maps sections one to one, so the count holds for
|
|
311
|
+
* the filtered article. (#206)
|
|
312
|
+
*/
|
|
313
|
+
function bodySectionCount(parsed, filters) {
|
|
314
|
+
return selectSections(parsed.sections, filters).length;
|
|
315
|
+
}
|
|
255
316
|
/**
|
|
256
317
|
* Apply the requested section/reference/table/asset filters, then clamp the
|
|
257
318
|
* section tree to the depth the output schema carries. All of it runs here so every path
|
|
@@ -269,13 +330,7 @@ function applyAssetFilters(article, filters) {
|
|
|
269
330
|
* folded-away section survives in the text it was folded into. (#111, #130)
|
|
270
331
|
*/
|
|
271
332
|
function applyPmcFilters(article, filters) {
|
|
272
|
-
let out = article;
|
|
273
|
-
if (filters.sections?.length) {
|
|
274
|
-
out = { ...out, sections: pruneSections(out.sections, filters.sections.map(lowerCase)) };
|
|
275
|
-
}
|
|
276
|
-
if (filters.maxSections !== undefined) {
|
|
277
|
-
out = { ...out, sections: out.sections.slice(0, filters.maxSections) };
|
|
278
|
-
}
|
|
333
|
+
let out = { ...article, sections: selectSections(article.sections, filters) };
|
|
279
334
|
if (!filters.includeReferences) {
|
|
280
335
|
const { references: _, ...rest } = out;
|
|
281
336
|
out = rest;
|
|
@@ -285,11 +340,12 @@ function applyPmcFilters(article, filters) {
|
|
|
285
340
|
return { ...out, sections: clampSectionDepth(out.sections) };
|
|
286
341
|
}
|
|
287
342
|
/**
|
|
288
|
-
* True when a `sections` filter removed every
|
|
289
|
-
* actually had sections upstream — the signal that the
|
|
290
|
-
* matched nothing, as opposed to the article genuinely
|
|
291
|
-
* the `sections` filter can zero a non-empty list:
|
|
292
|
-
* `.min(1)` floor, so it never reduces to zero. (#80
|
|
343
|
+
* True when a `sections` filter removed every section, back matter included,
|
|
344
|
+
* from an article that actually had sections upstream — the signal that the
|
|
345
|
+
* requested headings matched nothing, as opposed to the article genuinely
|
|
346
|
+
* shipping no body. Only the `sections` filter can zero a non-empty list:
|
|
347
|
+
* `maxSections` carries a `.min(1)` floor, so it never reduces to zero. (#80,
|
|
348
|
+
* #206)
|
|
293
349
|
*/
|
|
294
350
|
function isSectionFilterMiss(before, after, sectionFilter) {
|
|
295
351
|
return (Boolean(sectionFilter?.length) && before.sections.length > 0 && after.sections.length === 0);
|
|
@@ -301,10 +357,21 @@ function isSectionFilterMiss(before, after, sectionFilter) {
|
|
|
301
357
|
* while carrying nothing to read. Distinct from {@link isSectionFilterMiss},
|
|
302
358
|
* which needs a non-empty pre-filter body: the two never overlap. Evaluated
|
|
303
359
|
* against the *pre-filter* article so a `sections` filter can't be mistaken for
|
|
304
|
-
* an upstream absence
|
|
360
|
+
* an upstream absence, and before {@link withBackMatter}, so acknowledgments and
|
|
361
|
+
* a data-availability note alone are no body and the chain still moves on to the
|
|
362
|
+
* next tier. (#86, #206)
|
|
363
|
+
*/
|
|
364
|
+
function isBodylessArticle(parsed) {
|
|
365
|
+
return parsed.sections.length === 0;
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* The parsed article with its back-matter sections appended after the body
|
|
369
|
+
* ones, where every filter, slice, and budget downstream treats them as ordinary
|
|
370
|
+
* sections, and with no `backSections` field of its own left to reach the
|
|
371
|
+
* output. (#206)
|
|
305
372
|
*/
|
|
306
|
-
function
|
|
307
|
-
return
|
|
373
|
+
function withBackMatter({ backSections, ...article }) {
|
|
374
|
+
return backSections ? { ...article, sections: [...article.sections, ...backSections] } : article;
|
|
308
375
|
}
|
|
309
376
|
/** Pick the best human-readable identifier for an article, for recovery notices
|
|
310
377
|
* and character-budget accounting. Treats empty strings as absent — EPMC-only
|
|
@@ -325,7 +392,7 @@ function articleDisplayId(a) {
|
|
|
325
392
|
function buildSectionFilterMissNotice(affectedIds, sectionFilter) {
|
|
326
393
|
const terms = sectionFilter.join(', ');
|
|
327
394
|
const subject = affectedIds.length === 1 ? `article ${affectedIds[0]}` : `articles ${affectedIds.join(', ')}`;
|
|
328
|
-
return `No section or subsection title, at any nesting depth, matched the requested section filter (${terms}) for ${subject}. The full text was retrieved but every
|
|
395
|
+
return `No section or subsection title, at any nesting depth, matched the requested section filter (${terms}) for ${subject}. The full text was retrieved but every section, back matter included, was filtered out. Retry without \`sections\`, or filter on broader headings such as Introduction, Methods, Results, or Discussion.`;
|
|
329
396
|
}
|
|
330
397
|
/**
|
|
331
398
|
* Compose the recovery notice for identifiers whose only retrievable record was
|
|
@@ -376,7 +443,7 @@ const SectionSchema = z
|
|
|
376
443
|
text: z.string().describe('Section body text'),
|
|
377
444
|
subsections: z.array(SubsectionSchema).optional().describe('Nested subsections'),
|
|
378
445
|
})
|
|
379
|
-
.describe('Article body
|
|
446
|
+
.describe('Article section, from the body or from the back matter after it');
|
|
380
447
|
const AuthorSchema = z
|
|
381
448
|
.object({
|
|
382
449
|
collectiveName: z.string().optional().describe('Group name'),
|
|
@@ -432,14 +499,17 @@ const TableSchema = z
|
|
|
432
499
|
sectionTitle: z
|
|
433
500
|
.string()
|
|
434
501
|
.optional()
|
|
435
|
-
.describe('Title of the innermost section enclosing the table
|
|
502
|
+
.describe('Title of the innermost titled section enclosing the table — a body section or a back-matter one such as an appendix — as that section is titled in `sections`, or, for a heading nested deeper than `sections` carries, as its heading line inside the text of the section that contains it. A `sections` filter keeps the table with that section. In back matter the section name is the only positional cue there is. Absent for a table inside no titled section, such as a `<floats-group>` deposit.'),
|
|
436
503
|
headerRowCount: z
|
|
437
504
|
.number()
|
|
438
505
|
.describe('How many leading `rows` entries are header rows — a `<thead>` block, or leading rows made entirely of `<th>`. 0 when the table declares none. Several header rows stack: read one column top to bottom for its full header path.'),
|
|
439
506
|
rows: z
|
|
440
507
|
.array(z.array(z.string()).describe('One row, as cell text by grid column'))
|
|
441
|
-
.describe('Cell text by row, in document order, one entry per grid column. `colspan` and `rowspan` are expanded, so a cell covering several columns or rows repeats its text across each cell it covers and a well-formed table is rectangular — align on position from the left, and read a repeated value as one spanning cell rather than several measurements. Empty when `unextractableReason` is set.'),
|
|
442
|
-
footnotes: z
|
|
508
|
+
.describe('Cell text by row, in document order, one entry per grid column. `colspan` and `rowspan` are expanded, so a cell covering several columns or rows repeats its text across each cell it covers and a well-formed table is rectangular — align on position from the left, and read a repeated value as one spanning cell rather than several measurements. A line break in a cell — a `<break/>`, a paragraph, or a list item in the source — is kept as `\\n`, so `1.08\\n2.44` is two values; superscripts and other inline markup are never spaced (`0.108a`). Empty when `unextractableReason` is set.'),
|
|
509
|
+
footnotes: z
|
|
510
|
+
.string()
|
|
511
|
+
.optional()
|
|
512
|
+
.describe('`<table-wrap-foot>` text, one line per footnote or paragraph, joined by `\\n`. A footnote marker deposited as a `<label>` leads its line, followed by a space (`* No patient …`).'),
|
|
443
513
|
unextractableReason: z
|
|
444
514
|
.enum(['cals-tgroup', 'graphic-only', 'no-rows'])
|
|
445
515
|
.optional()
|
|
@@ -477,7 +547,7 @@ const AssetSchema = z
|
|
|
477
547
|
sectionTitle: z
|
|
478
548
|
.string()
|
|
479
549
|
.optional()
|
|
480
|
-
.describe('Title of the innermost section enclosing the asset
|
|
550
|
+
.describe('Title of the innermost titled section enclosing the asset — a body section or a back-matter one such as an appendix — as that section is titled in `sections`, or, for a heading nested deeper than `sections` carries, as its heading line inside the text of the section that contains it. Absent for an asset inside no titled section, such as a `<floats-group>` deposit.'),
|
|
481
551
|
href: z
|
|
482
552
|
.string()
|
|
483
553
|
.optional()
|
|
@@ -518,7 +588,9 @@ const PmcArticleSchema = z
|
|
|
518
588
|
keywords: z.array(z.string()).optional().describe('Keywords'),
|
|
519
589
|
articleType: z.string().optional().describe('Article type'),
|
|
520
590
|
publicationDate: PublicationDateSchema.optional(),
|
|
521
|
-
sections: z
|
|
591
|
+
sections: z
|
|
592
|
+
.array(SectionSchema)
|
|
593
|
+
.describe('Article sections in source order: the body, then the back matter — acknowledgments, notes and declarations, footnotes, glossary, appendices. A back-matter element with no heading of its own is titled by its kind, such as `Footnotes` or `Appendix`.'),
|
|
522
594
|
tables: z
|
|
523
595
|
.array(TableSchema)
|
|
524
596
|
.optional()
|
|
@@ -526,7 +598,7 @@ const PmcArticleSchema = z
|
|
|
526
598
|
assets: z
|
|
527
599
|
.array(AssetSchema)
|
|
528
600
|
.optional()
|
|
529
|
-
.describe('Every `<fig>` and `<supplementary-material>` the article carries, in document order — from the body and from `<floats-group>`, `<back>` and appendices alike. Each one lifted from
|
|
601
|
+
.describe('Every `<fig>` and `<supplementary-material>` the article carries, in document order — from the body and from `<floats-group>`, `<back>` and appendices alike. Each one lifted from a section, body or back matter, leaves a `[Figure: <label>]` or `[Supplementary: <label>]` marker at its position in the section text, so reading order survives the lift. Absent when the article deposits none, when `includeAssets` is false, or when a `sections` filter left none standing.'),
|
|
530
602
|
references: z.array(ReferenceSchema).optional().describe('Reference list'),
|
|
531
603
|
epmcId: z
|
|
532
604
|
.string()
|
|
@@ -552,11 +624,11 @@ const UnpaywallArticleSchema = z
|
|
|
552
624
|
pmcId: z
|
|
553
625
|
.string()
|
|
554
626
|
.optional()
|
|
555
|
-
.describe('PMC ID this article was requested under, in `PMC<digits>` form — present
|
|
627
|
+
.describe('PMC ID this article was requested under, in `PMC<digits>` form — present when a `pmcids` entry named it, absent when it was requested under a PMID or DOI. When several identifiers name one article, it is requested under the first to reach it: `pmcids`, then `pmids`, then `dois`. Ties the article back to the requested identifier, which `unavailable[]` keys on for the ids that found nothing.'),
|
|
556
628
|
pmid: z
|
|
557
629
|
.string()
|
|
558
630
|
.optional()
|
|
559
|
-
.describe('PubMed ID when
|
|
631
|
+
.describe('PubMed ID this article was requested under — present when it was requested under a `pmids` entry, absent when under a PMC ID or DOI (see `pmcId` for which identifier an article several of them name is requested under)'),
|
|
560
632
|
pubmedUrl: z.string().optional().describe('PubMed URL — present when `pmid` is set'),
|
|
561
633
|
doi: z.string().describe('DOI used to locate the open-access copy'),
|
|
562
634
|
sourceUrl: z.string().describe('URL the content was fetched from'),
|
|
@@ -638,7 +710,9 @@ const UnavailableSchema = z
|
|
|
638
710
|
id: z
|
|
639
711
|
.string()
|
|
640
712
|
.describe('Identifier the chain could not resolve — PMID, PMCID, or DOI per `idType`'),
|
|
641
|
-
idType: z
|
|
713
|
+
idType: z
|
|
714
|
+
.enum(['pmid', 'pmcid', 'doi'])
|
|
715
|
+
.describe('Which input field the id was sent in — `pmids`, `pmcids`, or `dois` respectively'),
|
|
642
716
|
reason: UnavailableReasonSchema,
|
|
643
717
|
triedTiers: z
|
|
644
718
|
.array(TriedTierSchema)
|
|
@@ -699,7 +773,7 @@ const TruncatedSectionSchema = z
|
|
|
699
773
|
.optional()
|
|
700
774
|
.describe('Per-subsection accounting for a shortened section, in document order, including subsections dropped for budget — where inside the section the cut landed. Absent when the section was returned whole or carries no subsections.'),
|
|
701
775
|
})
|
|
702
|
-
.describe('Character accounting for one
|
|
776
|
+
.describe('Character accounting for one section of a budgeted article');
|
|
703
777
|
const TruncatedArticleSchema = z
|
|
704
778
|
.object({
|
|
705
779
|
id: z
|
|
@@ -707,7 +781,7 @@ const TruncatedArticleSchema = z
|
|
|
707
781
|
.describe('Identifier for the article — PMCID, PMID, DOI, or Europe PMC id, whichever the article carries first'),
|
|
708
782
|
source: z
|
|
709
783
|
.enum(['pmc', 'unpaywall'])
|
|
710
|
-
.describe('Which output shape was budgeted: `pmc` budgets
|
|
784
|
+
.describe('Which output shape was budgeted: `pmc` budgets sections and subsections, `unpaywall` budgets the single `content` body'),
|
|
711
785
|
originalCharacters: z
|
|
712
786
|
.number()
|
|
713
787
|
.describe('Body characters this article carried before the budget pass'),
|
|
@@ -727,7 +801,7 @@ const TruncatedArticleSchema = z
|
|
|
727
801
|
omittedAssets: z
|
|
728
802
|
.number()
|
|
729
803
|
.optional()
|
|
730
|
-
.describe('Figures and supplementary items this article dropped whole because the budget left no room once sections and tables were served. An asset is never returned with a truncated caption, so it is either returned complete or counted here. Absent when none were dropped.'),
|
|
804
|
+
.describe('Figures and supplementary items this article dropped whole because the budget left no room once body sections and tables were served. An asset is never returned with a truncated caption, so it is either returned complete or counted here. Absent when none were dropped.'),
|
|
731
805
|
omittedAssetNames: z
|
|
732
806
|
.array(z.string())
|
|
733
807
|
.optional()
|
|
@@ -752,7 +826,7 @@ const TruncationSchema = z
|
|
|
752
826
|
.describe('Body characters the shortened articles carry in this response'),
|
|
753
827
|
omittedSections: z
|
|
754
828
|
.number()
|
|
755
|
-
.describe('
|
|
829
|
+
.describe('Sections and subsections, back matter included, dropped entirely because an article budget was exhausted before reaching them. Back matter is reached last, after tables and assets. A dropped section counts once, together with its subsections. Always 0 in `outline` mode, which keeps every heading.'),
|
|
756
830
|
omittedTables: z
|
|
757
831
|
.number()
|
|
758
832
|
.optional()
|
|
@@ -779,15 +853,54 @@ const DeferredSchema = z
|
|
|
779
853
|
.describe('Articles the chain resolved but withheld to stay under the ceiling'),
|
|
780
854
|
idType: z
|
|
781
855
|
.enum(['pmid', 'pmcid', 'doi'])
|
|
782
|
-
.
|
|
856
|
+
.optional()
|
|
857
|
+
.describe('The input field every deferred id was sent in — re-submit them as `pmids`, `pmcids`, or `dois` respectively. Matches the `idType` on `unavailable` entries. Absent when the deferred ids came from more than one field; `pmcids` / `pmids` / `dois` then group them.'),
|
|
783
858
|
ids: z
|
|
784
859
|
.array(z.string())
|
|
785
|
-
.describe('Identifiers of the deferred articles, in response order, keyed as they were requested (PMC IDs in `PMC<digits>` form). Re-call `pubmed_fetch_fulltext` with these under the `idType`
|
|
860
|
+
.describe('Identifiers of the deferred articles, in response order, keyed as they were requested (PMC IDs in `PMC<digits>` form). Re-call `pubmed_fetch_fulltext` with these under the `idType` field — or, when `idType` is absent, with the `pmcids` / `pmids` / `dois` lists below — and the same other inputs. Never contains an id from `unavailable`.'),
|
|
861
|
+
pmcids: z
|
|
862
|
+
.array(z.string())
|
|
863
|
+
.optional()
|
|
864
|
+
.describe('The deferred ids sent as PMC IDs, in response order — re-submit them as `pmcids`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a PMC ID.'),
|
|
865
|
+
pmids: z
|
|
866
|
+
.array(z.string())
|
|
867
|
+
.optional()
|
|
868
|
+
.describe('The deferred ids sent as PMIDs, in response order — re-submit them as `pmids`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a PMID.'),
|
|
869
|
+
dois: z
|
|
870
|
+
.array(z.string())
|
|
871
|
+
.optional()
|
|
872
|
+
.describe('The deferred ids sent as DOIs, in response order — re-submit them as `dois`. Present only when the deferred ids came from more than one field (so `idType` is absent) and one of them was a DOI.'),
|
|
786
873
|
nextDeferredCharacters: z
|
|
787
874
|
.number()
|
|
788
875
|
.describe('Serialized size of the next deferred article — the first entry in `ids`, where the response stopped. Raise `maxResponseCharacters` to at least this to make progress; a smaller article further down `ids` cannot be reached until this one fits.'),
|
|
789
876
|
})
|
|
790
877
|
.describe('Continuation state for articles the whole-response budget withheld. Present only when `maxResponseCharacters` deferred at least one article.');
|
|
878
|
+
const OverLimitSchema = z
|
|
879
|
+
.object({
|
|
880
|
+
limit: z
|
|
881
|
+
.number()
|
|
882
|
+
.describe('Distinct identifiers one call fetches. The first this many, in routing order — `pmcids`, then `pmids`, then `dois`, request order within each — were fetched; the ids below were not.'),
|
|
883
|
+
idType: z
|
|
884
|
+
.enum(['pmid', 'pmcid', 'doi'])
|
|
885
|
+
.optional()
|
|
886
|
+
.describe('The input field every over-limit id was sent in — re-submit them as `pmids`, `pmcids`, or `dois` respectively. Absent when the over-limit ids came from more than one field; `pmcids` / `pmids` / `dois` then group them.'),
|
|
887
|
+
ids: z
|
|
888
|
+
.array(z.string())
|
|
889
|
+
.describe('Identifiers past the per-call limit, none of them fetched, in routing order — each listed once, under the first spelling sent (PMC IDs in `PMC<digits>` form). Re-call `pubmed_fetch_fulltext` with these under the `idType` field — or, when `idType` is absent, with the `pmcids` / `pmids` / `dois` lists below — and the same other inputs, at most `limit` per call. Never contains an id from `unavailable` or `deferred`.'),
|
|
890
|
+
pmcids: z
|
|
891
|
+
.array(z.string())
|
|
892
|
+
.optional()
|
|
893
|
+
.describe('The over-limit ids sent as PMC IDs, in routing order — re-submit them as `pmcids`. Present only when the over-limit ids came from more than one field (so `idType` is absent) and one of them was a PMC ID.'),
|
|
894
|
+
pmids: z
|
|
895
|
+
.array(z.string())
|
|
896
|
+
.optional()
|
|
897
|
+
.describe('The over-limit ids sent as PMIDs, in routing order — re-submit them as `pmids`. Present only when the over-limit ids came from more than one field (so `idType` is absent) and one of them was a PMID.'),
|
|
898
|
+
dois: z
|
|
899
|
+
.array(z.string())
|
|
900
|
+
.optional()
|
|
901
|
+
.describe('The over-limit ids sent as DOIs, in routing order — re-submit them as `dois`. Present only when the over-limit ids came from more than one field (so `idType` is absent) and one of them was a DOI.'),
|
|
902
|
+
})
|
|
903
|
+
.describe('Identifiers the call named past the per-call fetch limit, returned unfetched for a follow-up call. Present only when the call named more than `limit` distinct identifiers. Separate from `deferred`, which lists articles that were fetched but withheld by `maxResponseCharacters`.');
|
|
791
904
|
/** True when the request asked for any budget at all. Without one, every budget
|
|
792
905
|
* helper returns its input untouched so the response is byte-identical. */
|
|
793
906
|
function budgetRequested(budget) {
|
|
@@ -993,11 +1106,16 @@ function allotSectionBudgets(sizes, budget) {
|
|
|
993
1106
|
* or cut — the budget spends on body text and table content, keeping every
|
|
994
1107
|
* article citable.
|
|
995
1108
|
*
|
|
996
|
-
* Body sections are served first, then tables, then assets,
|
|
997
|
-
* `maxCharacters` has left, in document order
|
|
998
|
-
*
|
|
999
|
-
*
|
|
1000
|
-
*
|
|
1109
|
+
* Body sections are served first, then tables, then assets, then back-matter
|
|
1110
|
+
* sections, each spending what `maxCharacters` has left, in document order. A
|
|
1111
|
+
* table or asset that does not fit is dropped whole and counted, never
|
|
1112
|
+
* truncated into a partial grid or a caption cut short. Back matter comes last
|
|
1113
|
+
* so it never displaces a table or figure: it is shortened or dropped the way
|
|
1114
|
+
* body sections are, from whatever the rest left. The first `bodySections`
|
|
1115
|
+
* entries of `article.sections` are the body; spend order aside, `sections` and
|
|
1116
|
+
* the ledger stay in document order. With no `maxCharacters` — a bare
|
|
1117
|
+
* `maxCharactersPerSection` request — nothing bounds the tables or assets and
|
|
1118
|
+
* every entry is kept. (#111, #130, #206)
|
|
1001
1119
|
*
|
|
1002
1120
|
* Returns the article untouched (same object identity) when no budget was
|
|
1003
1121
|
* requested or nothing exceeded it. A section or subsection left with zero
|
|
@@ -1005,7 +1123,7 @@ function allotSectionBudgets(sizes, budget) {
|
|
|
1005
1123
|
* keeps it as a heading-only entry. Dropped nodes still appear in the accounting
|
|
1006
1124
|
* so the caller can see which headings exist. (#81, #143)
|
|
1007
1125
|
*/
|
|
1008
|
-
function applyPmcBudget(article, budget) {
|
|
1126
|
+
function applyPmcBudget(article, budget, bodySections) {
|
|
1009
1127
|
const tables = article.tables ?? [];
|
|
1010
1128
|
const assets = article.assets ?? [];
|
|
1011
1129
|
if (!budgetRequested(budget) ||
|
|
@@ -1016,31 +1134,37 @@ function applyPmcBudget(article, budget) {
|
|
|
1016
1134
|
const tablesOriginal = tables.reduce((sum, table) => sum + tableCharacters(table), 0);
|
|
1017
1135
|
const assetsOriginal = assets.reduce((sum, asset) => sum + assetCharacters(asset), 0);
|
|
1018
1136
|
const originalCharacters = sizes.reduce((sum, size) => sum + size, 0) + tablesOriginal + assetsOriginal;
|
|
1019
|
-
const allowances = allotSectionBudgets(sizes, budget);
|
|
1020
1137
|
const kept = [];
|
|
1021
1138
|
const sectionReports = [];
|
|
1022
1139
|
let returnedCharacters = 0;
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
const fit = fitSectionTree(section, fitted, { i: 0 }, budget.overflowMode);
|
|
1026
|
-
returnedCharacters += fit.entry.returnedCharacters;
|
|
1027
|
-
sectionReports.push(fit.entry);
|
|
1028
|
-
if (fit.kept)
|
|
1029
|
-
kept.push(fit.kept);
|
|
1030
|
-
});
|
|
1031
|
-
const omittedSections = countDroppedSections(sectionReports, budget.overflowMode);
|
|
1032
|
-
// Sections are served first, then tables, then assets — each spending whatever
|
|
1033
|
-
// `maxCharacters` has left. A bare per-section budget sets no total, so nothing
|
|
1034
|
-
// bounds either list.
|
|
1140
|
+
// Each stage spends whatever `maxCharacters` has left. A bare per-section
|
|
1141
|
+
// budget sets no total, so nothing bounds the tables or assets.
|
|
1035
1142
|
const remainingAllowance = () => budget.maxCharacters === undefined
|
|
1036
1143
|
? Number.POSITIVE_INFINITY
|
|
1037
1144
|
: Math.max(budget.maxCharacters - returnedCharacters, 0);
|
|
1145
|
+
const fitSections = (sections, total) => {
|
|
1146
|
+
const allowances = allotSectionBudgets(sections.map(sectionCharacters), {
|
|
1147
|
+
...budget,
|
|
1148
|
+
maxCharacters: total,
|
|
1149
|
+
});
|
|
1150
|
+
sections.forEach((section, i) => {
|
|
1151
|
+
const fitted = fitFields(sectionTextFields(section), allowances[i] ?? 0);
|
|
1152
|
+
const fit = fitSectionTree(section, fitted, { i: 0 }, budget.overflowMode);
|
|
1153
|
+
returnedCharacters += fit.entry.returnedCharacters;
|
|
1154
|
+
sectionReports.push(fit.entry);
|
|
1155
|
+
if (fit.kept)
|
|
1156
|
+
kept.push(fit.kept);
|
|
1157
|
+
});
|
|
1158
|
+
};
|
|
1159
|
+
fitSections(article.sections.slice(0, bodySections), budget.maxCharacters);
|
|
1038
1160
|
const fittedTables = fitWholeNamed(tables, remainingAllowance(), tableCharacters, tableDisplayName);
|
|
1039
1161
|
returnedCharacters += fittedTables.spent;
|
|
1040
1162
|
const omittedTables = fittedTables.omittedNames.length;
|
|
1041
1163
|
const fittedAssets = fitWholeNamed(assets, remainingAllowance(), assetCharacters, assetDisplayName);
|
|
1042
1164
|
returnedCharacters += fittedAssets.spent;
|
|
1043
1165
|
const omittedAssets = fittedAssets.omittedNames.length;
|
|
1166
|
+
fitSections(article.sections.slice(bodySections), budget.maxCharacters === undefined ? undefined : remainingAllowance());
|
|
1167
|
+
const omittedSections = countDroppedSections(sectionReports, budget.overflowMode);
|
|
1044
1168
|
if (returnedCharacters === originalCharacters &&
|
|
1045
1169
|
omittedSections === 0 &&
|
|
1046
1170
|
omittedTables === 0 &&
|
|
@@ -1180,11 +1304,46 @@ function buildTruncationNotice(truncation) {
|
|
|
1180
1304
|
*/
|
|
1181
1305
|
function buildDeferralNotice(deferred) {
|
|
1182
1306
|
const spent = deferred.returnedCharacters === 0
|
|
1183
|
-
? `
|
|
1307
|
+
? `The first article alone exceeds the requested maxResponseCharacters of ${deferred.maxResponseCharacters}, so none were returned.`
|
|
1184
1308
|
: `Response character budget reached: ${deferred.returnedCharacters} of ${deferred.maxResponseCharacters} characters returned.`;
|
|
1185
|
-
|
|
1309
|
+
const resend = deferred.idType
|
|
1310
|
+
? `Re-call pubmed_fetch_fulltext with those ids under \`${deferred.idType}s\``
|
|
1311
|
+
: `Re-call pubmed_fetch_fulltext with each id under its own field (${idFieldLists(deferred)})`;
|
|
1312
|
+
return `${spent} ${deferred.deferredCount} resolved article(s) were deferred whole: ${deferred.ids.join(', ')}. ${resend} to retrieve them, or raise maxResponseCharacters to at least ${deferred.nextDeferredCharacters} — the size of the next deferred article.`;
|
|
1313
|
+
}
|
|
1314
|
+
/**
|
|
1315
|
+
* Compose the recovery notice for identifiers past the per-call fetch limit:
|
|
1316
|
+
* how many were left unfetched, which ones, and where to resend them — so a
|
|
1317
|
+
* caller reading only `content[]` can retrieve the rest without inspecting
|
|
1318
|
+
* `overLimit`. (#222)
|
|
1319
|
+
*/
|
|
1320
|
+
function buildOverLimitNotice(overLimit) {
|
|
1321
|
+
const resend = overLimit.idType
|
|
1322
|
+
? `Re-call pubmed_fetch_fulltext with those ids under \`${overLimit.idType}s\``
|
|
1323
|
+
: `Re-call pubmed_fetch_fulltext with each id under its own field (${idFieldLists(overLimit)})`;
|
|
1324
|
+
return `This call fetched the first ${overLimit.limit} distinct identifiers; ${overLimit.ids.length} more were not fetched: ${overLimit.ids.join(', ')}. ${resend}, at most ${overLimit.limit} per call, to retrieve them.`;
|
|
1325
|
+
}
|
|
1326
|
+
/**
|
|
1327
|
+
* The per-field lists of ids handed back across fields, in field order —
|
|
1328
|
+
* `` `pmcids`: PMC1; `dois`: 10.1/x `` — or `''` when `idType` covers them all.
|
|
1329
|
+
*/
|
|
1330
|
+
function idFieldLists(group) {
|
|
1331
|
+
return ['pmcids', 'pmids', 'dois']
|
|
1332
|
+
.flatMap((field) => {
|
|
1333
|
+
const ids = group[field];
|
|
1334
|
+
return ids?.length ? [`\`${field}\`: ${ids.join(', ')}`] : [];
|
|
1335
|
+
})
|
|
1336
|
+
.join('; ');
|
|
1186
1337
|
}
|
|
1187
1338
|
// ─── Tool Definition ─────────────────────────────────────────────────────────
|
|
1339
|
+
/**
|
|
1340
|
+
* Distinct identifiers one call fetches across `pmcids`, `pmids`, and `dois`,
|
|
1341
|
+
* in routing order; the rest are returned unfetched in `overLimit`. Bounds the
|
|
1342
|
+
* candidates every tier works per call. (#192, #222)
|
|
1343
|
+
*/
|
|
1344
|
+
const FETCH_LIMIT = 10;
|
|
1345
|
+
/** Distinct identifiers one call may name across the three fields. (#222) */
|
|
1346
|
+
const MAX_IDENTIFIERS = 50;
|
|
1188
1347
|
/**
|
|
1189
1348
|
* Compose the tool description for the fallback tiers enabled in this
|
|
1190
1349
|
* deployment. PMC EFetch is always present; Europe PMC (`EUROPEPMC_ENABLED`)
|
|
@@ -1193,7 +1352,7 @@ function buildDeferralNotice(deferred) {
|
|
|
1193
1352
|
* recoveries that silently can't happen.
|
|
1194
1353
|
*/
|
|
1195
1354
|
export function buildFulltextDescription(tiers) {
|
|
1196
|
-
const base = 'Fetch full-text articles from PubMed Central
|
|
1355
|
+
const base = 'Fetch full-text articles from PubMed Central as structured sections — the body, then back matter such as acknowledgments, declarations, and appendices — with tables and references.';
|
|
1197
1356
|
const epmcClause = 'Europe PMC `fullTextXML` (structured JATS for records with a PMC counterpart)';
|
|
1198
1357
|
const unpaywallClause = 'Unpaywall — publisher-hosted or institutional open-access copies as HTML-as-Markdown or PDF-as-text';
|
|
1199
1358
|
let fallback;
|
|
@@ -1217,7 +1376,7 @@ export function buildFulltextDescription(tiers) {
|
|
|
1217
1376
|
: tiers.unpaywall
|
|
1218
1377
|
? '; DOIs with no PMC copy recover via Unpaywall open access'
|
|
1219
1378
|
: '';
|
|
1220
|
-
const input = `
|
|
1379
|
+
const input = `Name articles by \`pmcids\` (PMC IDs directly), \`pmids\` (PubMed IDs, auto-resolved), \`dois\` (DOIs, auto-resolved to PMC via the ID Converter${doiTail}), or any mix of the three — up to ${MAX_IDENTIFIERS} distinct identifiers; the first ${FETCH_LIMIT} are fetched per call, the rest returned in \`overLimit\`; an article several of them name is fetched and returned once.`;
|
|
1221
1380
|
const budget = 'Two independent character controls: `maxCharacters` caps body text per article, `maxResponseCharacters` caps the whole response and defers articles past the ceiling whole, listing them in `deferred.ids` for a follow-up call.';
|
|
1222
1381
|
return `${base} ${fallback} ${input} ${budget}`;
|
|
1223
1382
|
}
|
|
@@ -1233,28 +1392,42 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1233
1392
|
// Only the ID routing's `idConvert` calls run unwrapped. Every Europe PMC and
|
|
1234
1393
|
// Unpaywall call sits behind a catch that folds the failure into
|
|
1235
1394
|
// `unavailable[].triedTiers`, so their service reasons never reach a caller
|
|
1236
|
-
// and are not declared here. (#168)
|
|
1237
|
-
|
|
1395
|
+
// and are not declared here. (#168) `blank_filter` is the handler's own input
|
|
1396
|
+
// check, raised before any upstream request. (#186)
|
|
1397
|
+
errors: [
|
|
1398
|
+
...NCBI_SERVICE_ERRORS,
|
|
1399
|
+
{
|
|
1400
|
+
reason: 'blank_filter',
|
|
1401
|
+
code: JsonRpcErrorCode.ValidationError,
|
|
1402
|
+
when: 'A `sections` element holds no visible character — only whitespace, or invisible characters such as a zero-width space — so it names no heading. An exactly-empty element is not blank here; it is skipped.',
|
|
1403
|
+
recovery: 'Give each named `sections` element a heading term such as Methods, or make it an empty string, which is skipped; omit `sections` for no filter. The same blank element will be rejected again.',
|
|
1404
|
+
retryable: false,
|
|
1405
|
+
},
|
|
1406
|
+
],
|
|
1407
|
+
// Never advertised; rewritten to the canonical key before the schema parses.
|
|
1408
|
+
// Singular spellings sibling tools and this tool's own output use; a lone
|
|
1409
|
+
// string under one becomes a one-element array. (#221)
|
|
1410
|
+
inputAliases: { pmcid: 'pmcids', pmcId: 'pmcids', pmid: 'pmids', doi: 'dois' },
|
|
1238
1411
|
input: z
|
|
1239
1412
|
.object({
|
|
1240
1413
|
pmcids: z
|
|
1241
1414
|
.array(pmcidStringSchema)
|
|
1242
1415
|
.min(1)
|
|
1243
|
-
.max(
|
|
1416
|
+
.max(MAX_IDENTIFIERS)
|
|
1244
1417
|
.optional()
|
|
1245
|
-
.describe(
|
|
1418
|
+
.describe(`PMC IDs to fetch (e.g. ["PMC9575052"]). Combinable with \`pmids\` and \`dois\` — up to ${MAX_IDENTIFIERS} distinct identifiers across the three; the first ${FETCH_LIMIT} are fetched per call, the rest returned in \`overLimit\`. PMC IDs with no retrievable full text fall through to Europe PMC, then to Unpaywall on the DOI the chain resolves for them.`),
|
|
1246
1419
|
pmids: z
|
|
1247
1420
|
.array(pmidStringSchema)
|
|
1248
1421
|
.min(1)
|
|
1249
|
-
.max(
|
|
1422
|
+
.max(MAX_IDENTIFIERS)
|
|
1250
1423
|
.optional()
|
|
1251
|
-
.describe(
|
|
1424
|
+
.describe(`PubMed IDs. Combinable with \`pmcids\` and \`dois\` — up to ${MAX_IDENTIFIERS} distinct identifiers across the three; the first ${FETCH_LIMIT} are fetched per call, the rest returned in \`overLimit\`. Articles in PMC are returned as structured JATS; articles not in PMC fall through to Europe PMC (when EPMC has a \`fullTextXML\`), then to Unpaywall when \`UNPAYWALL_EMAIL\` is set and a DOI is available.`),
|
|
1252
1425
|
dois: z
|
|
1253
1426
|
.array(doiStringSchema)
|
|
1254
1427
|
.min(1)
|
|
1255
|
-
.max(
|
|
1428
|
+
.max(MAX_IDENTIFIERS)
|
|
1256
1429
|
.optional()
|
|
1257
|
-
.describe(
|
|
1430
|
+
.describe(`DOIs to resolve (e.g. ["10.21203/rs.3.rs-9010375/v1"]), one per element. Combinable with \`pmcids\` and \`pmids\` — up to ${MAX_IDENTIFIERS} distinct identifiers across the three; the first ${FETCH_LIMIT} are fetched per call, the rest returned in \`overLimit\`. Resolved to a PMCID via the PMC ID Converter and returned as structured JATS when the article is in PMC; DOIs with no PMC counterpart (preprints, EPMC-only OA) fall through to Europe PMC, then Unpaywall, when those layers are enabled.`),
|
|
1258
1431
|
includeReferences: z
|
|
1259
1432
|
.boolean()
|
|
1260
1433
|
.default(false)
|
|
@@ -1271,41 +1444,52 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1271
1444
|
.number()
|
|
1272
1445
|
.int()
|
|
1273
1446
|
.min(1)
|
|
1274
|
-
.max(50)
|
|
1275
1447
|
.optional()
|
|
1276
|
-
.describe('Maximum top-level body
|
|
1448
|
+
.describe('Maximum top-level sections. Back-matter sections count and follow the body, so a cap drops them first. Applies to `source=pmc` results only.'),
|
|
1277
1449
|
sections: z
|
|
1278
1450
|
.array(z.string())
|
|
1279
1451
|
.optional()
|
|
1280
|
-
.describe('Filter to specific sections by title (e.g. ["Introduction", "Methods", "Results", "Discussion"]). A term matches a section or subsection title at any nesting depth, case-insensitively, as a substring — "resul" matches "Results". A section whose own title matches is returned whole; one kept only because a nested subsection matched keeps its heading as a breadcrumb, with its own text cleared and only the matching branch beneath it. Tables and assets narrow with the filter: one whose section did not survive, or that names no section, is dropped. Applies to `source=pmc` results only.'),
|
|
1452
|
+
.describe('Filter to specific sections by title (e.g. ["Introduction", "Methods", "Results", "Discussion"]). A term matches a section or subsection title at any nesting depth, case-insensitively, as a substring — "resul" matches "Results". Back-matter sections match the same way: by their own titles, or by their kind when they carry neither a title nor a label — "appendix" matches a section titled "Appendix A", "footnotes" an untitled footnote group. A label is never matched, so an appendix labelled "Appendix A" under a title of its own matches only on that title. A section whose own title matches is returned whole; one kept only because a nested subsection matched keeps its heading as a breadcrumb, with its own text cleared and only the matching branch beneath it. Tables and assets narrow with the filter: one whose section did not survive, or that names no section, is dropped. Empty strings are skipped, so `[""]` applies no filter; an element of only whitespace or invisible characters, such as a zero-width space, is rejected. Applies to `source=pmc` results only.'),
|
|
1281
1453
|
maxCharacters: z
|
|
1282
1454
|
.number()
|
|
1283
1455
|
.int()
|
|
1284
1456
|
.min(1)
|
|
1285
|
-
.max(1_000_000)
|
|
1286
1457
|
.optional()
|
|
1287
|
-
.describe('Per-article budget for body text, in characters. Counts `source=pmc` section and subsection text — which carries the inline blocks the parser renders in place, such as lists, definition lists, block quotes, boxed text, preformatted blocks and displayed formulae — plus table label, caption, cell and footnote text and asset label, caption and `href` text; or the `source=unpaywall` `content` body. Titles, abstracts, identifiers, and references are never counted or shortened. Shortened text ends at the last word boundary inside its allowance, so it can come back a few characters under it. The counted unit is that text alone — the Markdown grid `content[]` renders around the cells (pipes, padding, the divider row, headings) is scaffolding this budget does not measure, so a table renders longer than it costs here.
|
|
1458
|
+
.describe('Per-article budget for body text, in characters. Counts `source=pmc` section and subsection text — which carries the inline blocks the parser renders in place, such as lists, definition lists, block quotes, boxed text, preformatted blocks and displayed formulae — plus table label, caption, cell and footnote text and asset label, caption and `href` text; or the `source=unpaywall` `content` body. Titles, abstracts, identifiers, and references are never counted or shortened. Shortened text ends at the last word boundary inside its allowance, so it can come back a few characters under it. The counted unit is that text alone — the Markdown grid `content[]` renders around the cells (pipes, padding, the divider row, headings) is scaffolding this budget does not measure, so a table renders longer than it costs here. Body sections are served first, then tables, then assets, then back-matter sections, each spending what is left, in document order. For tables and assets, admission stops at the first entry that does not fit, and every entry from there on is dropped whole rather than cut mid-row or returned with a shortened caption, counted in `truncation.omittedTables` / `truncation.omittedAssets` and named in `truncation.articles[].omittedTableNames` / `omittedAssetNames`. Back matter takes only what remains and is shortened or dropped as body sections are; `sections` still returns it after the body. Applied after `sections`, `maxSections`, `includeReferences`, `includeTables`, and `includeAssets`, so semantic filtering is unaffected. This knob alone bounds only bodies: the response-wide ceiling it implies is this value times the number of articles returned, plus every uncounted field. Use `maxResponseCharacters` for a true whole-response ceiling. Omit for the full body.'),
|
|
1288
1459
|
maxCharactersPerSection: z
|
|
1289
1460
|
.number()
|
|
1290
1461
|
.int()
|
|
1291
1462
|
.min(1)
|
|
1292
|
-
.max(1_000_000)
|
|
1293
1463
|
.optional()
|
|
1294
|
-
.describe('Budget for a single top-level
|
|
1464
|
+
.describe('Budget for a single top-level section, in characters, counting the section text plus its subsections. Combine with `maxCharacters` to cap both one section and the article; the tighter of the two wins. Applies to `source=pmc` results only.'),
|
|
1295
1465
|
maxResponseCharacters: z
|
|
1296
1466
|
.number()
|
|
1297
1467
|
.int()
|
|
1298
1468
|
.min(1)
|
|
1299
|
-
.max(1_000_000)
|
|
1300
1469
|
.optional()
|
|
1301
|
-
.describe('Opt-in ceiling for the whole response, in characters — the true response-wide counterpart to the per-article `maxCharacters`. Each article is measured as the JSON record it is returned as, after every filter and the per-article body budget: title, abstract,
|
|
1470
|
+
.describe('Opt-in ceiling for the whole response, in characters — the true response-wide counterpart to the per-article `maxCharacters`. Each article is measured as the JSON record it is returned as, after every filter and the per-article body budget: title, abstract, sections, references, identifiers, license and source metadata — every field it carries. One ledger covers all tiers, so PMC-, Europe PMC-, and Unpaywall-served articles spend the same budget. Articles are kept in response order until the next one would cross the ceiling; that article and the rest are deferred whole (never partially populated) and listed in `deferred.ids`. Response envelope fields — counts, `unavailable`, `truncation`, `deferred` itself — are not counted. Omit to return every resolved article.'),
|
|
1302
1471
|
overflowMode: z
|
|
1303
1472
|
.enum(['truncate', 'outline'])
|
|
1304
1473
|
.default('truncate')
|
|
1305
|
-
.describe('How to spend `maxCharacters` across an article that exceeds it. truncate: fill sections in document order, so early sections stay whole, the section the budget runs out in is cut, and every section or subsection past that point is dropped (counted in `truncation.omittedSections`). outline: split the budget evenly so every section and subsection keeps its heading, and an excerpt as far as the budget reaches — a heading the budget left empty is marked as such in the rendered text. Use it to survey what an article contains before requesting specific `sections`. Ignored when no budget is set, and identical for `source=unpaywall` bodies, which have no headings to preserve.'),
|
|
1474
|
+
.describe('How to spend `maxCharacters` across an article that exceeds it. truncate: fill sections in document order, so early sections stay whole, the section the budget runs out in is cut, and every section or subsection past that point is dropped (counted in `truncation.omittedSections`). outline: split the budget evenly so every section and subsection keeps its heading, and an excerpt as far as the budget reaches — a heading the budget left empty is marked as such in the rendered text. In both modes the body sections spend the budget first and back-matter sections share what the body, tables, and assets leave. Use it to survey what an article contains before requesting specific `sections`. Ignored when no budget is set, and identical for `source=unpaywall` bodies, which have no headings to preserve.'),
|
|
1306
1475
|
})
|
|
1307
|
-
|
|
1308
|
-
|
|
1476
|
+
// Counted in each field's canonical form, the one its chain keys on, so the
|
|
1477
|
+
// ceiling is settled before any upstream request; the handler then fetches
|
|
1478
|
+
// the first FETCH_LIMIT the same way. (#192, #222)
|
|
1479
|
+
.superRefine((v, refinement) => {
|
|
1480
|
+
const distinct = new Set(requestedIds(v).map((r) => r.key));
|
|
1481
|
+
if (distinct.size === 0) {
|
|
1482
|
+
refinement.addIssue({
|
|
1483
|
+
code: 'custom',
|
|
1484
|
+
message: 'Provide at least one of `pmcids`, `pmids`, or `dois`.',
|
|
1485
|
+
});
|
|
1486
|
+
}
|
|
1487
|
+
else if (distinct.size > MAX_IDENTIFIERS) {
|
|
1488
|
+
refinement.addIssue({
|
|
1489
|
+
code: 'custom',
|
|
1490
|
+
message: `\`pmcids\`, \`pmids\`, and \`dois\` together name ${distinct.size} distinct identifiers; a call accepts at most ${MAX_IDENTIFIERS} and fetches the first ${FETCH_LIMIT} — split them across calls.`,
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1309
1493
|
}),
|
|
1310
1494
|
output: z.object({
|
|
1311
1495
|
articles: z.array(ArticleSchema).describe('Full-text articles'),
|
|
@@ -1315,39 +1499,53 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1315
1499
|
unavailable: z
|
|
1316
1500
|
.array(UnavailableSchema)
|
|
1317
1501
|
.optional()
|
|
1318
|
-
.describe('Per-identifier explanations for any requested PMIDs, PMCIDs, or DOIs with no returnable full text. `idType`
|
|
1502
|
+
.describe('Per-identifier explanations for any requested PMIDs, PMCIDs, or DOIs with no returnable full text. `idType` names the field each id was sent in. Distinct from `deferred`: nothing here is retrievable by re-calling, and an id never appears in both. Ids past the per-call limit are listed in `overLimit`, never here.'),
|
|
1319
1503
|
truncation: TruncationSchema.optional(),
|
|
1320
1504
|
deferred: DeferredSchema.optional(),
|
|
1505
|
+
overLimit: OverLimitSchema.optional(),
|
|
1321
1506
|
}),
|
|
1322
|
-
// Recovery guidance for
|
|
1323
|
-
//
|
|
1507
|
+
// Recovery guidance for six cases — a `sections` filter that removed every
|
|
1508
|
+
// section (#80), a record the chain could only retrieve as front matter
|
|
1324
1509
|
// (#86), a table returned with no cell values (#111), a body the per-article
|
|
1325
|
-
// character budget shortened (#81),
|
|
1326
|
-
// withheld (#100)
|
|
1327
|
-
// ctx.enrich.notice() to structuredContent
|
|
1328
|
-
// applies.
|
|
1510
|
+
// character budget shortened (#81), articles the whole-response budget
|
|
1511
|
+
// withheld (#100), and identifiers past the per-call fetch limit (#222).
|
|
1512
|
+
// Agent-facing context surfaced via ctx.enrich.notice() to structuredContent
|
|
1513
|
+
// and content[]; absent when none applies.
|
|
1329
1514
|
enrichment: {
|
|
1330
1515
|
notice: z
|
|
1331
1516
|
.string()
|
|
1332
1517
|
.optional()
|
|
1333
|
-
.describe('Optional guidance for a partial or empty body. A `sections`-filter miss names the requested terms and affected article id(s) and suggests retrying without `sections` or using broader headings. A metadata-only record names the id(s) the chain could retrieve as front matter only and points at `pubmed_fetch_articles` for the abstract. A table returned with no cell values names the affected table(s), the article each came from, and why the cells cannot be recovered. A budgeted response names the characters returned versus carried and points at `truncation`. A response-wide budget that deferred articles names the ids to re-request. Absent when none of those applies.'),
|
|
1518
|
+
.describe('Optional guidance for a partial or empty body. A `sections`-filter miss names the requested terms and affected article id(s) and suggests retrying without `sections` or using broader headings. A metadata-only record names the id(s) the chain could retrieve as front matter only and points at `pubmed_fetch_articles` for the abstract. A table returned with no cell values names the affected table(s), the article each came from, and why the cells cannot be recovered. A budgeted response names the characters returned versus carried and points at `truncation`. A response-wide budget that deferred articles names the ids to re-request. Identifiers past the per-call limit are named with the field to resend each under. Absent when none of those applies.'),
|
|
1334
1519
|
truncated: z
|
|
1335
1520
|
.boolean()
|
|
1336
1521
|
.optional()
|
|
1337
|
-
.describe('True when a character budget shortened at least one returned body
|
|
1522
|
+
.describe('True when a character budget shortened at least one returned body or withheld a whole article, or when the call named more distinct identifiers than one call fetches. Absent when every requested identifier was fetched and every resolved article is present with its full post-filter body. The per-article body accounting is in `truncation`; the withheld ids are in `deferred`; the unfetched ids are in `overLimit`.'),
|
|
1338
1523
|
},
|
|
1339
1524
|
async handler(input, ctx) {
|
|
1525
|
+
const idCount = (input.pmcids?.length ?? 0) + (input.pmids?.length ?? 0) + (input.dois?.length ?? 0);
|
|
1340
1526
|
ctx.log.info('Executing pubmed_fetch_fulltext', {
|
|
1341
1527
|
hasPmcids: !!input.pmcids,
|
|
1342
1528
|
hasPmids: !!input.pmids,
|
|
1343
1529
|
hasDois: !!input.dois,
|
|
1344
|
-
idCount
|
|
1530
|
+
idCount,
|
|
1345
1531
|
});
|
|
1532
|
+
// A blank `sections` element fails the call before any upstream request;
|
|
1533
|
+
// every stage filters on the nonblank terms alone. (#186)
|
|
1534
|
+
const sectionFilter = splitSectionFilter(input.sections);
|
|
1535
|
+
if (sectionFilter.blank.length > 0) {
|
|
1536
|
+
const one = sectionFilter.blank.length === 1;
|
|
1537
|
+
throw ctx.fail('blank_filter', `${sectionFilter.blank.map((path) => `\`${path}\``).join(', ')} ${one ? 'holds' : 'hold'} only whitespace or invisible characters, so ${one ? 'it names' : 'they name'} no section heading.`, { fields: sectionFilter.blank });
|
|
1538
|
+
}
|
|
1539
|
+
const filters = { ...input, sections: sectionFilter.terms };
|
|
1540
|
+
// Settled before any upstream request: every tier below works the fetched
|
|
1541
|
+
// ids alone, and the rest are handed back in `overLimit`. (#222)
|
|
1542
|
+
const { fetched, overLimit: overLimitIds } = splitAtFetchLimit(input);
|
|
1346
1543
|
// ── Chain tracking ──────────────────────────────────────────────────────
|
|
1347
1544
|
// Per-input-id tier history (the `triedTiers` array on unavailable entries).
|
|
1348
|
-
// Keys:
|
|
1349
|
-
//
|
|
1350
|
-
//
|
|
1545
|
+
// Keys: canonical PMID for a `pmids` entry, prefixed PMCID for a `pmcids`
|
|
1546
|
+
// entry, first-submitted casing for a `dois` entry — disjoint shapes, so
|
|
1547
|
+
// one map holds all three. `recoveredIds` collects ids the chain produced an
|
|
1548
|
+
// article for, so we can skip them when building `unavailable[]`.
|
|
1351
1549
|
const chainByInput = new Map();
|
|
1352
1550
|
const recoveredIds = new Set();
|
|
1353
1551
|
// Per-input-id set of tiers this deployment has not configured AND that
|
|
@@ -1361,16 +1559,17 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1361
1559
|
tiers.add(tier);
|
|
1362
1560
|
unqueriedByInput.set(inputId, tiers);
|
|
1363
1561
|
};
|
|
1364
|
-
// Back-map from a
|
|
1365
|
-
//
|
|
1366
|
-
//
|
|
1562
|
+
// Back-map from a prefixed PMCID to the input id that owns its record — the
|
|
1563
|
+
// PMC ID itself, or the PMID or DOI the converter placed on it — so the PMC
|
|
1564
|
+
// stage attributes recoveries and misses to the original input id.
|
|
1367
1565
|
const pmcidToInputId = new Map();
|
|
1368
|
-
// DOI
|
|
1369
|
-
//
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
//
|
|
1373
|
-
//
|
|
1566
|
+
// A DOI the routing learned for a PMC record — from the converter, or from a
|
|
1567
|
+
// `dois` entry that named it — so a PMC miss can still reach Unpaywall
|
|
1568
|
+
// without another lookup, whichever field owns the record.
|
|
1569
|
+
const doiByPmcRecord = new Map();
|
|
1570
|
+
// Ids of returned articles whose `sections` filter removed every section —
|
|
1571
|
+
// collected across the PMC and EPMC stages to drive one recovery notice via
|
|
1572
|
+
// ctx.enrich.notice (#80).
|
|
1374
1573
|
const sectionFilterMisses = [];
|
|
1375
1574
|
// Input ids whose PMC or EPMC record carried no body sections at all. Those
|
|
1376
1575
|
// records are not full-text hits, so the chain continues past them; ids still
|
|
@@ -1386,19 +1585,46 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1386
1585
|
// caller can re-submit rather than whatever id the article happens to
|
|
1387
1586
|
// carry — a `pmids` request recovers articles keyed by PMCID. (#100)
|
|
1388
1587
|
const inputIdByArticle = new Map();
|
|
1389
|
-
// The caller's own spellings of each chain key,
|
|
1390
|
-
// `deferred
|
|
1391
|
-
// PMID's chain runs on its canonical form
|
|
1392
|
-
//
|
|
1393
|
-
//
|
|
1394
|
-
//
|
|
1588
|
+
// The caller's own spellings of each chain key, each with the field it was
|
|
1589
|
+
// sent in, so `unavailable[]` and `deferred` report what was submitted and
|
|
1590
|
+
// where. A PMID's chain runs on its canonical form and a DOI's once however
|
|
1591
|
+
// many casings name it, so those keys gather several spellings; a PMC ID is
|
|
1592
|
+
// reported in its `PMC<digits>` key form. The first entry is the key's own,
|
|
1593
|
+
// the id that opened the chain; an id that reaches a record another one
|
|
1594
|
+
// already owns is folded in after it (see `joinChain`). (#161, #166, #192)
|
|
1395
1595
|
const callerIds = new Map();
|
|
1396
|
-
const addCallerId = (key,
|
|
1596
|
+
const addCallerId = (key, callerId) => {
|
|
1397
1597
|
const spellings = callerIds.get(key) ?? [];
|
|
1398
|
-
if (!spellings.
|
|
1399
|
-
spellings.push(
|
|
1598
|
+
if (!spellings.some((s) => s.id === callerId.id && s.idType === callerId.idType)) {
|
|
1599
|
+
spellings.push(callerId);
|
|
1600
|
+
}
|
|
1400
1601
|
callerIds.set(key, spellings);
|
|
1401
1602
|
};
|
|
1603
|
+
// A joined spelling's own tiers from before its join, and the length of the
|
|
1604
|
+
// owner's chain at that point: it reports those tiers, then only what the
|
|
1605
|
+
// owner's chain gained afterwards, never a step another id took. (#192)
|
|
1606
|
+
const ownChainOf = new Map();
|
|
1607
|
+
/**
|
|
1608
|
+
* Fold `joiner` into `owner` once both are known to name one record: the
|
|
1609
|
+
* joiner's spellings become caller ids of the owner's chain, so the record is
|
|
1610
|
+
* worked once and every spelling is recovered with it, or reported
|
|
1611
|
+
* unavailable with its own tiers up to the join and the owner's after. (#192)
|
|
1612
|
+
*/
|
|
1613
|
+
const joinChain = (joiner, owner) => {
|
|
1614
|
+
const joinerChain = chainByInput.get(joiner) ?? [];
|
|
1615
|
+
const from = chainByInput.get(owner)?.length ?? 0;
|
|
1616
|
+
for (const callerId of callerIds.get(joiner) ?? []) {
|
|
1617
|
+
const own = ownChainOf.get(callerId);
|
|
1618
|
+
const tried = [...(own?.tried ?? []), ...joinerChain.slice(own?.from ?? 0)];
|
|
1619
|
+
ownChainOf.set(callerId, { tried, from });
|
|
1620
|
+
addCallerId(owner, callerId);
|
|
1621
|
+
}
|
|
1622
|
+
callerIds.delete(joiner);
|
|
1623
|
+
chainByInput.delete(joiner);
|
|
1624
|
+
unqueriedByInput.delete(joiner);
|
|
1625
|
+
if (bodylessInputIds.delete(joiner))
|
|
1626
|
+
bodylessInputIds.add(owner);
|
|
1627
|
+
};
|
|
1402
1628
|
const budget = {
|
|
1403
1629
|
overflowMode: input.overflowMode,
|
|
1404
1630
|
...(input.maxCharacters !== undefined && { maxCharacters: input.maxCharacters }),
|
|
@@ -1406,18 +1632,19 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1406
1632
|
maxCharactersPerSection: input.maxCharactersPerSection,
|
|
1407
1633
|
}),
|
|
1408
1634
|
};
|
|
1409
|
-
|
|
1410
|
-
//
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
//
|
|
1417
|
-
//
|
|
1418
|
-
//
|
|
1419
|
-
//
|
|
1420
|
-
//
|
|
1635
|
+
// ── Routing → produce the buckets the staged chain consumes ─────────────
|
|
1636
|
+
// `pmcids`, then `pmids`, then `dois`: the first id to reach a record owns
|
|
1637
|
+
// it, so the PMC IDs claim their records before any converter answer is
|
|
1638
|
+
// placed. Fallback candidates are bucketed by the field that owns them and
|
|
1639
|
+
// run in that order, which is how a later tier picks a record's owner too.
|
|
1640
|
+
const pmcIds = [];
|
|
1641
|
+
const pending = { pmcid: [], pmid: [], doi: [] };
|
|
1642
|
+
// Send a PMCID to PMC EFetch under the input id that named it. A second
|
|
1643
|
+
// input id that reaches the same PMC record — two DOIs for one article, or
|
|
1644
|
+
// a PMID for a PMC ID already given — joins the first one's chain as
|
|
1645
|
+
// another spelling of it: the record is fetched once, and both ids are
|
|
1646
|
+
// recovered, or reported unavailable with that chain, rather than the later
|
|
1647
|
+
// id taking the PMCID over and leaving the earlier one with an empty chain.
|
|
1421
1648
|
const routeToPmc = (inputId, pmcid) => {
|
|
1422
1649
|
const normalized = normalizePmcId(pmcid);
|
|
1423
1650
|
const prefixed = withPmcPrefix(normalized);
|
|
@@ -1427,135 +1654,126 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1427
1654
|
pmcidToInputId.set(prefixed, inputId);
|
|
1428
1655
|
return;
|
|
1429
1656
|
}
|
|
1430
|
-
if (owner
|
|
1431
|
-
|
|
1432
|
-
for (const spelling of callerIds.get(inputId) ?? [inputId])
|
|
1433
|
-
addCallerId(owner, spelling);
|
|
1434
|
-
callerIds.delete(inputId);
|
|
1435
|
-
chainByInput.delete(inputId);
|
|
1657
|
+
if (owner !== inputId)
|
|
1658
|
+
joinChain(inputId, owner);
|
|
1436
1659
|
};
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
const
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1660
|
+
const noteRecordDoi = (pmcid, doi) => {
|
|
1661
|
+
const prefixed = withPmcPrefix(normalizePmcId(pmcid));
|
|
1662
|
+
if (doi && !doiByPmcRecord.has(prefixed))
|
|
1663
|
+
doiByPmcRecord.set(prefixed, doi);
|
|
1664
|
+
};
|
|
1665
|
+
// `PMC123`, `pmc123`, `123` and `PMC0123` name one record; it runs the
|
|
1666
|
+
// chain once, keyed and reported in `PMC<digits>` form. (#170)
|
|
1667
|
+
for (const id of fetched.pmcids) {
|
|
1668
|
+
const normalized = normalizePmcId(id);
|
|
1669
|
+
const key = withPmcPrefix(normalized);
|
|
1670
|
+
if (!chainByInput.has(key))
|
|
1671
|
+
chainByInput.set(key, []);
|
|
1672
|
+
addCallerId(key, { id: key, idType: 'pmcid' });
|
|
1673
|
+
routeToPmc(key, normalized);
|
|
1674
|
+
}
|
|
1675
|
+
// The ID Converter parses `00000001` as PMID 1 yet reports it "not found
|
|
1676
|
+
// in PMC", and every later stage answers with NCBI's own PMID, so the chain
|
|
1677
|
+
// runs once per distinct PMID in its canonical form. (#161)
|
|
1678
|
+
for (const id of fetched.pmids)
|
|
1679
|
+
addCallerId(normalizePmid(id), { id, idType: 'pmid' });
|
|
1680
|
+
const pmids = [...new Set(fetched.pmids.map(normalizePmid))];
|
|
1681
|
+
// DOIs resolve through the converter too, so PMC-indexed DOIs reach PMC
|
|
1682
|
+
// EFetch instead of going straight to the EPMC/Unpaywall fallback (which
|
|
1683
|
+
// misses articles whose only OA copy is the PMC JATS).
|
|
1684
|
+
//
|
|
1685
|
+
// DOIs are case-insensitive, and the converter echoes one casing in
|
|
1686
|
+
// `requested-id` for DOIs that differ only in case, so every casing of a
|
|
1687
|
+
// DOI shares one chain, keyed by the first spelling submitted, and
|
|
1688
|
+
// converter records are matched to it case-insensitively. (#166)
|
|
1689
|
+
const chainKeyByDoi = new Map();
|
|
1690
|
+
for (const doi of fetched.dois) {
|
|
1691
|
+
const key = chainKeyByDoi.get(doi.toLowerCase()) ?? doi;
|
|
1692
|
+
chainKeyByDoi.set(doi.toLowerCase(), key);
|
|
1693
|
+
addCallerId(key, { id: doi, idType: 'doi' });
|
|
1694
|
+
}
|
|
1695
|
+
const dois = [...new Set(chainKeyByDoi.values())];
|
|
1696
|
+
for (const key of [...pmids, ...dois])
|
|
1697
|
+
chainByInput.set(key, []);
|
|
1698
|
+
// The two lookups are independent; their answers are placed in field order.
|
|
1699
|
+
const convertOptions = ctx.signal ? { signal: ctx.signal } : undefined;
|
|
1700
|
+
const [pmidRecords, doiRecords] = await Promise.all([
|
|
1701
|
+
pmids.length > 0 ? getNcbiService().idConvert(pmids, 'pmid', convertOptions) : [],
|
|
1702
|
+
dois.length > 0 ? getNcbiService().idConvert(dois, 'doi', convertOptions) : [],
|
|
1703
|
+
]);
|
|
1704
|
+
const seenPmids = new Set();
|
|
1705
|
+
for (const r of pmidRecords) {
|
|
1706
|
+
if (r.pmid === undefined)
|
|
1707
|
+
continue;
|
|
1708
|
+
const pmid = String(r.pmid);
|
|
1709
|
+
seenPmids.add(pmid);
|
|
1710
|
+
if (r.pmcid) {
|
|
1711
|
+
noteRecordDoi(String(r.pmcid), r.doi);
|
|
1712
|
+
routeToPmc(pmid, String(r.pmcid));
|
|
1465
1713
|
}
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
pmidFallbackCandidates.push({ pmid: requested });
|
|
1474
|
-
}
|
|
1714
|
+
else {
|
|
1715
|
+
chainByInput.get(pmid)?.push({
|
|
1716
|
+
tier: 'pmc',
|
|
1717
|
+
outcome: 'not-attempted',
|
|
1718
|
+
detail: 'PMID has no PMC counterpart',
|
|
1719
|
+
});
|
|
1720
|
+
pending.pmid.push({ idType: 'pmid', id: pmid, ...(r.doi && { doi: r.doi }) });
|
|
1475
1721
|
}
|
|
1476
1722
|
}
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1723
|
+
for (const requested of pmids) {
|
|
1724
|
+
if (!seenPmids.has(requested)) {
|
|
1725
|
+
chainByInput.get(requested)?.push({
|
|
1726
|
+
tier: 'pmc',
|
|
1727
|
+
outcome: 'not-attempted',
|
|
1728
|
+
detail: 'ID Converter returned no record for this PMID',
|
|
1729
|
+
});
|
|
1730
|
+
pending.pmid.push({ idType: 'pmid', id: requested });
|
|
1731
|
+
}
|
|
1483
1732
|
}
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
//
|
|
1487
|
-
//
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
for (const doi of input.dois) {
|
|
1496
|
-
const key = chainKeyByDoi.get(doi.toLowerCase()) ?? doi;
|
|
1497
|
-
chainKeyByDoi.set(doi.toLowerCase(), key);
|
|
1498
|
-
addCallerId(key, doi);
|
|
1733
|
+
const seenDois = new Set();
|
|
1734
|
+
for (const r of doiRecords) {
|
|
1735
|
+
// Match on the echoed `requested-id`, not `r.doi`: the record's own DOI
|
|
1736
|
+
// can be cased differently from anything the caller sent.
|
|
1737
|
+
const doi = chainKeyByDoi.get(String(r['requested-id']).toLowerCase());
|
|
1738
|
+
if (doi === undefined)
|
|
1739
|
+
continue;
|
|
1740
|
+
seenDois.add(doi);
|
|
1741
|
+
if (r.pmcid) {
|
|
1742
|
+
noteRecordDoi(String(r.pmcid), doi);
|
|
1743
|
+
routeToPmc(doi, String(r.pmcid));
|
|
1499
1744
|
}
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
// can be cased differently from anything the caller sent.
|
|
1508
|
-
const doi = chainKeyByDoi.get(String(r['requested-id']).toLowerCase());
|
|
1509
|
-
if (doi === undefined)
|
|
1510
|
-
continue;
|
|
1511
|
-
seen.add(doi);
|
|
1512
|
-
if (r.pmcid) {
|
|
1513
|
-
routeToPmc(doi, String(r.pmcid));
|
|
1514
|
-
}
|
|
1515
|
-
else {
|
|
1516
|
-
chainByInput.get(doi)?.push({
|
|
1517
|
-
tier: 'pmc',
|
|
1518
|
-
outcome: 'not-attempted',
|
|
1519
|
-
detail: 'DOI has no PMC counterpart',
|
|
1520
|
-
});
|
|
1521
|
-
doiCandidates.push({ doi });
|
|
1522
|
-
}
|
|
1745
|
+
else {
|
|
1746
|
+
chainByInput.get(doi)?.push({
|
|
1747
|
+
tier: 'pmc',
|
|
1748
|
+
outcome: 'not-attempted',
|
|
1749
|
+
detail: 'DOI has no PMC counterpart',
|
|
1750
|
+
});
|
|
1751
|
+
pending.doi.push({ idType: 'doi', id: doi, doi });
|
|
1523
1752
|
}
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
}
|
|
1753
|
+
}
|
|
1754
|
+
for (const requested of dois) {
|
|
1755
|
+
if (!seenDois.has(requested)) {
|
|
1756
|
+
chainByInput.get(requested)?.push({
|
|
1757
|
+
tier: 'pmc',
|
|
1758
|
+
outcome: 'not-attempted',
|
|
1759
|
+
detail: 'ID Converter returned no record for this DOI',
|
|
1760
|
+
});
|
|
1761
|
+
pending.doi.push({ idType: 'doi', id: requested, doi: requested });
|
|
1533
1762
|
}
|
|
1534
1763
|
}
|
|
1535
|
-
// Route PMC-missed prefixed PMCIDs
|
|
1536
|
-
//
|
|
1537
|
-
//
|
|
1538
|
-
//
|
|
1539
|
-
//
|
|
1764
|
+
// Route PMC-missed prefixed PMCIDs on to the remaining tiers, each as a
|
|
1765
|
+
// candidate of the field that owns its record — which decides how Europe
|
|
1766
|
+
// PMC searches for it and which id an Unpaywall article is stamped with. A
|
|
1767
|
+
// DOI the routing learned for the record rides along, sparing Unpaywall a
|
|
1768
|
+
// lookup; a DOI owner carries its own.
|
|
1540
1769
|
const routePmcMissesToFallback = (missingPrefixed) => {
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
const pmid = pmcidToInputId.get(prefixed);
|
|
1549
|
-
if (pmid)
|
|
1550
|
-
pmidFallbackCandidates.push(pmidContext.get(pmid) ?? { pmid });
|
|
1551
|
-
}
|
|
1552
|
-
}
|
|
1553
|
-
else if (input.dois) {
|
|
1554
|
-
for (const prefixed of missingPrefixed) {
|
|
1555
|
-
const doi = pmcidToInputId.get(prefixed);
|
|
1556
|
-
if (doi)
|
|
1557
|
-
doiCandidates.push({ doi });
|
|
1558
|
-
}
|
|
1770
|
+
for (const prefixed of missingPrefixed) {
|
|
1771
|
+
const owner = pmcidToInputId.get(prefixed) ?? prefixed;
|
|
1772
|
+
const idType = callerIds.get(owner)?.[0]?.idType;
|
|
1773
|
+
if (idType === undefined)
|
|
1774
|
+
continue;
|
|
1775
|
+
const doi = idType === 'doi' ? owner : doiByPmcRecord.get(prefixed);
|
|
1776
|
+
pending[idType].push({ idType, id: owner, ...(doi && { doi }) });
|
|
1559
1777
|
}
|
|
1560
1778
|
};
|
|
1561
1779
|
// ── Stage 1: PMC EFetch ─────────────────────────────────────────────────
|
|
@@ -1581,17 +1799,18 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1581
1799
|
const bodylessPmcIds = new Set();
|
|
1582
1800
|
const parsed = [];
|
|
1583
1801
|
for (const node of findAll(articleSet, 'article')) {
|
|
1584
|
-
const
|
|
1585
|
-
if (isBodylessArticle(
|
|
1586
|
-
if (
|
|
1587
|
-
bodylessPmcIds.add(
|
|
1802
|
+
const parsedArticle = parsePmcArticle(node);
|
|
1803
|
+
if (isBodylessArticle(parsedArticle)) {
|
|
1804
|
+
if (parsedArticle.pmcId)
|
|
1805
|
+
bodylessPmcIds.add(parsedArticle.pmcId);
|
|
1588
1806
|
continue;
|
|
1589
1807
|
}
|
|
1590
|
-
const
|
|
1591
|
-
|
|
1808
|
+
const before = withBackMatter(parsedArticle);
|
|
1809
|
+
const after = applyPmcFilters(before, filters);
|
|
1810
|
+
if (isSectionFilterMiss(before, after, filters.sections)) {
|
|
1592
1811
|
sectionFilterMisses.push(articleDisplayId(after));
|
|
1593
1812
|
}
|
|
1594
|
-
const budgeted = applyPmcBudget(after, budget);
|
|
1813
|
+
const budgeted = applyPmcBudget(after, budget, bodySectionCount(parsedArticle, filters));
|
|
1595
1814
|
omittedSections += budgeted.omittedSections;
|
|
1596
1815
|
if (budgeted.truncation) {
|
|
1597
1816
|
truncatedArticles.push({
|
|
@@ -1650,85 +1869,72 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1650
1869
|
routePmcMissesToFallback(allPrefixed);
|
|
1651
1870
|
}
|
|
1652
1871
|
}
|
|
1872
|
+
// The records PMC EFetch served, by PMCID, under the id that owns each: the
|
|
1873
|
+
// ID Converter can miss a PMC record a Europe PMC hit links, and that hit
|
|
1874
|
+
// joins the owner rather than fetching the article again. (#192)
|
|
1875
|
+
const servedByPmc = new Map();
|
|
1876
|
+
for (const a of pmcArticles) {
|
|
1877
|
+
if (a.pmcId)
|
|
1878
|
+
servedByPmc.set(a.pmcId, inputIdByArticle.get(a) ?? a.pmcId);
|
|
1879
|
+
}
|
|
1653
1880
|
// ── Stage 2: Europe PMC fullTextXML ─────────────────────────────────────
|
|
1881
|
+
// Each candidate's `id` is its own chain key from here on: a PMC miss is
|
|
1882
|
+
// routed under the id that owns the record.
|
|
1883
|
+
let candidates = [...pending.pmcid, ...pending.pmid, ...pending.doi];
|
|
1654
1884
|
const epmc = getEuropePmcService();
|
|
1655
1885
|
const epmcOutcomes = epmc
|
|
1656
|
-
? await runEpmcStage(epmc, {
|
|
1657
|
-
pmidFallbackCandidates,
|
|
1658
|
-
pmcidFallbackCandidates,
|
|
1659
|
-
doiCandidates,
|
|
1660
|
-
input,
|
|
1661
|
-
budget,
|
|
1662
|
-
ctx,
|
|
1663
|
-
})
|
|
1886
|
+
? await runEpmcStage(epmc, { candidates, servedByPmc, input: filters, budget, ctx })
|
|
1664
1887
|
: {
|
|
1665
1888
|
articles: [],
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
pmcidOutcomes: new Map(),
|
|
1671
|
-
doiOutcomes: new Map(),
|
|
1889
|
+
articleInputIds: new Map(),
|
|
1890
|
+
joins: [],
|
|
1891
|
+
outcomes: new Map(),
|
|
1892
|
+
remaining: candidates,
|
|
1672
1893
|
sectionFilterMisses: [],
|
|
1673
1894
|
truncatedArticles: [],
|
|
1674
1895
|
omittedSections: 0,
|
|
1675
|
-
articleInputIds: new Map(),
|
|
1676
1896
|
};
|
|
1677
1897
|
pmcArticles = pmcArticles.concat(epmcOutcomes.articles);
|
|
1678
1898
|
truncatedArticles.push(...epmcOutcomes.truncatedArticles);
|
|
1679
1899
|
omittedSections += epmcOutcomes.omittedSections;
|
|
1680
1900
|
for (const [article, candidateId] of epmcOutcomes.articleInputIds) {
|
|
1681
|
-
inputIdByArticle.set(article,
|
|
1901
|
+
inputIdByArticle.set(article, candidateId);
|
|
1682
1902
|
}
|
|
1683
1903
|
// Fold EPMC outcomes into each id's chain. EPMC-served articles count as
|
|
1684
1904
|
// recovered, so their ids are added to `recoveredIds` here.
|
|
1685
1905
|
if (!epmc) {
|
|
1686
|
-
const epmcDisabledEntry = {
|
|
1687
|
-
tier: 'europepmc',
|
|
1688
|
-
outcome: 'not-attempted',
|
|
1689
|
-
detail: 'EUROPEPMC_ENABLED=false',
|
|
1690
|
-
};
|
|
1691
1906
|
// EPMC searches by PMID, PMCID, and DOI alike, so it could have served
|
|
1692
1907
|
// every candidate that reached this stage — no applicability test.
|
|
1693
|
-
const
|
|
1694
|
-
chainByInput.get(
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
const prefixed = withPmcPrefix(c.pmcid);
|
|
1701
|
-
skipEpmc(pmcidToInputId.get(prefixed) ?? prefixed);
|
|
1908
|
+
for (const c of candidates) {
|
|
1909
|
+
chainByInput.get(c.id)?.push({
|
|
1910
|
+
tier: 'europepmc',
|
|
1911
|
+
outcome: 'not-attempted',
|
|
1912
|
+
detail: 'EUROPEPMC_ENABLED=false',
|
|
1913
|
+
});
|
|
1914
|
+
markUnqueried(c.id, 'europepmc');
|
|
1702
1915
|
}
|
|
1703
|
-
for (const c of doiCandidates)
|
|
1704
|
-
skipEpmc(c.doi);
|
|
1705
1916
|
}
|
|
1706
1917
|
else {
|
|
1707
|
-
|
|
1918
|
+
// A candidate whose search reached a record an earlier one already owns
|
|
1919
|
+
// joins that chain before the owner's outcome lands on it. (#192)
|
|
1920
|
+
for (const { id, owner } of epmcOutcomes.joins)
|
|
1921
|
+
joinChain(id, owner);
|
|
1922
|
+
for (const [inputId, outcome] of epmcOutcomes.outcomes) {
|
|
1708
1923
|
if (outcome.kind === 'hit') {
|
|
1709
1924
|
recoveredIds.add(inputId);
|
|
1710
|
-
|
|
1925
|
+
continue;
|
|
1711
1926
|
}
|
|
1712
1927
|
if (outcome.kind === 'no-body')
|
|
1713
1928
|
bodylessInputIds.add(inputId);
|
|
1714
1929
|
chainByInput.get(inputId)?.push(epmcTierFromOutcome(outcome));
|
|
1715
|
-
};
|
|
1716
|
-
for (const [pmid, outcome] of epmcOutcomes.pmidOutcomes)
|
|
1717
|
-
foldEpmcOutcome(pmid, outcome);
|
|
1718
|
-
for (const [prefixed, outcome] of epmcOutcomes.pmcidOutcomes) {
|
|
1719
|
-
foldEpmcOutcome(pmcidToInputId.get(prefixed) ?? prefixed, outcome);
|
|
1720
1930
|
}
|
|
1721
|
-
for (const [doi, outcome] of epmcOutcomes.doiOutcomes)
|
|
1722
|
-
foldEpmcOutcome(doi, outcome);
|
|
1723
1931
|
}
|
|
1724
|
-
|
|
1725
|
-
pmcidFallbackCandidates = epmcOutcomes.remainingPmcid;
|
|
1726
|
-
doiCandidates = epmcOutcomes.remainingDoi;
|
|
1932
|
+
candidates = epmcOutcomes.remaining;
|
|
1727
1933
|
sectionFilterMisses.push(...epmcOutcomes.sectionFilterMisses);
|
|
1728
1934
|
// ── Stage 3: Unpaywall fallback ─────────────────────────────────────────
|
|
1729
1935
|
const unpaywall = getUnpaywallService();
|
|
1730
1936
|
const fallbackArticles = [];
|
|
1731
|
-
// Detail of a DOI-backfill lookup that threw, per
|
|
1937
|
+
// Detail of a DOI-backfill lookup that threw, per field. A candidate still
|
|
1732
1938
|
// DOI-less after one of these is an unknown, not a settled absence: it
|
|
1733
1939
|
// reports `doi-lookup-failed` rather than `no-doi`. (#119)
|
|
1734
1940
|
let pmcidDoiLookupFailure;
|
|
@@ -1739,200 +1945,129 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1739
1945
|
outcome: failure ? 'doi-lookup-failed' : 'no-doi',
|
|
1740
1946
|
...(failure && { detail: failure }),
|
|
1741
1947
|
});
|
|
1742
|
-
//
|
|
1743
|
-
//
|
|
1744
|
-
//
|
|
1745
|
-
// Converter, which returns DOIs for
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
detail: 'UNPAYWALL_EMAIL is not set',
|
|
1759
|
-
});
|
|
1760
|
-
markUnqueried(inputId, 'unpaywall');
|
|
1761
|
-
}
|
|
1762
|
-
}
|
|
1763
|
-
else {
|
|
1764
|
-
const needDoi = pmcidFallbackCandidates
|
|
1765
|
-
.filter((c) => !c.doi)
|
|
1766
|
-
.map((c) => withPmcPrefix(c.pmcid));
|
|
1767
|
-
if (needDoi.length > 0) {
|
|
1768
|
-
try {
|
|
1769
|
-
const records = await getNcbiService().idConvert(needDoi, 'pmcid', ctx.signal ? { signal: ctx.signal } : undefined);
|
|
1770
|
-
const doiByPmcid = new Map();
|
|
1771
|
-
for (const r of records) {
|
|
1772
|
-
if (r.pmcid && r.doi) {
|
|
1773
|
-
doiByPmcid.set(withPmcPrefix(normalizePmcId(String(r.pmcid))), String(r.doi));
|
|
1774
|
-
}
|
|
1775
|
-
}
|
|
1776
|
-
pmcidFallbackCandidates = pmcidFallbackCandidates.map((c) => {
|
|
1777
|
-
if (c.doi)
|
|
1778
|
-
return c;
|
|
1779
|
-
const doi = doiByPmcid.get(withPmcPrefix(c.pmcid));
|
|
1780
|
-
return doi ? { ...c, doi } : c;
|
|
1781
|
-
});
|
|
1782
|
-
}
|
|
1783
|
-
catch (error) {
|
|
1784
|
-
if (ctx.signal.aborted)
|
|
1785
|
-
throw error;
|
|
1786
|
-
pmcidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1787
|
-
ctx.log.warning('Failed to resolve PMCID → DOI for the Unpaywall fallback', {
|
|
1788
|
-
error: pmcidDoiLookupFailure,
|
|
1789
|
-
pmcidCount: needDoi.length,
|
|
1790
|
-
});
|
|
1791
|
-
}
|
|
1792
|
-
}
|
|
1793
|
-
const outcomes = await Promise.all(pmcidFallbackCandidates.map(async (candidate) => {
|
|
1794
|
-
// The prefixed PMCID is the id `unavailable[]` keys on, so stamping
|
|
1795
|
-
// it on the article makes a partially-recovered batch report its
|
|
1796
|
-
// successes and its failures under the same identifier. (#92)
|
|
1797
|
-
const pmcId = withPmcPrefix(candidate.pmcid);
|
|
1798
|
-
return {
|
|
1799
|
-
pmcId,
|
|
1800
|
-
result: candidate.doi
|
|
1801
|
-
? await resolveUnpaywall({ pmcId, doi: candidate.doi, epmcTitle: candidate.title, budget }, unpaywall, ctx)
|
|
1802
|
-
: undefined,
|
|
1803
|
-
};
|
|
1804
|
-
}));
|
|
1805
|
-
for (const { pmcId, result } of outcomes) {
|
|
1806
|
-
const inputId = pmcidToInputId.get(pmcId) ?? pmcId;
|
|
1807
|
-
if (result === undefined) {
|
|
1808
|
-
chainByInput.get(inputId)?.push(doilessTierEntry(pmcidDoiLookupFailure));
|
|
1809
|
-
}
|
|
1810
|
-
else if ('article' in result) {
|
|
1811
|
-
fallbackArticles.push(result.article);
|
|
1812
|
-
inputIdByArticle.set(result.article, inputId);
|
|
1813
|
-
if (result.truncation)
|
|
1814
|
-
truncatedArticles.push(result.truncation);
|
|
1815
|
-
recoveredIds.add(inputId);
|
|
1816
|
-
}
|
|
1817
|
-
else {
|
|
1818
|
-
const u = result.unavailable;
|
|
1819
|
-
chainByInput.get(inputId)?.push({
|
|
1820
|
-
tier: 'unpaywall',
|
|
1821
|
-
outcome: unpaywallReasonToTierOutcome(u.reason),
|
|
1822
|
-
...(u.detail && { detail: u.detail }),
|
|
1823
|
-
});
|
|
1948
|
+
// Candidates reach Unpaywall on the DOI the chain already holds — a `dois`
|
|
1949
|
+
// entry's own, one the routing learned for a PMC record, or one a Europe
|
|
1950
|
+
// PMC hit carried, captured on non-hit outcomes too (#88). The rest are
|
|
1951
|
+
// backfilled: a PMC ID through the PMC ID Converter, which returns DOIs for
|
|
1952
|
+
// PMC-indexed records, and only when Unpaywall is configured; a PMID from
|
|
1953
|
+
// PubMed metadata (db=pubmed), since the converter returns DOIs only for
|
|
1954
|
+
// articles it has in PMC. The two lookups are independent.
|
|
1955
|
+
const backfillPmcidDois = async (pmcids) => {
|
|
1956
|
+
const found = new Map();
|
|
1957
|
+
if (pmcids.length === 0)
|
|
1958
|
+
return found;
|
|
1959
|
+
try {
|
|
1960
|
+
const records = await getNcbiService().idConvert(pmcids, 'pmcid', ctx.signal ? { signal: ctx.signal } : undefined);
|
|
1961
|
+
for (const r of records) {
|
|
1962
|
+
if (r.pmcid && r.doi) {
|
|
1963
|
+
found.set(withPmcPrefix(normalizePmcId(String(r.pmcid))), String(r.doi));
|
|
1824
1964
|
}
|
|
1825
1965
|
}
|
|
1826
1966
|
}
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
const doiMap = await fetchPubmedDois(needDoi, ctx.signal);
|
|
1836
|
-
pmidFallbackCandidates = pmidFallbackCandidates.map((c) => {
|
|
1837
|
-
if (c.doi)
|
|
1838
|
-
return c;
|
|
1839
|
-
const doi = doiMap.get(c.pmid);
|
|
1840
|
-
return doi ? { ...c, doi } : c;
|
|
1841
|
-
});
|
|
1842
|
-
}
|
|
1843
|
-
catch (error) {
|
|
1844
|
-
if (ctx.signal.aborted)
|
|
1845
|
-
throw error;
|
|
1846
|
-
pmidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1847
|
-
ctx.log.warning('Failed to batch-fetch DOIs from PubMed for Unpaywall fallback', {
|
|
1848
|
-
error: pmidDoiLookupFailure,
|
|
1849
|
-
pmidCount: needDoi.length,
|
|
1850
|
-
});
|
|
1851
|
-
}
|
|
1967
|
+
catch (error) {
|
|
1968
|
+
if (ctx.signal.aborted)
|
|
1969
|
+
throw error;
|
|
1970
|
+
pmcidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1971
|
+
ctx.log.warning('Failed to resolve PMCID → DOI for the Unpaywall fallback', {
|
|
1972
|
+
error: pmcidDoiLookupFailure,
|
|
1973
|
+
pmcidCount: pmcids.length,
|
|
1974
|
+
});
|
|
1852
1975
|
}
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
chainByInput.get(c.pmid)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
1861
|
-
continue;
|
|
1862
|
-
}
|
|
1863
|
-
chainByInput.get(c.pmid)?.push({
|
|
1864
|
-
tier: 'unpaywall',
|
|
1865
|
-
outcome: 'not-attempted',
|
|
1866
|
-
detail: 'UNPAYWALL_EMAIL is not set',
|
|
1867
|
-
});
|
|
1868
|
-
markUnqueried(c.pmid, 'unpaywall');
|
|
1869
|
-
}
|
|
1976
|
+
return found;
|
|
1977
|
+
};
|
|
1978
|
+
const backfillPmidDois = async (pmidsNeedingDoi) => {
|
|
1979
|
+
if (pmidsNeedingDoi.length === 0)
|
|
1980
|
+
return new Map();
|
|
1981
|
+
try {
|
|
1982
|
+
return await fetchPubmedDois(pmidsNeedingDoi, ctx.signal);
|
|
1870
1983
|
}
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1984
|
+
catch (error) {
|
|
1985
|
+
if (ctx.signal.aborted)
|
|
1986
|
+
throw error;
|
|
1987
|
+
pmidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1988
|
+
ctx.log.warning('Failed to batch-fetch DOIs from PubMed for Unpaywall fallback', {
|
|
1989
|
+
error: pmidDoiLookupFailure,
|
|
1990
|
+
pmidCount: pmidsNeedingDoi.length,
|
|
1991
|
+
});
|
|
1992
|
+
return new Map();
|
|
1993
|
+
}
|
|
1994
|
+
};
|
|
1995
|
+
const needingDoi = (idType) => candidates.filter((c) => c.idType === idType && !c.doi).map((c) => c.id);
|
|
1996
|
+
const [pmcidDois, pmidDois] = await Promise.all([
|
|
1997
|
+
unpaywall ? backfillPmcidDois(needingDoi('pmcid')) : new Map(),
|
|
1998
|
+
backfillPmidDois(needingDoi('pmid')),
|
|
1999
|
+
]);
|
|
2000
|
+
const backfilled = { pmcid: pmcidDois, pmid: pmidDois, doi: new Map() };
|
|
2001
|
+
candidates = candidates.map((c) => {
|
|
2002
|
+
const doi = c.doi ?? backfilled[c.idType].get(c.id);
|
|
2003
|
+
return doi ? { ...c, doi } : c;
|
|
2004
|
+
});
|
|
2005
|
+
if (!unpaywall) {
|
|
2006
|
+
for (const c of candidates) {
|
|
2007
|
+
// `fetchPubmedDois` has already run, so a PMID with no DOI could not
|
|
2008
|
+
// have reached Unpaywall configured or not — that is `no-doi`, a real
|
|
2009
|
+
// answer, not an incomplete search, unless the lookup itself threw. A
|
|
2010
|
+
// PMC ID's DOI is looked up only for Unpaywall, so its absence means
|
|
2011
|
+
// "never looked up", and the tier stays a genuine unknown and is marked.
|
|
2012
|
+
if (!c.doi && c.idType === 'pmid') {
|
|
2013
|
+
chainByInput.get(c.id)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
2014
|
+
continue;
|
|
1897
2015
|
}
|
|
2016
|
+
chainByInput.get(c.id)?.push({
|
|
2017
|
+
tier: 'unpaywall',
|
|
2018
|
+
outcome: 'not-attempted',
|
|
2019
|
+
detail: 'UNPAYWALL_EMAIL is not set',
|
|
2020
|
+
});
|
|
2021
|
+
markUnqueried(c.id, 'unpaywall');
|
|
1898
2022
|
}
|
|
1899
2023
|
}
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
markUnqueried(c.doi, 'unpaywall');
|
|
2024
|
+
else {
|
|
2025
|
+
// One resolve per DOI, case-insensitively: a later candidate carrying a
|
|
2026
|
+
// DOI an earlier one holds joins that chain, and the article is stamped
|
|
2027
|
+
// with the owner's identifier. (#192)
|
|
2028
|
+
const ownerByDoi = new Map();
|
|
2029
|
+
for (const c of candidates) {
|
|
2030
|
+
if (!c.doi) {
|
|
2031
|
+
const failure = c.idType === 'pmcid' ? pmcidDoiLookupFailure : pmidDoiLookupFailure;
|
|
2032
|
+
chainByInput.get(c.id)?.push(doilessTierEntry(failure));
|
|
2033
|
+
continue;
|
|
1911
2034
|
}
|
|
2035
|
+
const owner = ownerByDoi.get(c.doi.toLowerCase());
|
|
2036
|
+
if (owner)
|
|
2037
|
+
joinChain(c.id, owner.id);
|
|
2038
|
+
else
|
|
2039
|
+
ownerByDoi.set(c.doi.toLowerCase(), { ...c, doi: c.doi });
|
|
1912
2040
|
}
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
2041
|
+
// `resolveUnpaywall` catches its own failures so this Promise.all
|
|
2042
|
+
// doesn't reject under normal operation.
|
|
2043
|
+
const outcomes = await Promise.all([...ownerByDoi.values()].map(async (c) => ({
|
|
2044
|
+
inputId: c.id,
|
|
2045
|
+
// A PMC ID or PMID is stamped on the article so a partially-recovered
|
|
2046
|
+
// batch reports its successes and its failures under the same
|
|
2047
|
+
// identifier; Unpaywall itself only knows the DOI. (#92)
|
|
2048
|
+
result: await resolveUnpaywall({
|
|
2049
|
+
...(c.idType === 'pmcid' && { pmcId: c.id }),
|
|
2050
|
+
...(c.idType === 'pmid' && { pmid: c.id }),
|
|
1917
2051
|
doi: c.doi,
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
2052
|
+
epmcTitle: c.title,
|
|
2053
|
+
budget,
|
|
2054
|
+
}, unpaywall, ctx),
|
|
2055
|
+
})));
|
|
2056
|
+
for (const { inputId, result } of outcomes) {
|
|
2057
|
+
if ('article' in result) {
|
|
2058
|
+
fallbackArticles.push(result.article);
|
|
2059
|
+
inputIdByArticle.set(result.article, inputId);
|
|
2060
|
+
if (result.truncation)
|
|
2061
|
+
truncatedArticles.push(result.truncation);
|
|
2062
|
+
recoveredIds.add(inputId);
|
|
2063
|
+
}
|
|
2064
|
+
else {
|
|
2065
|
+
const u = result.unavailable;
|
|
2066
|
+
chainByInput.get(inputId)?.push({
|
|
2067
|
+
tier: 'unpaywall',
|
|
2068
|
+
outcome: unpaywallReasonToTierOutcome(u.reason),
|
|
2069
|
+
...(u.detail && { detail: u.detail }),
|
|
2070
|
+
});
|
|
1936
2071
|
}
|
|
1937
2072
|
}
|
|
1938
2073
|
}
|
|
@@ -1942,12 +2077,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1942
2077
|
if (recoveredIds.has(id))
|
|
1943
2078
|
continue;
|
|
1944
2079
|
const unqueried = unqueriedByInput.get(id);
|
|
1945
|
-
for (const callerId of callerIds.get(id) ?? [
|
|
2080
|
+
for (const callerId of callerIds.get(id) ?? []) {
|
|
2081
|
+
const own = ownChainOf.get(callerId);
|
|
2082
|
+
const triedTiers = own ? [...own.tried, ...chain.slice(own.from)] : chain;
|
|
1946
2083
|
unavailable.push({
|
|
1947
|
-
id: callerId,
|
|
1948
|
-
idType,
|
|
1949
|
-
reason: reasonFromChain(
|
|
1950
|
-
triedTiers
|
|
2084
|
+
id: callerId.id,
|
|
2085
|
+
idType: callerId.idType,
|
|
2086
|
+
reason: reasonFromChain(triedTiers),
|
|
2087
|
+
triedTiers,
|
|
1951
2088
|
...(unqueried?.size && { unqueriedTiers: [...unqueried] }),
|
|
1952
2089
|
});
|
|
1953
2090
|
}
|
|
@@ -1962,21 +2099,28 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1962
2099
|
const fit = ceiling === undefined ? undefined : fitWholeItems(resolved, ceiling);
|
|
1963
2100
|
const articles = fit?.kept ?? resolved;
|
|
1964
2101
|
const nextDeferredCharacters = fit?.nextDeferredCharacters;
|
|
2102
|
+
// Each deferred article is handed back under the first spelling of the id
|
|
2103
|
+
// that owns it, in the field that id was sent in. Every recovery site
|
|
2104
|
+
// records the owner; a PMC record that came back without a PMC ID has none,
|
|
2105
|
+
// and is reported under the id it carries — the total-function fallback,
|
|
2106
|
+
// not an expected path.
|
|
2107
|
+
const deferredIds = (fit?.deferred ?? []).map((a) => {
|
|
2108
|
+
const key = inputIdByArticle.get(a);
|
|
2109
|
+
const owner = key === undefined ? undefined : callerIds.get(key)?.[0];
|
|
2110
|
+
return (owner ?? { id: articleDisplayId(a), idType: a.pmcId ? 'pmcid' : a.pmid ? 'pmid' : 'doi' });
|
|
2111
|
+
});
|
|
1965
2112
|
const deferred = ceiling !== undefined && nextDeferredCharacters !== undefined && fit
|
|
1966
2113
|
? {
|
|
1967
2114
|
maxResponseCharacters: ceiling,
|
|
1968
2115
|
returnedCharacters: fit.keptCharacters,
|
|
1969
2116
|
deferredCount: fit.deferred.length,
|
|
1970
|
-
|
|
1971
|
-
// Every recovery site records the input id; `articleDisplayId` is
|
|
1972
|
-
// the total-function fallback, not an expected path.
|
|
1973
|
-
ids: fit.deferred.map((a) => {
|
|
1974
|
-
const id = inputIdByArticle.get(a) ?? articleDisplayId(a);
|
|
1975
|
-
return callerIds.get(id)?.[0] ?? id;
|
|
1976
|
-
}),
|
|
2117
|
+
...groupIdsByField(deferredIds),
|
|
1977
2118
|
nextDeferredCharacters,
|
|
1978
2119
|
}
|
|
1979
2120
|
: undefined;
|
|
2121
|
+
const overLimit = overLimitIds.length > 0
|
|
2122
|
+
? { limit: FETCH_LIMIT, ...groupIdsByField(overLimitIds) }
|
|
2123
|
+
: undefined;
|
|
1980
2124
|
// A deferred article takes its body accounting out of the response with it —
|
|
1981
2125
|
// those counts describe text the caller never received.
|
|
1982
2126
|
if (fit) {
|
|
@@ -1992,13 +2136,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1992
2136
|
}
|
|
1993
2137
|
}
|
|
1994
2138
|
ctx.log.info('pubmed_fetch_fulltext completed', {
|
|
1995
|
-
requested:
|
|
2139
|
+
requested: idCount,
|
|
1996
2140
|
returned: articles.length,
|
|
1997
2141
|
pmcHits: pmcArticles.filter((a) => a.viaSource === 'pmc').length,
|
|
1998
2142
|
epmcHits: pmcArticles.filter((a) => a.viaSource === 'europepmc').length,
|
|
1999
2143
|
unpaywallHits: fallbackArticles.length,
|
|
2000
2144
|
unavailable: unavailable.length,
|
|
2001
2145
|
...(deferred && { deferred: deferred.deferredCount }),
|
|
2146
|
+
...(overLimit && { overLimit: overLimit.ids.length }),
|
|
2002
2147
|
});
|
|
2003
2148
|
// Summed off the per-article entries rather than carried through every
|
|
2004
2149
|
// stage, so a deferred article's dropped tables leave the roll-up together
|
|
@@ -2026,8 +2171,8 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2026
2171
|
// Only the last ctx.enrich.notice survives, so the applicable fragments are
|
|
2027
2172
|
// collected and emitted once.
|
|
2028
2173
|
const notices = [];
|
|
2029
|
-
if (
|
|
2030
|
-
notices.push(buildSectionFilterMissNotice(sectionFilterMisses,
|
|
2174
|
+
if (sectionFilterMisses.length > 0) {
|
|
2175
|
+
notices.push(buildSectionFilterMissNotice(sectionFilterMisses, sectionFilter.terms));
|
|
2031
2176
|
}
|
|
2032
2177
|
const unrecoveredBodyless = [...bodylessInputIds].filter((id) => !recoveredIds.has(id));
|
|
2033
2178
|
if (unrecoveredBodyless.length > 0)
|
|
@@ -2044,6 +2189,10 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2044
2189
|
notices.push(buildDeferralNotice(deferred));
|
|
2045
2190
|
ctx.enrich({ truncated: true });
|
|
2046
2191
|
}
|
|
2192
|
+
if (overLimit) {
|
|
2193
|
+
notices.push(buildOverLimitNotice(overLimit));
|
|
2194
|
+
ctx.enrich({ truncated: true });
|
|
2195
|
+
}
|
|
2047
2196
|
if (notices.length > 0)
|
|
2048
2197
|
ctx.enrich.notice(notices.join(' '));
|
|
2049
2198
|
return {
|
|
@@ -2052,6 +2201,7 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2052
2201
|
...(unavailable.length > 0 && { unavailable }),
|
|
2053
2202
|
...(truncation && { truncation }),
|
|
2054
2203
|
...(deferred && { deferred }),
|
|
2204
|
+
...(overLimit && { overLimit }),
|
|
2055
2205
|
};
|
|
2056
2206
|
},
|
|
2057
2207
|
format: (result) => {
|
|
@@ -2082,7 +2232,11 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2082
2232
|
}
|
|
2083
2233
|
if (result.deferred) {
|
|
2084
2234
|
const d = result.deferred;
|
|
2085
|
-
lines.push(`\n**Deferred by the response budget:** ${d.deferredCount} article(s) — ${d.returnedCharacters} of ${d.maxResponseCharacters} budgeted characters returned; next deferred article ${d.nextDeferredCharacters} characters`,
|
|
2235
|
+
lines.push(`\n**Deferred by the response budget:** ${d.deferredCount} article(s) — ${d.returnedCharacters} of ${d.maxResponseCharacters} budgeted characters returned; next deferred article ${d.nextDeferredCharacters} characters`, ...resendLines(d));
|
|
2236
|
+
}
|
|
2237
|
+
if (result.overLimit) {
|
|
2238
|
+
const o = result.overLimit;
|
|
2239
|
+
lines.push(`\n**Over the per-call limit:** ${o.ids.length} identifier(s) not fetched — a call fetches the first ${o.limit} distinct identifiers`, ...resendLines(o));
|
|
2086
2240
|
}
|
|
2087
2241
|
// An empty response under a budget is a deferral, not an absence — the
|
|
2088
2242
|
// articles resolved and the ids above retrieve them.
|
|
@@ -2103,9 +2257,84 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
2103
2257
|
return [{ type: 'text', text: lines.join('\n') }];
|
|
2104
2258
|
},
|
|
2105
2259
|
});
|
|
2260
|
+
/**
|
|
2261
|
+
* Every requested id in routing order — `pmcids`, then `pmids`, then `dois`,
|
|
2262
|
+
* request order within each — as sent. Every spelling one field accepts for an
|
|
2263
|
+
* identifier shares a key, and the same digits in two fields are two keys.
|
|
2264
|
+
* (#192, #222)
|
|
2265
|
+
*/
|
|
2266
|
+
function requestedIds(fields) {
|
|
2267
|
+
return [
|
|
2268
|
+
...(fields.pmcids ?? []).map((id) => ({ id, idType: 'pmcid', key: `pmcid:${normalizePmcId(id)}` })),
|
|
2269
|
+
...(fields.pmids ?? []).map((id) => ({ id, idType: 'pmid', key: `pmid:${normalizePmid(id)}` })),
|
|
2270
|
+
...(fields.dois ?? []).map((id) => ({ id, idType: 'doi', key: `doi:${id.toLowerCase()}` })),
|
|
2271
|
+
];
|
|
2272
|
+
}
|
|
2273
|
+
/**
|
|
2274
|
+
* Split the requested ids at the per-call fetch limit, before any upstream
|
|
2275
|
+
* request. The first {@link FETCH_LIMIT} distinct identifiers in routing order
|
|
2276
|
+
* are fetched, each with every spelling that names it; each later identifier is
|
|
2277
|
+
* handed back once, under its first spelling — a PMC ID in the `PMC<digits>`
|
|
2278
|
+
* form every output reports it in. (#222)
|
|
2279
|
+
*/
|
|
2280
|
+
function splitAtFetchLimit(fields) {
|
|
2281
|
+
const fetched = { pmcids: [], pmids: [], dois: [] };
|
|
2282
|
+
const overLimit = [];
|
|
2283
|
+
const admitted = new Set();
|
|
2284
|
+
const listed = new Set();
|
|
2285
|
+
for (const { id, idType, key } of requestedIds(fields)) {
|
|
2286
|
+
if (admitted.has(key) || admitted.size < FETCH_LIMIT) {
|
|
2287
|
+
admitted.add(key);
|
|
2288
|
+
fetched[`${idType}s`].push(id);
|
|
2289
|
+
}
|
|
2290
|
+
else if (!listed.has(key)) {
|
|
2291
|
+
listed.add(key);
|
|
2292
|
+
overLimit.push({ id: idType === 'pmcid' ? withPmcPrefix(normalizePmcId(id)) : id, idType });
|
|
2293
|
+
}
|
|
2294
|
+
}
|
|
2295
|
+
return { fetched, overLimit };
|
|
2296
|
+
}
|
|
2297
|
+
/**
|
|
2298
|
+
* Group ids handed back for a follow-up call: one field names them all —
|
|
2299
|
+
* always so for a single-field call — or each field's list says where to
|
|
2300
|
+
* resend its ids. (#192, #222)
|
|
2301
|
+
*/
|
|
2302
|
+
function groupIdsByField(callerIds) {
|
|
2303
|
+
const fields = new Set(callerIds.map((c) => c.idType));
|
|
2304
|
+
const [onlyField] = fields;
|
|
2305
|
+
const under = (idType) => callerIds.filter((c) => c.idType === idType).map((c) => c.id);
|
|
2306
|
+
return {
|
|
2307
|
+
...(fields.size === 1 && onlyField && { idType: onlyField }),
|
|
2308
|
+
ids: callerIds.map((c) => c.id),
|
|
2309
|
+
...(fields.size > 1 && {
|
|
2310
|
+
...(fields.has('pmcid') && { pmcids: under('pmcid') }),
|
|
2311
|
+
...(fields.has('pmid') && { pmids: under('pmid') }),
|
|
2312
|
+
...(fields.has('doi') && { dois: under('doi') }),
|
|
2313
|
+
}),
|
|
2314
|
+
};
|
|
2315
|
+
}
|
|
2316
|
+
/**
|
|
2317
|
+
* The `content[]` lines that resend a group of ids. Rendered on presence, each
|
|
2318
|
+
* independently: a group carries `idType` or the per-field lists, never both.
|
|
2319
|
+
*/
|
|
2320
|
+
function resendLines(group) {
|
|
2321
|
+
const fieldLists = idFieldLists(group);
|
|
2322
|
+
return [
|
|
2323
|
+
...(group.idType
|
|
2324
|
+
? [
|
|
2325
|
+
`Re-call \`pubmed_fetch_fulltext\` with these ${group.idType} ids as \`${group.idType}s\`: ${group.ids.join(', ')}`,
|
|
2326
|
+
]
|
|
2327
|
+
: []),
|
|
2328
|
+
...(fieldLists
|
|
2329
|
+
? [
|
|
2330
|
+
`Re-call \`pubmed_fetch_fulltext\` with these ids, each under its own field — ${fieldLists}`,
|
|
2331
|
+
]
|
|
2332
|
+
: []),
|
|
2333
|
+
];
|
|
2334
|
+
}
|
|
2106
2335
|
/**
|
|
2107
2336
|
* Merge what a Europe PMC hit carried onto a candidate bound for Unpaywall. A
|
|
2108
|
-
* DOI the candidate already holds wins — for `dois`
|
|
2337
|
+
* DOI the candidate already holds wins — for a `dois` entry it is the caller's
|
|
2109
2338
|
* own identifier.
|
|
2110
2339
|
*/
|
|
2111
2340
|
function carryEpmcHit(candidate, hit) {
|
|
@@ -2121,18 +2350,63 @@ function nonEmptyDisplayText(raw) {
|
|
|
2121
2350
|
}
|
|
2122
2351
|
/**
|
|
2123
2352
|
* Run the Europe PMC step against everything that fell through PMC EFetch
|
|
2124
|
-
* plus any
|
|
2125
|
-
*
|
|
2126
|
-
*
|
|
2353
|
+
* plus any DOI the converter could not place. Each candidate is searched by its
|
|
2354
|
+
* own id; each record the searches reach is then fetched as fullTextXML once,
|
|
2355
|
+
* for the first candidate in field order that reached it, and any later one
|
|
2356
|
+
* joins that candidate's chain. Hits become `source: 'pmc'` articles with
|
|
2357
|
+
* `viaSource: 'europepmc'`; misses flow through to the Unpaywall stage
|
|
2358
|
+
* unchanged.
|
|
2127
2359
|
*
|
|
2128
|
-
*
|
|
2129
|
-
* stays polite without serializing. Errors are caught and
|
|
2130
|
-
* helpers; a transient EPMC failure must not block the
|
|
2131
|
-
* fallback.
|
|
2360
|
+
* Searches and fetches run in parallel — the EPMC request queue caps
|
|
2361
|
+
* concurrency so this stays polite without serializing. Errors are caught and
|
|
2362
|
+
* logged inside the helpers; a transient EPMC failure must not block the
|
|
2363
|
+
* downstream Unpaywall fallback.
|
|
2132
2364
|
*/
|
|
2133
2365
|
async function runEpmcStage(epmc, args) {
|
|
2134
|
-
|
|
2135
|
-
|
|
2366
|
+
/**
|
|
2367
|
+
* Query shapes are load-bearing and not interchangeable with their quoted
|
|
2368
|
+
* variants. Europe PMC matches zero records for `EXT_ID:"<pmid>" AND SRC:MED`
|
|
2369
|
+
* and `PMCID:"PMC<digits>"` — the quotes only survive as long as no `AND SRC:`
|
|
2370
|
+
* clause follows. `SRC:PMC` is likewise wrong for a PMCID lookup: EPMC's
|
|
2371
|
+
* canonical record for a PMC-indexed article has `source: MED` and carries the
|
|
2372
|
+
* PMCID as a field, so the filter excludes the very record being sought. DOIs
|
|
2373
|
+
* keep their quotes — they carry slashes and dots that need them. (#85)
|
|
2374
|
+
*/
|
|
2375
|
+
const queryFor = (c) => {
|
|
2376
|
+
switch (c.idType) {
|
|
2377
|
+
case 'pmcid':
|
|
2378
|
+
return `PMCID:${c.id}`;
|
|
2379
|
+
case 'pmid':
|
|
2380
|
+
return `EXT_ID:${c.id} AND SRC:MED`;
|
|
2381
|
+
case 'doi':
|
|
2382
|
+
return `DOI:"${c.id}"`;
|
|
2383
|
+
}
|
|
2384
|
+
};
|
|
2385
|
+
const searched = await Promise.all(args.candidates.map(async (c) => ({
|
|
2386
|
+
c,
|
|
2387
|
+
search: await searchEpmcSafe(epmc, queryFor(c), args.ctx),
|
|
2388
|
+
})));
|
|
2389
|
+
// Every tier keys on the record: the first candidate whose search reached a
|
|
2390
|
+
// record owns it, and a later one joins its chain rather than fetching and
|
|
2391
|
+
// returning the same article again — as does one whose hit names a PMC ID
|
|
2392
|
+
// PMC EFetch already served, joining that article's owner. (#192)
|
|
2393
|
+
const joins = [];
|
|
2394
|
+
const ownerByRecord = new Map();
|
|
2395
|
+
const owned = searched.filter(({ c, search }) => {
|
|
2396
|
+
if (search.kind !== 'hit')
|
|
2397
|
+
return true;
|
|
2398
|
+
const record = `${search.hit.source}:${search.hit.id}`;
|
|
2399
|
+
const pmcid = search.hit.pmcid ?? (search.hit.source === 'PMC' ? search.hit.id : undefined);
|
|
2400
|
+
const servedOwner = pmcid === undefined ? undefined : args.servedByPmc.get(withPmcPrefix(normalizePmcId(pmcid)));
|
|
2401
|
+
const owner = servedOwner ?? ownerByRecord.get(record);
|
|
2402
|
+
if (owner === undefined) {
|
|
2403
|
+
ownerByRecord.set(record, c.id);
|
|
2404
|
+
return true;
|
|
2405
|
+
}
|
|
2406
|
+
joins.push({ id: c.id, owner });
|
|
2407
|
+
return false;
|
|
2408
|
+
});
|
|
2409
|
+
const runOne = async (c, search) => {
|
|
2136
2410
|
if (search.kind === 'error') {
|
|
2137
2411
|
return { c, outcome: { kind: 'service-error', detail: search.detail } };
|
|
2138
2412
|
}
|
|
@@ -2143,6 +2417,7 @@ async function runEpmcStage(epmc, args) {
|
|
|
2143
2417
|
...(search.hit.doi && { doi: search.hit.doi }),
|
|
2144
2418
|
...(title && { title }),
|
|
2145
2419
|
};
|
|
2420
|
+
const contextPmid = c.idType === 'pmid' ? c.id : undefined;
|
|
2146
2421
|
const fetched = await fetchEpmcArticle(epmc, search.hit, args, contextPmid);
|
|
2147
2422
|
if (fetched.kind === 'error') {
|
|
2148
2423
|
return { c, ...hit, outcome: { kind: 'service-error', detail: fetched.detail } };
|
|
@@ -2167,34 +2442,11 @@ async function runEpmcStage(epmc, args) {
|
|
|
2167
2442
|
...(fetched.truncation && { truncation: fetched.truncation }),
|
|
2168
2443
|
};
|
|
2169
2444
|
};
|
|
2170
|
-
|
|
2171
|
-
* Query shapes are load-bearing and not interchangeable with their quoted
|
|
2172
|
-
* variants. Europe PMC matches zero records for `EXT_ID:"<pmid>" AND SRC:MED`
|
|
2173
|
-
* and `PMCID:"PMC<digits>"` — the quotes only survive as long as no `AND SRC:`
|
|
2174
|
-
* clause follows. `SRC:PMC` is likewise wrong for a PMCID lookup: EPMC's
|
|
2175
|
-
* canonical record for a PMC-indexed article has `source: MED` and carries the
|
|
2176
|
-
* PMCID as a field, so the filter excludes the very record being sought. DOIs
|
|
2177
|
-
* keep their quotes — they carry slashes and dots that need them. (#85)
|
|
2178
|
-
*/
|
|
2179
|
-
const fetchForPmid = (c) => runOne(c, `EXT_ID:${c.pmid} AND SRC:MED`, c.pmid);
|
|
2180
|
-
const fetchForPmcid = (c) => {
|
|
2181
|
-
const normalized = withPmcPrefix(c.pmcid);
|
|
2182
|
-
return runOne({ c, normalized }, `PMCID:${normalized}`, undefined);
|
|
2183
|
-
};
|
|
2184
|
-
const fetchForDoi = (c) => runOne(c, `DOI:"${c.doi}"`, undefined);
|
|
2185
|
-
const [pmidResults, pmcidResults, doiResults] = await Promise.all([
|
|
2186
|
-
Promise.all(args.pmidFallbackCandidates.map(fetchForPmid)),
|
|
2187
|
-
Promise.all(args.pmcidFallbackCandidates.map(fetchForPmcid)),
|
|
2188
|
-
Promise.all(args.doiCandidates.map(fetchForDoi)),
|
|
2189
|
-
]);
|
|
2445
|
+
const runs = await Promise.all(owned.map(({ c, search }) => runOne(c, search)));
|
|
2190
2446
|
const articles = [];
|
|
2191
2447
|
const articleInputIds = new Map();
|
|
2192
|
-
const
|
|
2193
|
-
const
|
|
2194
|
-
const remainingDoi = [];
|
|
2195
|
-
const pmidOutcomes = new Map();
|
|
2196
|
-
const pmcidOutcomes = new Map();
|
|
2197
|
-
const doiOutcomes = new Map();
|
|
2448
|
+
const remaining = [];
|
|
2449
|
+
const outcomes = new Map();
|
|
2198
2450
|
const sectionFilterMisses = [];
|
|
2199
2451
|
const truncatedArticles = [];
|
|
2200
2452
|
let omittedSections = 0;
|
|
@@ -2207,39 +2459,22 @@ async function runEpmcStage(epmc, args) {
|
|
|
2207
2459
|
truncatedArticles.push(run.truncation);
|
|
2208
2460
|
omittedSections += run.omittedSections ?? 0;
|
|
2209
2461
|
};
|
|
2210
|
-
for (const run of
|
|
2211
|
-
|
|
2462
|
+
for (const run of runs) {
|
|
2463
|
+
outcomes.set(run.c.id, run.outcome);
|
|
2212
2464
|
if (run.article)
|
|
2213
|
-
collectHit(run.c.
|
|
2465
|
+
collectHit(run.c.id, { ...run, article: run.article });
|
|
2214
2466
|
// What the EPMC hit carried is evidence the next stage needs, whatever the
|
|
2215
|
-
// fetch outcome was: its DOI spares Unpaywall a
|
|
2216
|
-
//
|
|
2467
|
+
// fetch outcome was: its DOI spares Unpaywall a DOI lookup (#88, #119), and
|
|
2468
|
+
// its title backs up Unpaywall's own record (#144).
|
|
2217
2469
|
else
|
|
2218
|
-
|
|
2219
|
-
}
|
|
2220
|
-
for (const run of pmcidResults) {
|
|
2221
|
-
pmcidOutcomes.set(run.c.normalized, run.outcome);
|
|
2222
|
-
if (run.article)
|
|
2223
|
-
collectHit(run.c.normalized, { ...run, article: run.article });
|
|
2224
|
-
else
|
|
2225
|
-
remainingPmcid.push(carryEpmcHit(run.c.c, run));
|
|
2226
|
-
}
|
|
2227
|
-
for (const run of doiResults) {
|
|
2228
|
-
doiOutcomes.set(run.c.doi, run.outcome);
|
|
2229
|
-
if (run.article)
|
|
2230
|
-
collectHit(run.c.doi, { ...run, article: run.article });
|
|
2231
|
-
else
|
|
2232
|
-
remainingDoi.push(carryEpmcHit(run.c, run));
|
|
2470
|
+
remaining.push(carryEpmcHit(run.c, run));
|
|
2233
2471
|
}
|
|
2234
2472
|
return {
|
|
2235
2473
|
articles,
|
|
2236
2474
|
articleInputIds,
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
pmidOutcomes,
|
|
2241
|
-
pmcidOutcomes,
|
|
2242
|
-
doiOutcomes,
|
|
2475
|
+
joins,
|
|
2476
|
+
outcomes,
|
|
2477
|
+
remaining,
|
|
2243
2478
|
sectionFilterMisses,
|
|
2244
2479
|
truncatedArticles,
|
|
2245
2480
|
omittedSections,
|
|
@@ -2297,16 +2532,17 @@ async function fetchEpmcArticle(epmc, hit, args, contextPmid) {
|
|
|
2297
2532
|
if (!articleNode) {
|
|
2298
2533
|
return { kind: 'no-fulltext', detail: 'EPMC fullTextXML payload had no <article> element' };
|
|
2299
2534
|
}
|
|
2300
|
-
const
|
|
2301
|
-
if (isBodylessArticle(
|
|
2535
|
+
const parsedArticle = parsePmcArticle(articleNode);
|
|
2536
|
+
if (isBodylessArticle(parsedArticle)) {
|
|
2302
2537
|
return {
|
|
2303
2538
|
kind: 'no-body',
|
|
2304
2539
|
detail: 'EPMC fullTextXML carried front matter and abstract only, with no body sections',
|
|
2305
2540
|
};
|
|
2306
2541
|
}
|
|
2542
|
+
const beforeFilter = withBackMatter(parsedArticle);
|
|
2307
2543
|
const parsed = applyPmcFilters(beforeFilter, args.input);
|
|
2308
2544
|
const sectionFilterMiss = isSectionFilterMiss(beforeFilter, parsed, args.input.sections);
|
|
2309
|
-
const budgeted = applyPmcBudget(parsed, args.budget);
|
|
2545
|
+
const budgeted = applyPmcBudget(parsed, args.budget, bodySectionCount(parsedArticle, args.input));
|
|
2310
2546
|
// `parsePmcArticle` always returns string fields (sometimes empty). Strip
|
|
2311
2547
|
// empty `pmcId`/`pmcUrl` for EPMC-only records (preprints) so the schema's
|
|
2312
2548
|
// optional shape is respected — agents read `epmcId`/`epmcSource` for those.
|
|
@@ -2381,8 +2617,8 @@ async function fetchPubmedDois(pmids, signal) {
|
|
|
2381
2617
|
}
|
|
2382
2618
|
/**
|
|
2383
2619
|
* Resolve a DOI to an open-access article via Unpaywall. `pmcId` and `pmid`,
|
|
2384
|
-
* when set, are stamped onto the resulting article so the
|
|
2385
|
-
*
|
|
2620
|
+
* when set, are stamped onto the resulting article so the id that requested it
|
|
2621
|
+
* carries its identifier through — Unpaywall itself only knows the DOI.
|
|
2386
2622
|
*
|
|
2387
2623
|
* The article's title is the first one found in Unpaywall's own record, then
|
|
2388
2624
|
* `epmcTitle` — the Europe PMC record the chain searched for this id — then,
|
|
@@ -2709,8 +2945,8 @@ function formatPmcArticle(a, lines, truncation, mode) {
|
|
|
2709
2945
|
}
|
|
2710
2946
|
if (a.affiliations?.length) {
|
|
2711
2947
|
lines.push(`\n**Affiliations:**`);
|
|
2712
|
-
for (const
|
|
2713
|
-
lines.push(
|
|
2948
|
+
for (const aff of a.affiliations)
|
|
2949
|
+
lines.push(`- ${aff}`);
|
|
2714
2950
|
}
|
|
2715
2951
|
if (a.journal) {
|
|
2716
2952
|
const parts = [];
|
|
@@ -2781,6 +3017,9 @@ function formatPmcArticle(a, lines, truncation, mode) {
|
|
|
2781
3017
|
* `rowspan`, so a well-formed table arrives rectangular and every value renders
|
|
2782
3018
|
* under the header it belongs to; a row still short of the widest is padded on
|
|
2783
3019
|
* the right with empty cells only, never a neighbour's value.
|
|
3020
|
+
*
|
|
3021
|
+
* A grid row and the Footnotes line are single lines, so the `\n` a cell or the
|
|
3022
|
+
* footer keeps between its lines renders as {@link inlineLines} says. (#185)
|
|
2784
3023
|
*/
|
|
2785
3024
|
function formatTables(tables, lines) {
|
|
2786
3025
|
lines.push(`\n#### Tables (${tables.length})`);
|
|
@@ -2799,10 +3038,23 @@ function formatTables(tables, lines) {
|
|
|
2799
3038
|
}
|
|
2800
3039
|
if (table.rows.length > 0)
|
|
2801
3040
|
lines.push(...renderTableGrid(table.rows, table.headerRowCount));
|
|
2802
|
-
if (table.footnotes)
|
|
2803
|
-
lines.push(`\nFootnotes: ${escapeMarkdownInline(table.footnotes)}`);
|
|
3041
|
+
if (table.footnotes) {
|
|
3042
|
+
lines.push(`\nFootnotes: ${escapeMarkdownInline(inlineLines(table.footnotes))}`);
|
|
3043
|
+
}
|
|
2804
3044
|
}
|
|
2805
3045
|
}
|
|
3046
|
+
/**
|
|
3047
|
+
* Lay a multi-line cell or footer value onto one Markdown line, its lines joined
|
|
3048
|
+
* by ` · ` — the joiner the header fold and the meta lines already use. A bare
|
|
3049
|
+
* space would run one footnote into the next, and a line per footnote would put
|
|
3050
|
+
* markers such as `*`, `#` and `1.` at the start of a line, where they become
|
|
3051
|
+
* Markdown structure. Callers escape the joined result, so emphasis pairing is
|
|
3052
|
+
* judged across the whole rendered line rather than one source line at a time.
|
|
3053
|
+
* (#185)
|
|
3054
|
+
*/
|
|
3055
|
+
function inlineLines(text) {
|
|
3056
|
+
return text.replaceAll('\n', ' · ');
|
|
3057
|
+
}
|
|
2806
3058
|
/**
|
|
2807
3059
|
* Render every figure and supplementary item, so a `content[]` reader gets the
|
|
2808
3060
|
* same caption, pointer and placement `structuredContent` carries rather than a
|
|
@@ -2868,7 +3120,7 @@ function foldHeaderColumn(headerRows, column) {
|
|
|
2868
3120
|
/** One table's rows as Markdown grid lines, preceded by a blank line. */
|
|
2869
3121
|
function renderTableGrid(rows, headerRowCount) {
|
|
2870
3122
|
const columns = rows.reduce((widest, row) => Math.max(widest, row.length), 0);
|
|
2871
|
-
const renderRow = (cells) => `| ${Array.from({ length: columns }, (_, i) => escapeMarkdownTableCell(cells[i] ?? '')).join(' | ')} |`;
|
|
3123
|
+
const renderRow = (cells) => `| ${Array.from({ length: columns }, (_, i) => escapeMarkdownTableCell(inlineLines(cells[i] ?? ''))).join(' | ')} |`;
|
|
2872
3124
|
const headerRows = rows.slice(0, headerRowCount);
|
|
2873
3125
|
const header = Array.from({ length: columns }, (_, i) => foldHeaderColumn(headerRows, i));
|
|
2874
3126
|
return [
|
|
@@ -2941,7 +3193,7 @@ function sectionHeading(section) {
|
|
|
2941
3193
|
return section.label ? escapeMarkdownInline(section.label) : UNTITLED_SECTION_LABEL;
|
|
2942
3194
|
}
|
|
2943
3195
|
/**
|
|
2944
|
-
* Render one
|
|
3196
|
+
* Render one section and everything nested under it, one markdown heading
|
|
2945
3197
|
* level per nesting level. Walks the full depth the output schema carries, so
|
|
2946
3198
|
* `content[]` shows every section `structuredContent` does. Headings stop
|
|
2947
3199
|
* deepening at `######`, the deepest markdown supports. (#112)
|