@cyanheads/pubmed-mcp-server 2.10.8 → 2.10.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CLAUDE.md +1 -1
- package/README.md +6 -4
- package/dist/mcp-server/tools/definitions/_text.d.ts +23 -1
- package/dist/mcp-server/tools/definitions/_text.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/_text.js +25 -1
- package/dist/mcp-server/tools/definitions/_text.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts +35 -0
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js +657 -44
- package/dist/mcp-server/tools/definitions/fetch-fulltext.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts +8 -0
- package/dist/mcp-server/tools/definitions/find-related.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-related.tool.js +131 -23
- package/dist/mcp-server/tools/definitions/find-related.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts +6 -0
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js +14 -3
- package/dist/mcp-server/tools/definitions/lookup-mesh.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts +6 -0
- package/dist/mcp-server/tools/definitions/search-articles.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-articles.tool.js +15 -3
- package/dist/mcp-server/tools/definitions/search-articles.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts +6 -0
- package/dist/mcp-server/tools/definitions/spell-check.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/spell-check.tool.js +15 -3
- package/dist/mcp-server/tools/definitions/spell-check.tool.js.map +1 -1
- package/dist/services/error-contracts.d.ts +22 -2
- package/dist/services/error-contracts.d.ts.map +1 -1
- package/dist/services/error-contracts.js +24 -2
- package/dist/services/error-contracts.js.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.d.ts +3 -3
- package/dist/services/europe-pmc/europe-pmc-service.d.ts.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.js +8 -16
- package/dist/services/europe-pmc/europe-pmc-service.js.map +1 -1
- package/dist/services/ncbi/parsing/ordered-xml-parser-options.d.ts +44 -0
- package/dist/services/ncbi/parsing/ordered-xml-parser-options.d.ts.map +1 -0
- package/dist/services/ncbi/parsing/ordered-xml-parser-options.js +41 -0
- package/dist/services/ncbi/parsing/ordered-xml-parser-options.js.map +1 -0
- package/dist/services/ncbi/parsing/pmc-article-parser.d.ts +70 -10
- package/dist/services/ncbi/parsing/pmc-article-parser.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-article-parser.js +795 -79
- package/dist/services/ncbi/parsing/pmc-article-parser.js.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts +18 -0
- package/dist/services/ncbi/parsing/pmc-xml-helpers.d.ts.map +1 -1
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js +39 -3
- package/dist/services/ncbi/parsing/pmc-xml-helpers.js.map +1 -1
- package/dist/services/ncbi/response-handler.d.ts +3 -9
- package/dist/services/ncbi/response-handler.d.ts.map +1 -1
- package/dist/services/ncbi/response-handler.js +6 -29
- package/dist/services/ncbi/response-handler.js.map +1 -1
- package/dist/services/ncbi/types.d.ts +82 -0
- package/dist/services/ncbi/types.d.ts.map +1 -1
- package/dist/services/openalex/api-client.d.ts +13 -4
- package/dist/services/openalex/api-client.d.ts.map +1 -1
- package/dist/services/openalex/api-client.js +19 -8
- package/dist/services/openalex/api-client.js.map +1 -1
- package/dist/services/openalex/openalex-service.d.ts +34 -17
- package/dist/services/openalex/openalex-service.d.ts.map +1 -1
- package/dist/services/openalex/openalex-service.js +119 -35
- package/dist/services/openalex/openalex-service.js.map +1 -1
- package/dist/services/openalex/types.d.ts +34 -1
- package/dist/services/openalex/types.d.ts.map +1 -1
- package/dist/services/openalex/types.js +20 -0
- package/dist/services/openalex/types.js.map +1 -1
- package/package.json +1 -1
- package/server.json +3 -3
|
@@ -31,16 +31,66 @@ import { getUnpaywallService, } from '../../../services/unpaywall/unpaywall-serv
|
|
|
31
31
|
import { fitWholeItems } from './_budget.js';
|
|
32
32
|
import { conceptMeta, EDAM_DATA_RETRIEVAL, SCHEMA_SCHOLARLY_ARTICLE } from './_concepts.js';
|
|
33
33
|
import { pmidStringSchema } from './_schemas.js';
|
|
34
|
-
import { escapeMarkdownInline, sliceCodeUnits } from './_text.js';
|
|
34
|
+
import { escapeMarkdownInline, escapeMarkdownTableCell, sliceCodeUnits } from './_text.js';
|
|
35
35
|
function normalizePmcId(id) {
|
|
36
36
|
return id.replace(/^PMC/i, '');
|
|
37
37
|
}
|
|
38
38
|
function withPmcPrefix(id) {
|
|
39
39
|
return id.startsWith('PMC') ? id : `PMC${id}`;
|
|
40
40
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
/** Case-insensitive substring match of one section heading against the filter. */
|
|
42
|
+
function matchesSectionFilter(title, lowerFilter) {
|
|
43
|
+
const lowered = title?.toLowerCase();
|
|
44
|
+
return lowered !== undefined && lowerFilter.some((f) => lowered.includes(f));
|
|
45
|
+
}
|
|
46
|
+
function lowerCase(s) {
|
|
47
|
+
return s.toLowerCase();
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Prune a section tree to the branches a `sections` filter selects, matching
|
|
51
|
+
* titles at every nesting depth rather than the top level alone. (#126)
|
|
52
|
+
*
|
|
53
|
+
* A section whose own title matches is returned whole — object identity
|
|
54
|
+
* included, so an unfiltered subtree is never rebuilt. A section kept only
|
|
55
|
+
* because a descendant matched becomes a breadcrumb: its `title` and `label`
|
|
56
|
+
* survive so the match can be placed in the document, its own `text` is cleared
|
|
57
|
+
* because the caller filtered that prose away, and it carries just the matching
|
|
58
|
+
* branch of its subsections. Promoting the match to the top level instead would
|
|
59
|
+
* make `maxSections` — which caps genuine top-level sections — count nested
|
|
60
|
+
* content as top-level.
|
|
61
|
+
*/
|
|
62
|
+
function pruneSections(sections, lowerFilter) {
|
|
63
|
+
const kept = [];
|
|
64
|
+
for (const section of sections) {
|
|
65
|
+
if (matchesSectionFilter(section.title, lowerFilter)) {
|
|
66
|
+
kept.push(section);
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
const subsections = pruneSections(section.subsections ?? [], lowerFilter);
|
|
70
|
+
if (subsections.length > 0)
|
|
71
|
+
kept.push({ ...section, text: '', subsections });
|
|
72
|
+
}
|
|
73
|
+
return kept;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Titles of the sections a `sections` filter actually selected — a section that
|
|
77
|
+
* matched directly plus everything beneath it. Breadcrumb ancestors are left
|
|
78
|
+
* out: their own text was cleared because the caller did not ask for it, and a
|
|
79
|
+
* table sitting in one was not asked for either. Walks the *pruned* tree, so a
|
|
80
|
+
* top-level section the `maxSections` slice removed contributes nothing.
|
|
81
|
+
*/
|
|
82
|
+
function matchedSectionTitles(sections, lowerFilter) {
|
|
83
|
+
const titles = new Set();
|
|
84
|
+
const walk = (nodes, inherited) => {
|
|
85
|
+
for (const section of nodes) {
|
|
86
|
+
const matched = inherited || matchesSectionFilter(section.title, lowerFilter);
|
|
87
|
+
if (matched && section.title)
|
|
88
|
+
titles.add(section.title);
|
|
89
|
+
walk(section.subsections ?? [], matched);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
walk(sections, false);
|
|
93
|
+
return titles;
|
|
44
94
|
}
|
|
45
95
|
/**
|
|
46
96
|
* Render a section subtree as text blocks, in document order: each section's
|
|
@@ -73,16 +123,133 @@ function clampSectionDepth(sections, depth = 1) {
|
|
|
73
123
|
});
|
|
74
124
|
}
|
|
75
125
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
|
|
126
|
+
* Replace an article's table list, dropping the field entirely when nothing is
|
|
127
|
+
* left. An empty array would read as "this article has no tables", which is the
|
|
128
|
+
* one thing an absent field already says and a filtered-to-nothing list does
|
|
129
|
+
* not mean.
|
|
130
|
+
*/
|
|
131
|
+
function withTables(article, tables) {
|
|
132
|
+
const { tables: _replaced, ...rest } = article;
|
|
133
|
+
return (tables.length > 0 ? { ...rest, tables } : rest);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Narrow the table list to what the request asked for. `includeTables: false`
|
|
137
|
+
* is the wholesale off switch. An active `sections` filter narrows tables with
|
|
138
|
+
* it: a table names the section it sat in — body, back matter or appendix
|
|
139
|
+
* alike — so it survives when that section did, and a table that names no
|
|
140
|
+
* section, such as a `<floats-group>` deposit, is dropped because the caller
|
|
141
|
+
* asked for named headings and it belongs to none. With no `sections` filter
|
|
142
|
+
* every table is returned. (#111)
|
|
143
|
+
*/
|
|
144
|
+
function applyTableFilters(article, filters) {
|
|
145
|
+
if (!article.tables?.length)
|
|
146
|
+
return article;
|
|
147
|
+
if (!filters.includeTables)
|
|
148
|
+
return withTables(article, []);
|
|
149
|
+
if (!filters.sections?.length)
|
|
150
|
+
return article;
|
|
151
|
+
const surviving = matchedSectionTitles(article.sections, filters.sections.map(lowerCase));
|
|
152
|
+
return withTables(article, article.tables.filter((t) => t.sectionTitle !== undefined && surviving.has(t.sectionTitle)));
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Replace an article's asset list, dropping the field entirely when nothing is
|
|
156
|
+
* left — the same rule {@link withTables} follows, and for the same reason: an
|
|
157
|
+
* empty array claims the article deposits no figures, which a filtered-to-nothing
|
|
158
|
+
* list does not mean.
|
|
159
|
+
*/
|
|
160
|
+
function withAssets(article, assets) {
|
|
161
|
+
const { assets: _replaced, ...rest } = article;
|
|
162
|
+
return (assets.length > 0 ? { ...rest, assets } : rest);
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* The positional marker the parser leaves in section text where an asset was
|
|
166
|
+
* lifted out — `[Figure: Fig. 1]`, `[Supplementary: Table S3]`, or the bare
|
|
167
|
+
* `[Figure]` / `[Supplementary]` when the deposit carries no label.
|
|
168
|
+
*
|
|
169
|
+
* Mirrors `assetMarker` in `pmc-article-parser.ts`, which is the only producer.
|
|
170
|
+
* The two must stay byte-identical: this is what {@link stripAssetMarkers}
|
|
171
|
+
* removes, and a marker built any other way would leave the real one in place.
|
|
172
|
+
* Derived per asset rather than matched as a pattern, so prose that happens to
|
|
173
|
+
* carry a bracketed word is never touched. (#130)
|
|
174
|
+
*/
|
|
175
|
+
function assetMarkerText(asset) {
|
|
176
|
+
const kind = asset.assetType === 'figure' ? 'Figure' : 'Supplementary';
|
|
177
|
+
return asset.label ? `[${kind}: ${asset.label}]` : `[${kind}]`;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Remove every marker in `markers` from one text field and close the gap it
|
|
181
|
+
* leaves: trailing whitespace on a line the marker ended, and the blank line a
|
|
182
|
+
* marker that stood alone as its own block leaves behind.
|
|
183
|
+
*/
|
|
184
|
+
function stripAssetMarkers(text, markers) {
|
|
185
|
+
if (!text)
|
|
186
|
+
return text;
|
|
187
|
+
let out = text;
|
|
188
|
+
for (const marker of markers)
|
|
189
|
+
out = out.split(marker).join('');
|
|
190
|
+
if (out === text)
|
|
191
|
+
return text;
|
|
192
|
+
return out
|
|
193
|
+
.split('\n')
|
|
194
|
+
.map((line) => line.replace(/[ \t]+$/, ''))
|
|
195
|
+
.join('\n')
|
|
196
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
197
|
+
.trim();
|
|
198
|
+
}
|
|
199
|
+
/** Strip asset markers from a section and every subsection beneath it. */
|
|
200
|
+
function sectionWithoutMarkers(section, markers) {
|
|
201
|
+
const text = stripAssetMarkers(section.text, markers);
|
|
202
|
+
const subsections = section.subsections?.map((sub) => sectionWithoutMarkers(sub, markers));
|
|
203
|
+
if (text === section.text && subsections === undefined)
|
|
204
|
+
return section;
|
|
205
|
+
return { ...section, text, ...(subsections && { subsections }) };
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Narrow the asset list to what the request asked for, mirroring {@link
|
|
209
|
+
* applyTableFilters}: `includeAssets: false` is the wholesale off switch, and an
|
|
210
|
+
* active `sections` filter keeps an asset whose named section survived while
|
|
211
|
+
* dropping one that names no section at all — a `<floats-group>` deposit — since
|
|
212
|
+
* the caller asked for named headings and it belongs to none.
|
|
213
|
+
*
|
|
214
|
+
* Where it goes beyond the table counterpart: turning assets off also removes the
|
|
215
|
+
* positional markers the parser left in the section text. The marker is the
|
|
216
|
+
* lift's anchor, and without the array to resolve it against it points at
|
|
217
|
+
* nothing. Prose-shaped blocks — lists, quotes, formulae — are section text
|
|
218
|
+
* rather than assets and this switch never touches them. (#130)
|
|
219
|
+
*/
|
|
220
|
+
function applyAssetFilters(article, filters) {
|
|
221
|
+
const assets = article.assets;
|
|
222
|
+
if (!assets?.length)
|
|
223
|
+
return article;
|
|
224
|
+
if (!filters.includeAssets) {
|
|
225
|
+
const markers = assets.map(assetMarkerText);
|
|
226
|
+
return withAssets({ ...article, sections: article.sections.map((s) => sectionWithoutMarkers(s, markers)) }, []);
|
|
227
|
+
}
|
|
228
|
+
if (!filters.sections?.length)
|
|
229
|
+
return article;
|
|
230
|
+
const surviving = matchedSectionTitles(article.sections, filters.sections.map(lowerCase));
|
|
231
|
+
return withAssets(article, assets.filter((a) => a.sectionTitle !== undefined && surviving.has(a.sectionTitle)));
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Apply the requested section/reference/table/asset filters, then clamp the
|
|
235
|
+
* section tree to the depth the output schema carries. All of it runs here so every path
|
|
236
|
+
* producing a `pmc` article — PMC EFetch and the Europe PMC stage — shares one
|
|
237
|
+
* shape, and the budget helpers downstream count the text that will actually
|
|
238
|
+
* survive validation. (#112)
|
|
239
|
+
*
|
|
240
|
+
* Order is load-bearing. Tables and assets are matched against the section tree
|
|
241
|
+
* *after* `filterSections` and the `maxSections` slice, so they narrow with
|
|
242
|
+
* exactly what the response returns, and *before* `clampSectionDepth`, which
|
|
243
|
+
* folds sections past {@link MAX_SECTION_DEPTH} into a parent's text — their
|
|
244
|
+
* titles vanish from the output while a table or figure still names them, so a
|
|
245
|
+
* title set built after the clamp would drop entries that should have survived.
|
|
246
|
+
* Stripping the asset markers also has to precede the clamp, or a marker in a
|
|
247
|
+
* folded-away section survives in the text it was folded into. (#111, #130)
|
|
81
248
|
*/
|
|
82
249
|
function applyPmcFilters(article, filters) {
|
|
83
250
|
let out = article;
|
|
84
251
|
if (filters.sections?.length) {
|
|
85
|
-
out = { ...out, sections:
|
|
252
|
+
out = { ...out, sections: pruneSections(out.sections, filters.sections.map(lowerCase)) };
|
|
86
253
|
}
|
|
87
254
|
if (filters.maxSections !== undefined) {
|
|
88
255
|
out = { ...out, sections: out.sections.slice(0, filters.maxSections) };
|
|
@@ -91,6 +258,8 @@ function applyPmcFilters(article, filters) {
|
|
|
91
258
|
const { references: _, ...rest } = out;
|
|
92
259
|
out = rest;
|
|
93
260
|
}
|
|
261
|
+
out = applyTableFilters(out, filters);
|
|
262
|
+
out = applyAssetFilters(out, filters);
|
|
94
263
|
return { ...out, sections: clampSectionDepth(out.sections) };
|
|
95
264
|
}
|
|
96
265
|
/**
|
|
@@ -125,11 +294,16 @@ function articleDisplayId(a) {
|
|
|
125
294
|
* Compose the single recovery notice for `sections`-filter misses. Names the
|
|
126
295
|
* requested terms and the affected article id(s) so the agent can distinguish a
|
|
127
296
|
* filtered-empty body from one absent upstream, and points at the recovery. (#80)
|
|
297
|
+
*
|
|
298
|
+
* States the scope the filter actually searches — section and subsection titles
|
|
299
|
+
* at every depth — so a caller who named a nested heading learns the term itself
|
|
300
|
+
* matched nothing, rather than being left to suspect the filter never looked
|
|
301
|
+
* that deep. (#126)
|
|
128
302
|
*/
|
|
129
303
|
function buildSectionFilterMissNotice(affectedIds, sectionFilter) {
|
|
130
304
|
const terms = sectionFilter.join(', ');
|
|
131
305
|
const subject = affectedIds.length === 1 ? `article ${affectedIds[0]}` : `articles ${affectedIds.join(', ')}`;
|
|
132
|
-
return `No
|
|
306
|
+
return `No section or subsection title, at any nesting depth, matched the requested section filter (${terms}) for ${subject}. The full text was retrieved but every body section was filtered out. Retry without \`sections\`, or filter on broader headings such as Introduction, Methods, Results, or Discussion.`;
|
|
133
307
|
}
|
|
134
308
|
/**
|
|
135
309
|
* Compose the recovery notice for identifiers whose only retrievable record was
|
|
@@ -204,6 +378,86 @@ const ReferenceSchema = z
|
|
|
204
378
|
label: z.string().optional().describe('Reference label'),
|
|
205
379
|
})
|
|
206
380
|
.describe('Reference entry');
|
|
381
|
+
/**
|
|
382
|
+
* One `<table-wrap>`, hung off the article rather than off a section.
|
|
383
|
+
*
|
|
384
|
+
* The article is the only level that can hold every table: roughly a quarter of
|
|
385
|
+
* real `<table-wrap>` elements sit in `<floats-group>`, `<back>`, a
|
|
386
|
+
* `<table-wrap-group>` or an appendix, with no `<sec>` to attach to. A table
|
|
387
|
+
* inside a section names it in {@link sectionTitle} instead of being placed by
|
|
388
|
+
* position.
|
|
389
|
+
*
|
|
390
|
+
* The shape also lands exactly on the `format-parity` sentinel walker's eight-hop
|
|
391
|
+
* budget — `articles[]` → the article union → `tables[]` → `rows[][]` — with no
|
|
392
|
+
* headroom, and a section-hung variant would spend a hop the existing
|
|
393
|
+
* `sections[]` → `subsections[]` chain already needs. Re-run
|
|
394
|
+
* `bun run lint:mcp` after any change here; a level added anywhere inside puts
|
|
395
|
+
* its own leaves out of the walker's reach and ships their `format()` parity
|
|
396
|
+
* unverified. (#111, #112)
|
|
397
|
+
*/
|
|
398
|
+
const TableSchema = z
|
|
399
|
+
.object({
|
|
400
|
+
label: z.string().optional().describe('Table label as printed, e.g. `TABLE 1`'),
|
|
401
|
+
caption: z.string().optional().describe('Caption text, with the label excluded'),
|
|
402
|
+
id: z
|
|
403
|
+
.string()
|
|
404
|
+
.optional()
|
|
405
|
+
.describe('JATS `id` attribute — the target body-text cross-references point at'),
|
|
406
|
+
sectionTitle: z
|
|
407
|
+
.string()
|
|
408
|
+
.optional()
|
|
409
|
+
.describe('Title of the innermost section enclosing the table, wherever that section sits — body, `<back>` matter, or an appendix all count, and in back matter the section name is the only positional cue there is. Absent only for a table inside no section at all, such as a `<floats-group>` deposit.'),
|
|
410
|
+
headerRowCount: z
|
|
411
|
+
.number()
|
|
412
|
+
.describe('How many leading `rows` entries are header rows — a `<thead>` block, or leading rows made entirely of `<th>`. 0 when the table declares none. Several header rows stack: read one column top to bottom for its full header path.'),
|
|
413
|
+
rows: z
|
|
414
|
+
.array(z.array(z.string()).describe('One row, as cell text by grid column'))
|
|
415
|
+
.describe('Cell text by row, in document order, one entry per grid column. `colspan` and `rowspan` are expanded, so a cell covering several columns or rows repeats its text across each cell it covers and a well-formed table is rectangular — align on position from the left, and read a repeated value as one spanning cell rather than several measurements. Empty when `unextractableReason` is set.'),
|
|
416
|
+
footnotes: z.string().optional().describe('`<table-wrap-foot>` text, flattened to one string'),
|
|
417
|
+
unextractableReason: z
|
|
418
|
+
.enum(['cals-tgroup', 'graphic-only', 'no-rows'])
|
|
419
|
+
.optional()
|
|
420
|
+
.describe('Why `rows` is empty — set only then. graphic-only: the table was deposited as an image with no underlying markup. cals-tgroup: the table uses the CALS `<tgroup>` model, which this server does not extract (0 of 283 tables in an open-access survey used it). no-rows: the markup carried no rows. The label and caption are still returned, so a table that could not be read is visible rather than silently missing.'),
|
|
421
|
+
})
|
|
422
|
+
.describe('One table from the article, with its cells, caption, and owning section');
|
|
423
|
+
/**
|
|
424
|
+
* One `<fig>` or `<supplementary-material>`, hung off the article beside
|
|
425
|
+
* {@link TableSchema} and for the same reason: a sixth of them sit in
|
|
426
|
+
* `<floats-group>`, `<back>` or an appendix with no `<sec>` to attach to, so an
|
|
427
|
+
* asset inside a section names it in {@link sectionTitle} rather than being
|
|
428
|
+
* placed by position.
|
|
429
|
+
*
|
|
430
|
+
* Scalar leaves only. `articles[]` → the article union → `assets[]` → a leaf is
|
|
431
|
+
* six of the eight hops the `format-parity` sentinel walker allows; a nested
|
|
432
|
+
* object here (a `files[]` list, say) spends the remaining two and leaves nothing
|
|
433
|
+
* for a later field. Re-run `bun run lint:mcp` after any change to this shape.
|
|
434
|
+
*
|
|
435
|
+
* There is no media-type field and no per-asset unextractable reason: `mimetype`
|
|
436
|
+
* appears on no observed deposit, and an uncaptioned supplement is still fully
|
|
437
|
+
* described by its `id` and `href`, unlike a table that promises cells and
|
|
438
|
+
* carries none. (#130)
|
|
439
|
+
*/
|
|
440
|
+
const AssetSchema = z
|
|
441
|
+
.object({
|
|
442
|
+
assetType: z
|
|
443
|
+
.enum(['figure', 'supplementary-material'])
|
|
444
|
+
.describe('Which captioned element this came from — `figure` for a `<fig>`, `supplementary-material` for a `<supplementary-material>` deposit'),
|
|
445
|
+
label: z.string().optional().describe('Display label as printed, e.g. `Fig. 1`'),
|
|
446
|
+
caption: z.string().optional().describe('Caption text, with the label excluded'),
|
|
447
|
+
id: z
|
|
448
|
+
.string()
|
|
449
|
+
.optional()
|
|
450
|
+
.describe('JATS `id` attribute — the target body-text cross-references point at'),
|
|
451
|
+
sectionTitle: z
|
|
452
|
+
.string()
|
|
453
|
+
.optional()
|
|
454
|
+
.describe('Title of the innermost section enclosing the asset, wherever that section sits — body, `<back>` matter, or an appendix all count. Absent for an asset inside no section at all, such as a `<floats-group>` deposit.'),
|
|
455
|
+
href: z
|
|
456
|
+
.string()
|
|
457
|
+
.optional()
|
|
458
|
+
.describe('The `<graphic>`/`<media>` `@xlink:href` exactly as deposited — a pointer into the PMC deposit (`MOL2-20-1253-g001.jpg`), not a fetchable URL. No absolute form of it resolves; read the rendered article at `pmcUrl` instead. Absent when the deposit names no file.'),
|
|
459
|
+
})
|
|
460
|
+
.describe('One figure or supplementary-material item, with its caption, pointer, and section');
|
|
207
461
|
const PublicationDateSchema = z
|
|
208
462
|
.object({
|
|
209
463
|
year: z.string().optional().describe('Publication year'),
|
|
@@ -239,6 +493,14 @@ const PmcArticleSchema = z
|
|
|
239
493
|
articleType: z.string().optional().describe('Article type'),
|
|
240
494
|
publicationDate: PublicationDateSchema.optional(),
|
|
241
495
|
sections: z.array(SectionSchema).describe('Article body sections'),
|
|
496
|
+
tables: z
|
|
497
|
+
.array(TableSchema)
|
|
498
|
+
.optional()
|
|
499
|
+
.describe('Every `<table-wrap>` the article carries, in document order — from the body and from `<floats-group>`, `<back>` and appendices alike. Absent when the article deposits none, when `includeTables` is false, or when a `sections` filter left none standing.'),
|
|
500
|
+
assets: z
|
|
501
|
+
.array(AssetSchema)
|
|
502
|
+
.optional()
|
|
503
|
+
.describe('Every `<fig>` and `<supplementary-material>` the article carries, in document order — from the body and from `<floats-group>`, `<back>` and appendices alike. Each one lifted from the body leaves a `[Figure: <label>]` or `[Supplementary: <label>]` marker at its position in the section text, so reading order survives the lift. Absent when the article deposits none, when `includeAssets` is false, or when a `sections` filter left none standing.'),
|
|
242
504
|
references: z.array(ReferenceSchema).optional().describe('Reference list'),
|
|
243
505
|
epmcId: z
|
|
244
506
|
.string()
|
|
@@ -303,12 +565,13 @@ const UnavailableReasonSchema = z
|
|
|
303
565
|
'no-epmc-fulltext',
|
|
304
566
|
'no-body',
|
|
305
567
|
'no-doi',
|
|
568
|
+
'doi-lookup-failed',
|
|
306
569
|
'no-oa',
|
|
307
570
|
'fetch-failed',
|
|
308
571
|
'parse-failed',
|
|
309
572
|
'service-error',
|
|
310
573
|
])
|
|
311
|
-
.describe('Why no full text was returned — the most specific signal any tier that answered reported. not-found: upstream returned no record for this ID. no-pmc-fallback-disabled: every tier was skipped (`triedTiers` is all `not-attempted`) — typically because EPMC (`EUROPEPMC_ENABLED`) and Unpaywall (`UNPAYWALL_EMAIL`) are not configured. no-epmc-fulltext: EPMC indexed the record but publishes no fullTextXML. no-body: the record was retrieved but carries front matter and abstract only, with no body sections — use `pubmed_fetch_articles` for the metadata. no-doi:
|
|
574
|
+
.describe('Why no full text was returned — the most specific signal any tier that answered reported. not-found: upstream returned no record for this ID. no-pmc-fallback-disabled: every tier was skipped (`triedTiers` is all `not-attempted`) — typically because EPMC (`EUROPEPMC_ENABLED`) and Unpaywall (`UNPAYWALL_EMAIL`) are not configured. no-epmc-fulltext: EPMC indexed the record but publishes no fullTextXML. no-body: the record was retrieved but carries front matter and abstract only, with no body sections — use `pubmed_fetch_articles` for the metadata. no-doi: the DOI lookup ran and this record has none, so Unpaywall could not be queried. doi-lookup-failed: the DOI lookup itself errored, so whether a DOI exists is unknown and Unpaywall was never reached — retry the request; unlike no-doi this is a transient failure, not a settled answer. no-oa: Unpaywall has no OA copy. fetch-failed: download failed. parse-failed: extraction empty. service-error: upstream server failure (threw, timed out, or returned malformed data). A reason never means the chain ran to completion — read `unqueriedTiers` for that.');
|
|
312
575
|
const UnqueriedTierSchema = z
|
|
313
576
|
.enum(['europepmc', 'unpaywall'])
|
|
314
577
|
.describe('A fallback tier this deployment has not configured');
|
|
@@ -319,12 +582,13 @@ const TierOutcomeSchema = z
|
|
|
319
582
|
'no-fulltext',
|
|
320
583
|
'no-body',
|
|
321
584
|
'no-doi',
|
|
585
|
+
'doi-lookup-failed',
|
|
322
586
|
'no-oa',
|
|
323
587
|
'fetch-failed',
|
|
324
588
|
'parse-failed',
|
|
325
589
|
'service-error',
|
|
326
590
|
])
|
|
327
|
-
.describe('Per-tier outcome. not-attempted: tier was skipped. miss: tier returned no record. no-fulltext: EPMC indexed the record but publishes no fullTextXML. no-body: the tier returned a record with front matter and abstract but no body sections, so the chain continued. no-doi:
|
|
591
|
+
.describe('Per-tier outcome. not-attempted: tier was skipped. miss: tier returned no record. no-fulltext: EPMC indexed the record but publishes no fullTextXML. no-body: the tier returned a record with front matter and abstract but no body sections, so the chain continued. no-doi: the DOI lookup ran and this record has none, so Unpaywall could not be queried. doi-lookup-failed: the DOI lookup itself errored, so whether a DOI exists is unknown and Unpaywall was never reached — retry the request. no-oa: Unpaywall reports no open-access copy. fetch-failed: OA copy download failed. parse-failed: extraction produced empty content. service-error: tier service threw.');
|
|
328
592
|
const TriedTierSchema = z
|
|
329
593
|
.object({
|
|
330
594
|
tier: z.enum(['pmc', 'europepmc', 'unpaywall']).describe('Which tier in the resolution chain'),
|
|
@@ -379,6 +643,22 @@ const TruncatedArticleSchema = z
|
|
|
379
643
|
.array(TruncatedSectionSchema)
|
|
380
644
|
.optional()
|
|
381
645
|
.describe('Per-section accounting for `source: pmc` articles, in document order, including sections dropped for budget. Absent for `source: unpaywall`, whose body has no section structure.'),
|
|
646
|
+
omittedTables: z
|
|
647
|
+
.number()
|
|
648
|
+
.optional()
|
|
649
|
+
.describe('Tables this article dropped whole because the budget left no room for them. A table is never cut mid-row, so it is either returned complete or counted here. Absent when none were dropped.'),
|
|
650
|
+
omittedTableNames: z
|
|
651
|
+
.array(z.string())
|
|
652
|
+
.optional()
|
|
653
|
+
.describe("The dropped tables by name, in document order — each table's label, else its `id`, else `table <n>` for its position in the article. Names the tables a bare count only hints at, the way `deferred.ids` names deferred articles. Every table from the first that did not fit onward is here: admission stops at that table rather than skipping ahead to a smaller one, so these are contiguous. Absent when none were dropped."),
|
|
654
|
+
omittedAssets: z
|
|
655
|
+
.number()
|
|
656
|
+
.optional()
|
|
657
|
+
.describe('Figures and supplementary items this article dropped whole because the budget left no room once sections and tables were served. An asset is never returned with a truncated caption, so it is either returned complete or counted here. Absent when none were dropped.'),
|
|
658
|
+
omittedAssetNames: z
|
|
659
|
+
.array(z.string())
|
|
660
|
+
.optional()
|
|
661
|
+
.describe("The dropped assets by name, in document order — each asset's label, else its `id`, else `asset <n>` for its position in the article. Contiguous for the same reason `omittedTableNames` is: admission stops at the first asset that did not fit rather than skipping ahead to a smaller one. Absent when none were dropped."),
|
|
382
662
|
})
|
|
383
663
|
.describe('Character accounting for one article the budget shortened');
|
|
384
664
|
const TruncationSchema = z
|
|
@@ -400,6 +680,14 @@ const TruncationSchema = z
|
|
|
400
680
|
omittedSections: z
|
|
401
681
|
.number()
|
|
402
682
|
.describe('Body sections dropped entirely because an article budget was exhausted before reaching them. Always 0 in `outline` mode, which keeps every heading.'),
|
|
683
|
+
omittedTables: z
|
|
684
|
+
.number()
|
|
685
|
+
.optional()
|
|
686
|
+
.describe('Tables dropped whole across every budgeted article, because the budget left no room once body sections were served. Absent when none were dropped. Re-request the affected articles with a higher `maxCharacters`, or with `sections` narrowed, to receive them.'),
|
|
687
|
+
omittedAssets: z
|
|
688
|
+
.number()
|
|
689
|
+
.optional()
|
|
690
|
+
.describe('Figures and supplementary items dropped whole across every budgeted article, because the budget left no room once body sections and tables were served. Absent when none were dropped. Re-request the affected articles with a higher `maxCharacters`, or with `sections` narrowed, to receive them.'),
|
|
403
691
|
articles: z
|
|
404
692
|
.array(TruncatedArticleSchema)
|
|
405
693
|
.describe('Per-article accounting, covering only the articles the budget shortened'),
|
|
@@ -436,6 +724,10 @@ function budgetRequested(budget) {
|
|
|
436
724
|
function sectionTextFields(section) {
|
|
437
725
|
return [section.text, ...(section.subsections ?? []).flatMap(sectionTextFields)];
|
|
438
726
|
}
|
|
727
|
+
/** Combined length of the strings given, skipping the absent ones. */
|
|
728
|
+
function totalLength(parts) {
|
|
729
|
+
return parts.reduce((n, part) => n + (part?.length ?? 0), 0);
|
|
730
|
+
}
|
|
439
731
|
/**
|
|
440
732
|
* Body characters a section carries — its own text plus every nested
|
|
441
733
|
* subsection's. Measured off {@link sectionTextFields} rather than its own walk,
|
|
@@ -443,7 +735,49 @@ function sectionTextFields(section) {
|
|
|
443
735
|
* exactly the fields {@link fitFields} shortens.
|
|
444
736
|
*/
|
|
445
737
|
function sectionCharacters(section) {
|
|
446
|
-
return sectionTextFields(section)
|
|
738
|
+
return totalLength(sectionTextFields(section));
|
|
739
|
+
}
|
|
740
|
+
/**
|
|
741
|
+
* Characters a table costs the budget: everything it renders — label, caption,
|
|
742
|
+
* every cell, footnotes. The whole figure is what admitting the table spends,
|
|
743
|
+
* and a table is admitted or dropped whole, so there is no partial measure to
|
|
744
|
+
* take. (#111)
|
|
745
|
+
*/
|
|
746
|
+
function tableCharacters(table) {
|
|
747
|
+
return totalLength([table.label, table.caption, table.footnotes, ...table.rows.flat()]);
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* Characters an asset costs the budget: the text it carries — label, caption,
|
|
751
|
+
* and the `href` pointer. Positional metadata is excluded, exactly as {@link
|
|
752
|
+
* tableCharacters} excludes `sectionTitle` and `id`: it places the asset rather
|
|
753
|
+
* than being content the caller asked for. An asset is admitted or dropped
|
|
754
|
+
* whole — a caption cut in half is a caption that says something else — so there
|
|
755
|
+
* is no partial measure to take. (#130)
|
|
756
|
+
*/
|
|
757
|
+
function assetCharacters(asset) {
|
|
758
|
+
return totalLength([asset.label, asset.caption, asset.href]);
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* Admit tables, then assets, in document order until the allowance is spent,
|
|
762
|
+
* then drop the rest whole and name them. The split is {@link fitWholeItems}'s
|
|
763
|
+
* prefix cut, the same one `maxResponseCharacters` applies to whole articles.
|
|
764
|
+
*
|
|
765
|
+
* Admission stops at the first entry that does not fit rather than skipping past
|
|
766
|
+
* it to a smaller one further down: the returned set stays a document-order
|
|
767
|
+
* prefix, so a caller reading it knows where the response stopped instead of
|
|
768
|
+
* receiving a late entry with nothing saying the earlier ones exist. An entry
|
|
769
|
+
* that does not fit is never cut either — half a grid reads as a complete table
|
|
770
|
+
* carrying values that were never deposited, the defect the table extraction
|
|
771
|
+
* exists to fix, and half a caption says something the deposit does not.
|
|
772
|
+
* (#111, #130)
|
|
773
|
+
*/
|
|
774
|
+
function fitWholeNamed(items, allowance, measure, name) {
|
|
775
|
+
const fit = fitWholeItems(items, allowance, measure);
|
|
776
|
+
return {
|
|
777
|
+
kept: fit.kept,
|
|
778
|
+
omittedNames: fit.deferred.map((item, i) => name(item, fit.kept.length + i)),
|
|
779
|
+
spent: fit.keptCharacters,
|
|
780
|
+
};
|
|
447
781
|
}
|
|
448
782
|
/**
|
|
449
783
|
* Rebuild a section subtree from `fitted`, consuming one entry per node in the
|
|
@@ -530,9 +864,16 @@ function allotSectionBudgets(sizes, budget) {
|
|
|
530
864
|
/**
|
|
531
865
|
* Apply the character budget to a JATS article's body. Runs as a pure
|
|
532
866
|
* post-processing pass after `applyPmcFilters`, so `sections` / `maxSections` /
|
|
533
|
-
* `includeReferences` and the empty-body signals they feed are
|
|
534
|
-
* Titles, abstracts, identifiers, and references are never counted
|
|
535
|
-
* the budget
|
|
867
|
+
* `includeReferences` / `includeTables` and the empty-body signals they feed are
|
|
868
|
+
* unaffected. Titles, abstracts, identifiers, and references are never counted
|
|
869
|
+
* or cut — the budget spends on body text and table content, keeping every
|
|
870
|
+
* article citable.
|
|
871
|
+
*
|
|
872
|
+
* Body sections are served first, then tables, then assets, each spending what
|
|
873
|
+
* `maxCharacters` has left, in document order: an item that does not fit is
|
|
874
|
+
* dropped whole and counted, never truncated into a partial grid or a caption cut
|
|
875
|
+
* short. With no `maxCharacters` — a bare `maxCharactersPerSection` request —
|
|
876
|
+
* nothing bounds either list and every entry is kept. (#111, #130)
|
|
536
877
|
*
|
|
537
878
|
* Returns the article untouched (same object identity) when no budget was
|
|
538
879
|
* requested or nothing exceeded it. A section left with zero characters is
|
|
@@ -541,11 +882,16 @@ function allotSectionBudgets(sizes, budget) {
|
|
|
541
882
|
* caller can see which headings exist. (#81)
|
|
542
883
|
*/
|
|
543
884
|
function applyPmcBudget(article, budget) {
|
|
544
|
-
|
|
885
|
+
const tables = article.tables ?? [];
|
|
886
|
+
const assets = article.assets ?? [];
|
|
887
|
+
if (!budgetRequested(budget) ||
|
|
888
|
+
(article.sections.length === 0 && tables.length === 0 && assets.length === 0)) {
|
|
545
889
|
return { article, omittedSections: 0 };
|
|
546
890
|
}
|
|
547
891
|
const sizes = article.sections.map(sectionCharacters);
|
|
548
|
-
const
|
|
892
|
+
const tablesOriginal = tables.reduce((sum, table) => sum + tableCharacters(table), 0);
|
|
893
|
+
const assetsOriginal = assets.reduce((sum, asset) => sum + assetCharacters(asset), 0);
|
|
894
|
+
const originalCharacters = sizes.reduce((sum, size) => sum + size, 0) + tablesOriginal + assetsOriginal;
|
|
549
895
|
const allowances = allotSectionBudgets(sizes, budget);
|
|
550
896
|
const kept = [];
|
|
551
897
|
const sectionReports = [];
|
|
@@ -554,7 +900,7 @@ function applyPmcBudget(article, budget) {
|
|
|
554
900
|
article.sections.forEach((section, i) => {
|
|
555
901
|
const original = sizes[i] ?? 0;
|
|
556
902
|
const fitted = fitFields(sectionTextFields(section), allowances[i] ?? 0);
|
|
557
|
-
const returned = fitted
|
|
903
|
+
const returned = totalLength(fitted);
|
|
558
904
|
returnedCharacters += returned;
|
|
559
905
|
sectionReports.push({
|
|
560
906
|
...(section.title !== undefined && { title: section.title }),
|
|
@@ -568,13 +914,40 @@ function applyPmcBudget(article, budget) {
|
|
|
568
914
|
}
|
|
569
915
|
kept.push(withFittedTexts(section, fitted, { i: 0 }));
|
|
570
916
|
});
|
|
571
|
-
|
|
917
|
+
// Sections are served first, then tables, then assets — each spending whatever
|
|
918
|
+
// `maxCharacters` has left. A bare per-section budget sets no total, so nothing
|
|
919
|
+
// bounds either list.
|
|
920
|
+
const remainingAllowance = () => budget.maxCharacters === undefined
|
|
921
|
+
? Number.POSITIVE_INFINITY
|
|
922
|
+
: Math.max(budget.maxCharacters - returnedCharacters, 0);
|
|
923
|
+
const fittedTables = fitWholeNamed(tables, remainingAllowance(), tableCharacters, tableDisplayName);
|
|
924
|
+
returnedCharacters += fittedTables.spent;
|
|
925
|
+
const omittedTables = fittedTables.omittedNames.length;
|
|
926
|
+
const fittedAssets = fitWholeNamed(assets, remainingAllowance(), assetCharacters, assetDisplayName);
|
|
927
|
+
returnedCharacters += fittedAssets.spent;
|
|
928
|
+
const omittedAssets = fittedAssets.omittedNames.length;
|
|
929
|
+
if (returnedCharacters === originalCharacters &&
|
|
930
|
+
omittedSections === 0 &&
|
|
931
|
+
omittedTables === 0 &&
|
|
932
|
+
omittedAssets === 0) {
|
|
572
933
|
return { article, omittedSections: 0 };
|
|
573
934
|
}
|
|
574
935
|
return {
|
|
575
|
-
article: { ...article, sections: kept },
|
|
936
|
+
article: withAssets(withTables({ ...article, sections: kept }, fittedTables.kept), fittedAssets.kept),
|
|
576
937
|
omittedSections,
|
|
577
|
-
truncation: {
|
|
938
|
+
truncation: {
|
|
939
|
+
originalCharacters,
|
|
940
|
+
returnedCharacters,
|
|
941
|
+
sections: sectionReports,
|
|
942
|
+
...(omittedTables > 0 && {
|
|
943
|
+
omittedTables,
|
|
944
|
+
omittedTableNames: fittedTables.omittedNames,
|
|
945
|
+
}),
|
|
946
|
+
...(omittedAssets > 0 && {
|
|
947
|
+
omittedAssets,
|
|
948
|
+
omittedAssetNames: fittedAssets.omittedNames,
|
|
949
|
+
}),
|
|
950
|
+
},
|
|
578
951
|
};
|
|
579
952
|
}
|
|
580
953
|
/**
|
|
@@ -593,6 +966,66 @@ function applyContentBudget(content, budget) {
|
|
|
593
966
|
truncation: { originalCharacters: content.length, returnedCharacters: kept.length },
|
|
594
967
|
};
|
|
595
968
|
}
|
|
969
|
+
/** Why a table arrived with no rows, in the reader's terms. */
|
|
970
|
+
const UNEXTRACTABLE_TABLE_EXPLANATIONS = {
|
|
971
|
+
'cals-tgroup': 'it uses the CALS `<tgroup>` model, which this server does not extract',
|
|
972
|
+
'graphic-only': 'it was deposited as an image, with no underlying markup to read',
|
|
973
|
+
'no-rows': 'its markup carried no rows',
|
|
974
|
+
};
|
|
975
|
+
/** How a table is named in a notice: its label, else its id, else its position. */
|
|
976
|
+
function tableDisplayName(table, index) {
|
|
977
|
+
return table.label ?? table.id ?? `table ${index + 1}`;
|
|
978
|
+
}
|
|
979
|
+
/** How an asset is named in a notice: its label, else its id, else its position. */
|
|
980
|
+
function assetDisplayName(asset, index) {
|
|
981
|
+
return asset.label ?? asset.id ?? `asset ${index + 1}`;
|
|
982
|
+
}
|
|
983
|
+
/**
|
|
984
|
+
* Collect the returned tables that carry no cell values. Read off the articles
|
|
985
|
+
* the response actually ships — after every filter and both budgets — so a table
|
|
986
|
+
* the `sections` filter removed, or one belonging to a deferred article, is
|
|
987
|
+
* never named as though the caller received it. (#111)
|
|
988
|
+
*/
|
|
989
|
+
function collectUnextractableTables(articles) {
|
|
990
|
+
const entries = [];
|
|
991
|
+
for (const article of articles) {
|
|
992
|
+
if (article.source !== 'pmc')
|
|
993
|
+
continue;
|
|
994
|
+
for (const [index, table] of (article.tables ?? []).entries()) {
|
|
995
|
+
const reason = table.unextractableReason;
|
|
996
|
+
if (!reason)
|
|
997
|
+
continue;
|
|
998
|
+
entries.push({
|
|
999
|
+
articleId: articleDisplayId(article),
|
|
1000
|
+
name: tableDisplayName(table, index),
|
|
1001
|
+
reason,
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
return entries;
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Compose the recovery notice for tables returned with a label and caption but
|
|
1009
|
+
* no cells. Without it a table-bearing response carries no response-level signal
|
|
1010
|
+
* that some of the numbers the caller asked for are absent — the per-table
|
|
1011
|
+
* `unextractableReason` only helps a reader who already went looking at that
|
|
1012
|
+
* table. One notice covers the whole response, aggregated across articles, the
|
|
1013
|
+
* way the `sections`-filter miss notice does. (#111)
|
|
1014
|
+
*
|
|
1015
|
+
* The wording turns on recoverability: nothing the caller changes produces these
|
|
1016
|
+
* cells, because the markup does not exist upstream. Tables the character budget
|
|
1017
|
+
* dropped are the opposite case — recoverable by raising `maxCharacters` — so
|
|
1018
|
+
* they stay with the rest of the budget accounting in {@link
|
|
1019
|
+
* buildTruncationNotice} rather than being mixed in here.
|
|
1020
|
+
*/
|
|
1021
|
+
function buildUnextractableTablesNotice(entries) {
|
|
1022
|
+
const subject = entries.length === 1 ? '1 table was' : `${entries.length} tables were`;
|
|
1023
|
+
const named = entries.map((e) => `${e.name} (${e.articleId}, ${e.reason})`).join(', ');
|
|
1024
|
+
const reasons = [...new Set(entries.map((e) => e.reason))]
|
|
1025
|
+
.map((reason) => `${reason} — ${UNEXTRACTABLE_TABLE_EXPLANATIONS[reason]}`)
|
|
1026
|
+
.join('; ');
|
|
1027
|
+
return `${subject} returned with a label and caption but no cell values: ${named}. Re-calling will not recover the cells (${reasons}); see \`unextractableReason\` on each table.`;
|
|
1028
|
+
}
|
|
596
1029
|
/**
|
|
597
1030
|
* Compose the recovery notice for a budgeted response. Names what was spent and
|
|
598
1031
|
* where the detail lives so an agent reading only `content[]` knows the body it
|
|
@@ -603,13 +1036,26 @@ function buildTruncationNotice(truncation) {
|
|
|
603
1036
|
const omitted = truncation.omittedSections > 0
|
|
604
1037
|
? ` ${truncation.omittedSections} section(s) were dropped once the budget ran out.`
|
|
605
1038
|
: '';
|
|
1039
|
+
// Tables are admitted after sections and only whole, so a dropped one is
|
|
1040
|
+
// absent rather than partial — say so, and name them: a bare count leaves the
|
|
1041
|
+
// reader unable to tell which numbers are missing from what they received.
|
|
1042
|
+
const droppedNames = truncation.articles.flatMap((a) => a.omittedTableNames ?? []);
|
|
1043
|
+
const omittedTables = truncation.omittedTables
|
|
1044
|
+
? ` ${truncation.omittedTables} table(s) were dropped whole rather than cut mid-row: ${droppedNames.join(', ')}.`
|
|
1045
|
+
: '';
|
|
1046
|
+
// Assets are admitted last and only whole — a caption cut in half says
|
|
1047
|
+
// something the deposit does not — so name them on the same terms.
|
|
1048
|
+
const droppedAssetNames = truncation.articles.flatMap((a) => a.omittedAssetNames ?? []);
|
|
1049
|
+
const omittedAssets = truncation.omittedAssets
|
|
1050
|
+
? ` ${truncation.omittedAssets} figure/supplementary item(s) were dropped whole rather than returned with a shortened caption: ${droppedAssetNames.join(', ')}.`
|
|
1051
|
+
: '';
|
|
606
1052
|
// Name only the budgets the request actually set — pointing at `maxCharacters`
|
|
607
1053
|
// when the caller only capped per-section sends them to a knob that is unset.
|
|
608
1054
|
const knobs = [
|
|
609
1055
|
truncation.maxCharacters !== undefined ? '`maxCharacters`' : undefined,
|
|
610
1056
|
truncation.maxCharactersPerSection !== undefined ? '`maxCharactersPerSection`' : undefined,
|
|
611
1057
|
].filter((k) => k !== undefined);
|
|
612
|
-
return `Full text was shortened to fit the requested character budget: ${truncation.returnedCharacters} of ${truncation.originalCharacters} body characters returned across ${subject} in ${truncation.mode} mode.${omitted} See \`truncation\` for per-article and per-section counts, and raise ${knobs.join(' or ')} or narrow \`sections\` to retrieve more.`;
|
|
1058
|
+
return `Full text was shortened to fit the requested character budget: ${truncation.returnedCharacters} of ${truncation.originalCharacters} body characters returned across ${subject} in ${truncation.mode} mode.${omitted}${omittedTables}${omittedAssets} See \`truncation\` for per-article and per-section counts, and raise ${knobs.join(' or ')} or narrow \`sections\` to retrieve more.`;
|
|
613
1059
|
}
|
|
614
1060
|
/**
|
|
615
1061
|
* Compose the recovery notice for articles the whole-response budget withheld.
|
|
@@ -643,7 +1089,7 @@ function countOmittedSections(entry, mode) {
|
|
|
643
1089
|
* recoveries that silently can't happen.
|
|
644
1090
|
*/
|
|
645
1091
|
export function buildFulltextDescription(tiers) {
|
|
646
|
-
const base = 'Fetch full-text articles from PubMed Central with structured sections and references.';
|
|
1092
|
+
const base = 'Fetch full-text articles from PubMed Central with structured sections, tables, and references.';
|
|
647
1093
|
const epmcClause = 'Europe PMC `fullTextXML` (structured JATS for records with a PMC counterpart)';
|
|
648
1094
|
const unpaywallClause = 'Unpaywall — publisher-hosted or institutional open-access copies as HTML-as-Markdown or PDF-as-text';
|
|
649
1095
|
let fallback;
|
|
@@ -711,6 +1157,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
711
1157
|
.boolean()
|
|
712
1158
|
.default(false)
|
|
713
1159
|
.describe('Include reference list. Applies to `source=pmc` results only.'),
|
|
1160
|
+
includeTables: z
|
|
1161
|
+
.boolean()
|
|
1162
|
+
.default(true)
|
|
1163
|
+
.describe("Include the article's tables — cells, captions, labels and footnotes. On by default because a dropped table takes its numbers with it. Table-dense articles pay for it: rendered tables typically add 12–17% to an article record and can more than double it. Set false to omit them, or cap the cost with `maxCharacters`, which drops tables it cannot fit whole. Applies to `source=pmc` results only."),
|
|
1164
|
+
includeAssets: z
|
|
1165
|
+
.boolean()
|
|
1166
|
+
.default(true)
|
|
1167
|
+
.describe("Include the article's figures and supplementary material — `assets[]`, each with its label, caption, enclosing section and deposit pointer. On by default because it is cheaper than tables: a median asset-bearing article grows about 10%, and the body prose already refers to these by label. Set false to omit them, which also removes the `[Figure: …]` / `[Supplementary: …]` markers from the section text, since without the array they point at nothing. Prose-shaped blocks — lists, definition lists, block quotes, boxed text, preformatted blocks, displayed formulae — are section text rather than assets and this switch never affects them. Applies to `source=pmc` results only."),
|
|
714
1168
|
maxSections: z
|
|
715
1169
|
.number()
|
|
716
1170
|
.int()
|
|
@@ -721,14 +1175,14 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
721
1175
|
sections: z
|
|
722
1176
|
.array(z.string())
|
|
723
1177
|
.optional()
|
|
724
|
-
.describe('Filter to specific sections by title
|
|
1178
|
+
.describe('Filter to specific sections by title (e.g. ["Introduction", "Methods", "Results", "Discussion"]). A term matches a section or subsection title at any nesting depth, case-insensitively, as a substring — "resul" matches "Results". A section whose own title matches is returned whole; one kept only because a nested subsection matched keeps its heading as a breadcrumb, with its own text cleared and only the matching branch beneath it. Tables and assets narrow with the filter: one whose section did not survive, or that names no section, is dropped. Applies to `source=pmc` results only.'),
|
|
725
1179
|
maxCharacters: z
|
|
726
1180
|
.number()
|
|
727
1181
|
.int()
|
|
728
1182
|
.min(1)
|
|
729
1183
|
.max(1_000_000)
|
|
730
1184
|
.optional()
|
|
731
|
-
.describe('Per-article budget for body text, in characters. Counts `source=pmc` section and subsection text, or the `source=unpaywall` `content` body
|
|
1185
|
+
.describe('Per-article budget for body text, in characters. Counts `source=pmc` section and subsection text — which carries the inline blocks the parser renders in place, such as lists, definition lists, block quotes, boxed text, preformatted blocks and displayed formulae — plus table label, caption, cell and footnote text and asset label, caption and `href` text; or the `source=unpaywall` `content` body. Titles, abstracts, identifiers, and references are never counted or shortened. The counted unit is that text alone — the Markdown grid `content[]` renders around the cells (pipes, padding, the divider row, headings) is scaffolding this budget does not measure, so a table renders longer than it costs here. Sections are served first, then tables, then assets, each spending what is left, in document order — admission stops at the first entry that does not fit, and every entry from there on is dropped whole rather than cut mid-row or returned with a shortened caption, counted in `truncation.omittedTables` / `truncation.omittedAssets` and named in `truncation.articles[].omittedTableNames` / `omittedAssetNames`. Applied after `sections`, `maxSections`, `includeReferences`, `includeTables`, and `includeAssets`, so semantic filtering is unaffected. This knob alone bounds only bodies: the response-wide ceiling it implies is this value times the number of articles returned, plus every uncounted field. Use `maxResponseCharacters` for a true whole-response ceiling. Omit for the full body.'),
|
|
732
1186
|
maxCharactersPerSection: z
|
|
733
1187
|
.number()
|
|
734
1188
|
.int()
|
|
@@ -763,17 +1217,18 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
763
1217
|
truncation: TruncationSchema.optional(),
|
|
764
1218
|
deferred: DeferredSchema.optional(),
|
|
765
1219
|
}),
|
|
766
|
-
// Recovery guidance for
|
|
1220
|
+
// Recovery guidance for five cases — a `sections` filter that removed every
|
|
767
1221
|
// body section (#80), a record the chain could only retrieve as front matter
|
|
768
|
-
// (#86), a
|
|
769
|
-
//
|
|
1222
|
+
// (#86), a table returned with no cell values (#111), a body the per-article
|
|
1223
|
+
// character budget shortened (#81), and articles the whole-response budget
|
|
1224
|
+
// withheld (#100). Agent-facing context surfaced via
|
|
770
1225
|
// ctx.enrich.notice() to structuredContent and content[]; absent when none
|
|
771
1226
|
// applies.
|
|
772
1227
|
enrichment: {
|
|
773
1228
|
notice: z
|
|
774
1229
|
.string()
|
|
775
1230
|
.optional()
|
|
776
|
-
.describe('Optional guidance for a partial or empty body. A `sections`-filter miss names the requested terms and affected article id(s) and suggests retrying without `sections` or using broader headings. A metadata-only record names the id(s) the chain could retrieve as front matter only and points at `pubmed_fetch_articles` for the abstract. A budgeted response names the characters returned versus carried and points at `truncation`. A response-wide budget that deferred articles names the ids to re-request. Absent when none of those applies.'),
|
|
1231
|
+
.describe('Optional guidance for a partial or empty body. A `sections`-filter miss names the requested terms and affected article id(s) and suggests retrying without `sections` or using broader headings. A metadata-only record names the id(s) the chain could retrieve as front matter only and points at `pubmed_fetch_articles` for the abstract. A table returned with no cell values names the affected table(s), the article each came from, and why the cells cannot be recovered. A budgeted response names the characters returned versus carried and points at `truncation`. A response-wide budget that deferred articles names the ids to re-request. Absent when none of those applies.'),
|
|
777
1232
|
truncated: z
|
|
778
1233
|
.boolean()
|
|
779
1234
|
.optional()
|
|
@@ -1119,6 +1574,17 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1119
1574
|
// ── Stage 3: Unpaywall fallback ─────────────────────────────────────────
|
|
1120
1575
|
const unpaywall = getUnpaywallService();
|
|
1121
1576
|
const fallbackArticles = [];
|
|
1577
|
+
// Detail of a DOI-backfill lookup that threw, per branch. A candidate still
|
|
1578
|
+
// DOI-less after one of these is an unknown, not a settled absence: it
|
|
1579
|
+
// reports `doi-lookup-failed` rather than `no-doi`. (#119)
|
|
1580
|
+
let pmcidDoiLookupFailure;
|
|
1581
|
+
let pmidDoiLookupFailure;
|
|
1582
|
+
/** The Unpaywall tier entry for a candidate that reached the stage with no DOI. */
|
|
1583
|
+
const doilessTierEntry = (failure) => ({
|
|
1584
|
+
tier: 'unpaywall',
|
|
1585
|
+
outcome: failure ? 'doi-lookup-failed' : 'no-doi',
|
|
1586
|
+
...(failure && { detail: failure }),
|
|
1587
|
+
});
|
|
1122
1588
|
// `pmcids` input reaches Unpaywall on the DOI the chain already holds: the
|
|
1123
1589
|
// EPMC stage searches by PMCID and its hit carries one, captured on non-hit
|
|
1124
1590
|
// outcomes too. PMCIDs EPMC never resolved fall back to the PMC ID
|
|
@@ -1161,8 +1627,9 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1161
1627
|
});
|
|
1162
1628
|
}
|
|
1163
1629
|
catch (error) {
|
|
1630
|
+
pmcidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1164
1631
|
ctx.log.warning('Failed to resolve PMCID → DOI for the Unpaywall fallback', {
|
|
1165
|
-
error:
|
|
1632
|
+
error: pmcidDoiLookupFailure,
|
|
1166
1633
|
pmcidCount: needDoi.length,
|
|
1167
1634
|
});
|
|
1168
1635
|
}
|
|
@@ -1176,12 +1643,15 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1176
1643
|
pmcId,
|
|
1177
1644
|
result: candidate.doi
|
|
1178
1645
|
? await resolveUnpaywall({ pmcId, doi: candidate.doi, budget }, unpaywall, ctx)
|
|
1179
|
-
:
|
|
1646
|
+
: undefined,
|
|
1180
1647
|
};
|
|
1181
1648
|
}));
|
|
1182
1649
|
for (const { pmcId, result } of outcomes) {
|
|
1183
1650
|
const inputId = pmcidToInputId.get(pmcId) ?? pmcId;
|
|
1184
|
-
if (
|
|
1651
|
+
if (result === undefined) {
|
|
1652
|
+
chainByInput.get(inputId)?.push(doilessTierEntry(pmcidDoiLookupFailure));
|
|
1653
|
+
}
|
|
1654
|
+
else if ('article' in result) {
|
|
1185
1655
|
fallbackArticles.push(result.article);
|
|
1186
1656
|
inputIdByArticle.set(result.article, inputId);
|
|
1187
1657
|
if (result.truncation)
|
|
@@ -1215,19 +1685,21 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1215
1685
|
});
|
|
1216
1686
|
}
|
|
1217
1687
|
catch (error) {
|
|
1688
|
+
pmidDoiLookupFailure = error instanceof Error ? error.message : String(error);
|
|
1218
1689
|
ctx.log.warning('Failed to batch-fetch DOIs from PubMed for Unpaywall fallback', {
|
|
1219
|
-
error:
|
|
1690
|
+
error: pmidDoiLookupFailure,
|
|
1220
1691
|
pmidCount: needDoi.length,
|
|
1221
1692
|
});
|
|
1222
1693
|
}
|
|
1223
1694
|
}
|
|
1224
1695
|
if (!unpaywall) {
|
|
1225
|
-
// `fetchPubmedDois` has already run, so
|
|
1226
|
-
//
|
|
1227
|
-
//
|
|
1696
|
+
// `fetchPubmedDois` has already run, so a candidate with no DOI could
|
|
1697
|
+
// not have reached Unpaywall configured or not — that is `no-doi`, a
|
|
1698
|
+
// real answer, not an incomplete search. Unless the lookup itself
|
|
1699
|
+
// threw, in which case the DOI state is unknown rather than absent.
|
|
1228
1700
|
for (const c of pmidFallbackCandidates) {
|
|
1229
1701
|
if (!c.doi) {
|
|
1230
|
-
chainByInput.get(c.pmid)?.push(
|
|
1702
|
+
chainByInput.get(c.pmid)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
1231
1703
|
continue;
|
|
1232
1704
|
}
|
|
1233
1705
|
chainByInput.get(c.pmid)?.push({
|
|
@@ -1243,10 +1715,13 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1243
1715
|
candidate,
|
|
1244
1716
|
result: candidate.doi
|
|
1245
1717
|
? await resolveUnpaywall({ pmid: candidate.pmid, doi: candidate.doi, budget }, unpaywall, ctx)
|
|
1246
|
-
:
|
|
1718
|
+
: undefined,
|
|
1247
1719
|
})));
|
|
1248
1720
|
for (const { candidate, result } of outcomes) {
|
|
1249
|
-
if (
|
|
1721
|
+
if (result === undefined) {
|
|
1722
|
+
chainByInput.get(candidate.pmid)?.push(doilessTierEntry(pmidDoiLookupFailure));
|
|
1723
|
+
}
|
|
1724
|
+
else if ('article' in result) {
|
|
1250
1725
|
fallbackArticles.push(result.article);
|
|
1251
1726
|
inputIdByArticle.set(result.article, candidate.pmid);
|
|
1252
1727
|
if (result.truncation)
|
|
@@ -1361,6 +1836,11 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1361
1836
|
unavailable: unavailable.length,
|
|
1362
1837
|
...(deferred && { deferred: deferred.deferredCount }),
|
|
1363
1838
|
});
|
|
1839
|
+
// Summed off the per-article entries rather than carried through every
|
|
1840
|
+
// stage, so a deferred article's dropped tables leave the roll-up together
|
|
1841
|
+
// with its entry when the splice above removes it. (#111)
|
|
1842
|
+
const omittedTables = truncatedArticles.reduce((n, a) => n + (a.omittedTables ?? 0), 0);
|
|
1843
|
+
const omittedAssets = truncatedArticles.reduce((n, a) => n + (a.omittedAssets ?? 0), 0);
|
|
1364
1844
|
// Rolled up only when the budget actually removed characters, so an
|
|
1365
1845
|
// under-budget request returns exactly what it did before the budget
|
|
1366
1846
|
// controls existed. (#81)
|
|
@@ -1374,6 +1854,8 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1374
1854
|
originalCharacters: truncatedArticles.reduce((n, a) => n + a.originalCharacters, 0),
|
|
1375
1855
|
returnedCharacters: truncatedArticles.reduce((n, a) => n + a.returnedCharacters, 0),
|
|
1376
1856
|
omittedSections,
|
|
1857
|
+
...(omittedTables > 0 && { omittedTables }),
|
|
1858
|
+
...(omittedAssets > 0 && { omittedAssets }),
|
|
1377
1859
|
articles: truncatedArticles,
|
|
1378
1860
|
}
|
|
1379
1861
|
: undefined;
|
|
@@ -1386,6 +1868,10 @@ export const fetchFulltextTool = tool('pubmed_fetch_fulltext', {
|
|
|
1386
1868
|
const unrecoveredBodyless = [...bodylessInputIds].filter((id) => !recoveredIds.has(id));
|
|
1387
1869
|
if (unrecoveredBodyless.length > 0)
|
|
1388
1870
|
notices.push(buildBodylessNotice(unrecoveredBodyless));
|
|
1871
|
+
const unextractableTables = collectUnextractableTables(articles);
|
|
1872
|
+
if (unextractableTables.length > 0) {
|
|
1873
|
+
notices.push(buildUnextractableTablesNotice(unextractableTables));
|
|
1874
|
+
}
|
|
1389
1875
|
if (truncation) {
|
|
1390
1876
|
notices.push(buildTruncationNotice(truncation));
|
|
1391
1877
|
ctx.enrich({ truncated: true });
|
|
@@ -1541,8 +2027,11 @@ async function runEpmcStage(epmc, args) {
|
|
|
1541
2027
|
pmidOutcomes.set(run.c.pmid, run.outcome);
|
|
1542
2028
|
if (run.article)
|
|
1543
2029
|
collectHit(run.c.pmid, { ...run, article: run.article });
|
|
2030
|
+
// A DOI the EPMC hit carried is evidence the next stage needs, whatever the
|
|
2031
|
+
// fetch outcome was — merge it in like the pmcid branch below, so Unpaywall
|
|
2032
|
+
// gets it without a redundant PubMed metadata round-trip. (#119)
|
|
1544
2033
|
else
|
|
1545
|
-
remainingPmid.push(run.c);
|
|
2034
|
+
remainingPmid.push(run.doi && !run.c.doi ? { ...run.c, doi: run.doi } : run.c);
|
|
1546
2035
|
}
|
|
1547
2036
|
for (const run of pmcidResults) {
|
|
1548
2037
|
pmcidOutcomes.set(run.c.normalized, run.outcome);
|
|
@@ -1849,6 +2338,7 @@ function unpaywallReasonToTierOutcome(reason) {
|
|
|
1849
2338
|
switch (reason) {
|
|
1850
2339
|
case 'no-body':
|
|
1851
2340
|
case 'no-doi':
|
|
2341
|
+
case 'doi-lookup-failed':
|
|
1852
2342
|
case 'no-oa':
|
|
1853
2343
|
case 'fetch-failed':
|
|
1854
2344
|
case 'parse-failed':
|
|
@@ -1900,6 +2390,11 @@ function reasonFromChain(chain) {
|
|
|
1900
2390
|
return 'no-body';
|
|
1901
2391
|
case 'unpaywall:no-doi':
|
|
1902
2392
|
return 'no-doi';
|
|
2393
|
+
// Its own case, never folded into the `unpaywall:no-doi` → `not-found`
|
|
2394
|
+
// collapse above: that collapse says record absence is the specific signal,
|
|
2395
|
+
// and a lookup that never answered has established no absence at all. (#119)
|
|
2396
|
+
case 'unpaywall:doi-lookup-failed':
|
|
2397
|
+
return 'doi-lookup-failed';
|
|
1903
2398
|
case 'unpaywall:no-oa':
|
|
1904
2399
|
return 'no-oa';
|
|
1905
2400
|
case 'unpaywall:fetch-failed':
|
|
@@ -1942,7 +2437,9 @@ function formatUnqueriedTiers(tiers, chain) {
|
|
|
1942
2437
|
* separators — so the numbers stay greppable. (#81)
|
|
1943
2438
|
*/
|
|
1944
2439
|
function formatTruncation(t, lines) {
|
|
1945
|
-
|
|
2440
|
+
const tablesOmitted = t.omittedTables === undefined ? '' : `; ${t.omittedTables} table(s) omitted whole`;
|
|
2441
|
+
const assetsOmitted = t.omittedAssets === undefined ? '' : `; ${t.omittedAssets} asset(s) omitted whole`;
|
|
2442
|
+
lines.push(`\n**Truncated (${t.mode} mode):** ${t.returnedCharacters} of ${t.originalCharacters} body characters returned across ${t.articles.length} article(s); ${t.omittedSections} section(s) omitted${tablesOmitted}${assetsOmitted}`);
|
|
1946
2443
|
const budgets = [
|
|
1947
2444
|
t.maxCharacters === undefined ? undefined : `maxCharacters ${t.maxCharacters}`,
|
|
1948
2445
|
t.maxCharactersPerSection === undefined
|
|
@@ -1952,7 +2449,13 @@ function formatTruncation(t, lines) {
|
|
|
1952
2449
|
if (budgets.length)
|
|
1953
2450
|
lines.push(`Budget applied: ${budgets.join(', ')}`);
|
|
1954
2451
|
for (const a of t.articles) {
|
|
1955
|
-
|
|
2452
|
+
const tablesDropped = a.omittedTables === undefined
|
|
2453
|
+
? ''
|
|
2454
|
+
: `, ${a.omittedTables} table(s) dropped whole: ${(a.omittedTableNames ?? []).join(', ')}`;
|
|
2455
|
+
const assetsDropped = a.omittedAssets === undefined
|
|
2456
|
+
? ''
|
|
2457
|
+
: `, ${a.omittedAssets} asset(s) dropped whole: ${(a.omittedAssetNames ?? []).join(', ')}`;
|
|
2458
|
+
lines.push(`- ${a.id} (${a.source}): ${a.returnedCharacters} of ${a.originalCharacters} characters${tablesDropped}${assetsDropped}`);
|
|
1956
2459
|
for (const s of a.sections ?? []) {
|
|
1957
2460
|
lines.push(` - ${s.title ?? 'untitled section'} — ${s.returnedCharacters} of ${s.originalCharacters} characters (truncated: ${s.truncated})`);
|
|
1958
2461
|
}
|
|
@@ -2021,6 +2524,10 @@ function formatPmcArticle(a, lines, truncation) {
|
|
|
2021
2524
|
lines.push(`\n#### Abstract\n${a.abstract}`);
|
|
2022
2525
|
for (const sec of a.sections)
|
|
2023
2526
|
formatSection(sec, lines, 4);
|
|
2527
|
+
if (a.tables?.length)
|
|
2528
|
+
formatTables(a.tables, lines);
|
|
2529
|
+
if (a.assets?.length)
|
|
2530
|
+
formatAssets(a.assets, lines);
|
|
2024
2531
|
if (a.references?.length) {
|
|
2025
2532
|
lines.push(`\n#### References (${a.references.length})`);
|
|
2026
2533
|
for (const ref of a.references) {
|
|
@@ -2029,6 +2536,112 @@ function formatPmcArticle(a, lines, truncation) {
|
|
|
2029
2536
|
}
|
|
2030
2537
|
}
|
|
2031
2538
|
}
|
|
2539
|
+
/**
|
|
2540
|
+
* Render every table as a Markdown grid, so a `content[]` reader gets the same
|
|
2541
|
+
* cells `structuredContent` carries rather than a note that tables exist. (#111)
|
|
2542
|
+
*
|
|
2543
|
+
* Cell text goes through {@link escapeMarkdownTableCell} — the inline escape
|
|
2544
|
+
* plus the `|` a cell cannot carry raw. The parser expands `colspan` and
|
|
2545
|
+
* `rowspan`, so a well-formed table arrives rectangular and every value renders
|
|
2546
|
+
* under the header it belongs to; a row still short of the widest is padded on
|
|
2547
|
+
* the right with empty cells only, never a neighbour's value.
|
|
2548
|
+
*/
|
|
2549
|
+
function formatTables(tables, lines) {
|
|
2550
|
+
lines.push(`\n#### Tables (${tables.length})`);
|
|
2551
|
+
for (const table of tables) {
|
|
2552
|
+
const heading = [table.label, table.caption].filter(Boolean).join(' — ');
|
|
2553
|
+
lines.push(`\n##### ${escapeMarkdownInline(heading || 'Table')}`);
|
|
2554
|
+
const meta = [
|
|
2555
|
+
table.sectionTitle ? `Section: ${escapeMarkdownInline(table.sectionTitle)}` : undefined,
|
|
2556
|
+
table.id ? `id: ${escapeMarkdownInline(table.id)}` : undefined,
|
|
2557
|
+
describeHeaderRows(table),
|
|
2558
|
+
].filter((part) => part !== undefined);
|
|
2559
|
+
if (meta.length)
|
|
2560
|
+
lines.push(`*${meta.join(' · ')}*`);
|
|
2561
|
+
if (table.unextractableReason) {
|
|
2562
|
+
lines.push(`\n> Table body could not be read (${table.unextractableReason}) — ${UNEXTRACTABLE_TABLE_EXPLANATIONS[table.unextractableReason]}. The label and caption above are all this deposit carries; no cell values exist to return.`);
|
|
2563
|
+
}
|
|
2564
|
+
if (table.rows.length > 0)
|
|
2565
|
+
lines.push(...renderTableGrid(table.rows, table.headerRowCount));
|
|
2566
|
+
if (table.footnotes)
|
|
2567
|
+
lines.push(`\nFootnotes: ${escapeMarkdownInline(table.footnotes)}`);
|
|
2568
|
+
}
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Render every figure and supplementary item, so a `content[]` reader gets the
|
|
2572
|
+
* same caption, pointer and placement `structuredContent` carries rather than a
|
|
2573
|
+
* count saying they exist. Each field of `AssetSchema` appears here — that is
|
|
2574
|
+
* what `format-parity` verifies. (#130)
|
|
2575
|
+
*
|
|
2576
|
+
* Label, caption and `href` go through {@link escapeMarkdownInline}: the parser
|
|
2577
|
+
* stores the raw upstream text, and a caption carrying `[`, `*` or a tag-shaped
|
|
2578
|
+
* `<` would otherwise form a link, emphasis or raw HTML at the render boundary.
|
|
2579
|
+
*
|
|
2580
|
+
* The note about `href` is worth its line: the value is a filename inside the PMC
|
|
2581
|
+
* deposit, and an agent that reads it as a URL will spend a request on a 404 for
|
|
2582
|
+
* every figure in the article.
|
|
2583
|
+
*/
|
|
2584
|
+
function formatAssets(assets, lines) {
|
|
2585
|
+
lines.push(`\n#### Assets (${assets.length})`);
|
|
2586
|
+
lines.push(`\n> \`file\` is the pointer exactly as deposited — a name inside the PMC deposit, not a fetchable URL. Open the article at the PMC link above to view it.`);
|
|
2587
|
+
for (const asset of assets) {
|
|
2588
|
+
const heading = [asset.label, asset.caption].filter(Boolean).join(' — ');
|
|
2589
|
+
lines.push(`\n##### ${escapeMarkdownInline(heading || 'Asset')}`);
|
|
2590
|
+
const meta = [
|
|
2591
|
+
asset.assetType,
|
|
2592
|
+
asset.sectionTitle ? `Section: ${escapeMarkdownInline(asset.sectionTitle)}` : undefined,
|
|
2593
|
+
asset.id ? `id: ${escapeMarkdownInline(asset.id)}` : undefined,
|
|
2594
|
+
asset.href ? `file: ${escapeMarkdownInline(asset.href)}` : undefined,
|
|
2595
|
+
].filter((part) => part !== undefined);
|
|
2596
|
+
lines.push(`*${meta.join(' · ')}*`);
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
/**
|
|
2600
|
+
* The meta-line clause describing a table's header rows.
|
|
2601
|
+
*
|
|
2602
|
+
* A Markdown grid carries exactly one header row, so a table declaring several
|
|
2603
|
+
* has them folded into it — say how many were folded, or the grid understates
|
|
2604
|
+
* what the deposit declared. A table declaring none still needs the empty header
|
|
2605
|
+
* row Markdown requires above the divider; naming that keeps a reader from
|
|
2606
|
+
* taking the blank row for a header the publisher deposited and left empty.
|
|
2607
|
+
*/
|
|
2608
|
+
function describeHeaderRows(table) {
|
|
2609
|
+
if (table.rows.length === 0)
|
|
2610
|
+
return;
|
|
2611
|
+
if (table.headerRowCount === 0)
|
|
2612
|
+
return 'header rows: none declared — every row below is data';
|
|
2613
|
+
if (table.headerRowCount === 1)
|
|
2614
|
+
return 'header rows: 1';
|
|
2615
|
+
return `header rows: ${table.headerRowCount} (folded into one)`;
|
|
2616
|
+
}
|
|
2617
|
+
/**
|
|
2618
|
+
* Join one grid column's header cells into the single header path Markdown can
|
|
2619
|
+
* carry. Consecutive repeats — what expanding a `colspan` produces — collapse to
|
|
2620
|
+
* one, so a group header spanning three columns reads once per column rather
|
|
2621
|
+
* than three times in each.
|
|
2622
|
+
*/
|
|
2623
|
+
function foldHeaderColumn(headerRows, column) {
|
|
2624
|
+
const path = [];
|
|
2625
|
+
for (const row of headerRows) {
|
|
2626
|
+
const cell = row[column] ?? '';
|
|
2627
|
+
if (cell && cell !== path.at(-1))
|
|
2628
|
+
path.push(cell);
|
|
2629
|
+
}
|
|
2630
|
+
return path.join(' · ');
|
|
2631
|
+
}
|
|
2632
|
+
/** One table's rows as Markdown grid lines, preceded by a blank line. */
|
|
2633
|
+
function renderTableGrid(rows, headerRowCount) {
|
|
2634
|
+
const columns = rows.reduce((widest, row) => Math.max(widest, row.length), 0);
|
|
2635
|
+
const renderRow = (cells) => `| ${Array.from({ length: columns }, (_, i) => escapeMarkdownTableCell(cells[i] ?? '')).join(' | ')} |`;
|
|
2636
|
+
const headerRows = rows.slice(0, headerRowCount);
|
|
2637
|
+
const header = Array.from({ length: columns }, (_, i) => foldHeaderColumn(headerRows, i));
|
|
2638
|
+
return [
|
|
2639
|
+
'',
|
|
2640
|
+
renderRow(header),
|
|
2641
|
+
`| ${Array.from({ length: columns }, () => '---').join(' | ')} |`,
|
|
2642
|
+
...rows.slice(headerRowCount).map(renderRow),
|
|
2643
|
+
];
|
|
2644
|
+
}
|
|
2032
2645
|
function formatUnpaywallArticle(a, lines, truncation) {
|
|
2033
2646
|
const requestedId = a.pmcId ? `PMCID ${a.pmcId}` : a.pmid ? `PMID ${a.pmid}` : `DOI ${a.doi}`;
|
|
2034
2647
|
const heading = a.title ?? requestedId;
|