@cyanheads/biorxiv-mcp-server 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +17 -13
- package/CLAUDE.md +17 -13
- package/Dockerfile +3 -3
- package/README.md +34 -25
- package/changelog/0.4.x/0.4.0.md +33 -0
- package/changelog/0.4.x/0.4.1.md +27 -0
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-fulltext.tool.d.ts +12 -5
- package/dist/mcp-server/tools/definitions/biorxiv-get-fulltext.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-fulltext.tool.js +113 -29
- package/dist/mcp-server/tools/definitions/biorxiv-get-fulltext.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-preprint.tool.d.ts +21 -9
- package/dist/mcp-server/tools/definitions/biorxiv-get-preprint.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-preprint.tool.js +130 -49
- package/dist/mcp-server/tools/definitions/biorxiv-get-preprint.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-published-version.tool.d.ts +4 -2
- package/dist/mcp-server/tools/definitions/biorxiv-get-published-version.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-get-published-version.tool.js +33 -20
- package/dist/mcp-server/tools/definitions/biorxiv-get-published-version.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-list-published-versions.tool.d.ts +102 -0
- package/dist/mcp-server/tools/definitions/biorxiv-list-published-versions.tool.d.ts.map +1 -0
- package/dist/mcp-server/tools/definitions/biorxiv-list-published-versions.tool.js +372 -0
- package/dist/mcp-server/tools/definitions/biorxiv-list-published-versions.tool.js.map +1 -0
- package/dist/mcp-server/tools/definitions/biorxiv-list-recent.tool.d.ts +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-list-recent.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-list-recent.tool.js +35 -21
- package/dist/mcp-server/tools/definitions/biorxiv-list-recent.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-search-preprints.tool.d.ts +18 -6
- package/dist/mcp-server/tools/definitions/biorxiv-search-preprints.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/biorxiv-search-preprints.tool.js +78 -48
- package/dist/mcp-server/tools/definitions/biorxiv-search-preprints.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +101 -15
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/index.js +3 -0
- package/dist/mcp-server/tools/definitions/index.js.map +1 -1
- package/dist/services/biorxiv/biorxiv-service.d.ts +33 -8
- package/dist/services/biorxiv/biorxiv-service.d.ts.map +1 -1
- package/dist/services/biorxiv/biorxiv-service.js +91 -24
- package/dist/services/biorxiv/biorxiv-service.js.map +1 -1
- package/dist/services/biorxiv/types.d.ts +25 -3
- package/dist/services/biorxiv/types.d.ts.map +1 -1
- package/dist/services/biorxiv-fulltext/biorxiv-fulltext-service.d.ts +22 -5
- package/dist/services/biorxiv-fulltext/biorxiv-fulltext-service.d.ts.map +1 -1
- package/dist/services/biorxiv-fulltext/biorxiv-fulltext-service.js +99 -17
- package/dist/services/biorxiv-fulltext/biorxiv-fulltext-service.js.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.d.ts +7 -3
- package/dist/services/europe-pmc/europe-pmc-service.d.ts.map +1 -1
- package/dist/services/europe-pmc/europe-pmc-service.js +20 -13
- package/dist/services/europe-pmc/europe-pmc-service.js.map +1 -1
- package/dist/services/shared.d.ts +81 -5
- package/dist/services/shared.d.ts.map +1 -1
- package/dist/services/shared.js +130 -13
- package/dist/services/shared.js.map +1 -1
- package/package.json +8 -8
- package/server.json +3 -3
|
@@ -59,7 +59,7 @@ export interface RawDetailsResponse {
|
|
|
59
59
|
count_new_papers?: number | string;
|
|
60
60
|
}>;
|
|
61
61
|
}
|
|
62
|
-
/** A single crosswalk record from /pubs
|
|
62
|
+
/** A single crosswalk record from /pubs — keyed by DOI or listed by publication-date interval */
|
|
63
63
|
export interface RawPublishedRecord {
|
|
64
64
|
preprint_abstract?: string;
|
|
65
65
|
preprint_author_corresponding?: string;
|
|
@@ -73,13 +73,19 @@ export interface RawPublishedRecord {
|
|
|
73
73
|
published_doi?: string;
|
|
74
74
|
published_journal?: string;
|
|
75
75
|
}
|
|
76
|
-
/**
|
|
76
|
+
/**
|
|
77
|
+
* Envelope returned by /pubs. An interval page reports `total` as a string
|
|
78
|
+
* ("449") and echoes `cursor` as a number at 0 and a string after it; a page
|
|
79
|
+
* with no rows ("no articles found …") carries `status` alone.
|
|
80
|
+
*/
|
|
77
81
|
export interface RawPublishedResponse {
|
|
78
82
|
collection?: RawPublishedRecord[];
|
|
79
83
|
messages?: Array<{
|
|
80
84
|
status?: string;
|
|
81
|
-
|
|
85
|
+
interval?: string;
|
|
86
|
+
total?: number | string;
|
|
82
87
|
count?: number;
|
|
88
|
+
cursor?: number | string;
|
|
83
89
|
}>;
|
|
84
90
|
}
|
|
85
91
|
/** A single preprint revision with normalized optional fields */
|
|
@@ -144,6 +150,22 @@ export interface PublishedVersion {
|
|
|
144
150
|
publishedDoi?: string;
|
|
145
151
|
publishedJournal?: string;
|
|
146
152
|
}
|
|
153
|
+
/** Per-server result from the /pubs publication-date interval listing */
|
|
154
|
+
export interface PublishedListingResult {
|
|
155
|
+
pagination: {
|
|
156
|
+
cursor: number;
|
|
157
|
+
/** Rows in the interval; absent when the page held none, since upstream then reports no total */
|
|
158
|
+
total?: number;
|
|
159
|
+
};
|
|
160
|
+
publications: PublishedVersion[];
|
|
161
|
+
/**
|
|
162
|
+
* Upstream rows this page served — the requested slice of the upstream
|
|
163
|
+
* page, counting any dropped for naming no preprint. The cursor plus this
|
|
164
|
+
* count is the offset the next page starts from; zero only when the page
|
|
165
|
+
* was empty upstream.
|
|
166
|
+
*/
|
|
167
|
+
rowCount: number;
|
|
168
|
+
}
|
|
147
169
|
/** Hardcoded subject category taxonomy for bioRxiv and medRxiv */
|
|
148
170
|
export interface CategoryTaxonomy {
|
|
149
171
|
biorxiv: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/biorxiv/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,oDAAoD;AACpD,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,SAAS,CAAC;AAElD,8EAA8E;AAC9E,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,MAAM,CAAC;AAIjD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gCAAgC,CAAC,EAAE,MAAM,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,mDAAmD;AACnD,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,+DAA+D;QAC/D,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,8FAA8F;QAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;KACpC,CAAC,CAAC;CACJ;AAED,
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/biorxiv/types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,oDAAoD;AACpD,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,SAAS,CAAC;AAElD,8EAA8E;AAC9E,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,MAAM,CAAC;AAIjD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,gCAAgC,CAAC,EAAE,MAAM,CAAC;IAC1C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,GAAG,cAAc,EAAE,CAAC;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,mDAAmD;AACnD,MAAM,WAAW,kBAAkB;IACjC,UAAU,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACnC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,+DAA+D;QAC/D,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACzB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,8FAA8F;QAC9F,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,gBAAgB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;KACpC,CAAC,CAAC;CACJ;AAED,iGAAiG;AACjG,MAAM,WAAW,kBAAkB;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,yCAAyC,CAAC,EAAE,MAAM,CAAC;IACnD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,kBAAkB,EAAE,CAAC;IAClC,QAAQ,CAAC,EAAE,KAAK,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;KAC1B,CAAC,CAAC;CACJ;AAID,iEAAiE;AACjE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,8BAA8B,CAAC,EAAE,MAAM,CAAC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,8CAA8C;AAC9C,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,iEAAiE;IACjE,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,kDAAkD;AAClD,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,eAAe,CAAC,EAAE,IAAI,CAAC;IACvB;;;;OAIG;IACH,cAAc,CAAC,EAAE,IAAI,CAAC;IACtB,UAAU,EAAE,qBAAqB,CAAC;IAClC,SAAS,EAAE,gBAAgB,EAAE,CAAC;CAC/B;AAED,+CAA+C;AAC/C,MAAM,WAAW,gBAAgB;IAC/B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2BAA2B,CAAC,EAAE,MAAM,CAAC;IACrC,sCAAsC,CAAC,EAAE,MAAM,CAAC;IAChD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,yEAAyE;AACzE,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE;QACV,MAAM,EAAE,MAAM,CAAC;QACf,iGAAiG;QACjG,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,YAAY,EAAE,gBAAgB,EAAE,CAAC;IACjC;;;;;OAKG;IACH,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,kEAAkE;AAClE,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB"}
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview BiorxivFullTextService — fetches a preprint's rendered full-text
|
|
3
3
|
* HTML page from the bioRxiv/medRxiv website (`www.{server}.org/content/{doi}v{N}.full`)
|
|
4
|
-
* and extracts readable Markdown via the framework HTML extractor.
|
|
4
|
+
* and extracts readable Markdown via the framework HTML extractor. The page is
|
|
5
|
+
* reshaped first (`prepareArticlePage`) so the extractor keeps inline citation
|
|
6
|
+
* labels and reference text, and the tables the page carries no cells for are
|
|
7
|
+
* returned by name with the URL that serves them.
|
|
5
8
|
*
|
|
6
9
|
* This is a DISTINCT fetch path from BiorxivApiService: it EXPECTS an HTML
|
|
7
10
|
* document (the rendered article page), so it never routes through the JSON
|
|
@@ -11,7 +14,7 @@
|
|
|
11
14
|
* the extractor.
|
|
12
15
|
*
|
|
13
16
|
* Extracted articles are cached in `ctx.state` under
|
|
14
|
-
* `fulltext/
|
|
17
|
+
* `fulltext/v3/{server}/{doi}/{version}` for {@link FULLTEXT_CACHE_TTL_SECONDS}
|
|
15
18
|
* seconds, so paging a long article with offset/limit costs one origin fetch and
|
|
16
19
|
* one extraction instead of one per chunk. `ctx.state` is tenant-scoped, which is
|
|
17
20
|
* the intended blast radius — no cross-tenant or process-wide sharing. Only
|
|
@@ -25,6 +28,17 @@ import type { Context } from '@cyanheads/mcp-ts-core';
|
|
|
25
28
|
import type { AppConfig } from '@cyanheads/mcp-ts-core/config';
|
|
26
29
|
import type { StorageService } from '@cyanheads/mcp-ts-core/storage';
|
|
27
30
|
import type { BiorxivServer } from '../biorxiv/types.js';
|
|
31
|
+
import { type CallDeadline } from '../shared.js';
|
|
32
|
+
/**
|
|
33
|
+
* A table the article page names but carries no cells for. Highwire serves a
|
|
34
|
+
* table's cells from a separate expansion page; `url` is that page.
|
|
35
|
+
*/
|
|
36
|
+
export interface OmittedTable {
|
|
37
|
+
/** The table's label as the page prints it, without trailing punctuation (`Table 1`). */
|
|
38
|
+
label: string;
|
|
39
|
+
/** Absolute URL of the origin page that serves this table's cells. */
|
|
40
|
+
url: string;
|
|
41
|
+
}
|
|
28
42
|
/**
|
|
29
43
|
* Discriminated result of a full-text fetch. `article` carries the extracted
|
|
30
44
|
* Markdown plus best-effort metadata; `unavailable` records why no readable full
|
|
@@ -40,6 +54,8 @@ export type FullTextFetchResult = {
|
|
|
40
54
|
markdown: string;
|
|
41
55
|
title?: string;
|
|
42
56
|
sourceUrl: string;
|
|
57
|
+
/** Tables the page names without cells, in page order; absent when none. */
|
|
58
|
+
omittedTables?: OmittedTable[];
|
|
43
59
|
} | {
|
|
44
60
|
kind: 'unavailable';
|
|
45
61
|
reason: 'blocked' | 'empty' | 'rate_limited';
|
|
@@ -63,14 +79,15 @@ export declare class BiorxivFullTextService {
|
|
|
63
79
|
* Resolves to `{ kind: 'article', markdown, ... }` on success, or
|
|
64
80
|
* `{ kind: 'unavailable', reason }` when the page is blocked (Cloudflare 403 /
|
|
65
81
|
* challenge), rate-limited (429), missing, or yields no extractable text
|
|
66
|
-
* (PDF-only preprints). Transient failures (5xx, timeout, network) bubble
|
|
67
|
-
*
|
|
82
|
+
* (PDF-only preprints). Transient failures (5xx, timeout, network) bubble
|
|
83
|
+
* unchanged after retry — `ServiceUnavailable`, or a `Timeout` once `deadline`
|
|
84
|
+
* runs out — for the caller to classify.
|
|
68
85
|
*
|
|
69
86
|
* A successful extraction is served from the tenant-scoped `ctx.state` cache on
|
|
70
87
|
* subsequent calls for the same server/DOI/version, so offset paging does not
|
|
71
88
|
* refetch the origin per chunk.
|
|
72
89
|
*/
|
|
73
|
-
fetchFullText(server: BiorxivServer, doi: string, version: string, ctx: Context): Promise<FullTextFetchResult>;
|
|
90
|
+
fetchFullText(server: BiorxivServer, doi: string, version: string, deadline: CallDeadline, ctx: Context): Promise<FullTextFetchResult>;
|
|
74
91
|
}
|
|
75
92
|
export declare function initBiorxivFullTextService(config: AppConfig, storage: StorageService): void;
|
|
76
93
|
export declare function getBiorxivFullTextService(): BiorxivFullTextService;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"biorxiv-fulltext-service.d.ts","sourceRoot":"","sources":["../../../src/services/biorxiv-fulltext/biorxiv-fulltext-service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"biorxiv-fulltext-service.d.ts","sourceRoot":"","sources":["../../../src/services/biorxiv-fulltext/biorxiv-fulltext-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAE/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAIrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AACjE,OAAO,EACL,KAAK,YAAY,EAIlB,MAAM,sBAAsB,CAAC;AAmL9B;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B,yFAAyF;IACzF,KAAK,EAAE,MAAM,CAAC;IACd,sEAAsE;IACtE,GAAG,EAAE,MAAM,CAAC;CACb;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,mBAAmB,GAC3B;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,4EAA4E;IAC5E,aAAa,CAAC,EAAE,YAAY,EAAE,CAAC;CAChC,GACD;IACE,IAAI,EAAE,aAAa,CAAC;IACpB,MAAM,EAAE,SAAS,GAAG,OAAO,GAAG,cAAc,CAAC;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf;;;;;OAKG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAKN,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgC;IAC5D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,YAAY,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EASvD;IAED;;;;;;;;;;;;;OAaG;IACG,aAAa,CACjB,MAAM,EAAE,aAAa,EACrB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,mBAAmB,CAAC,CAyH9B;CACF;AAMD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAE3F;AAED,wBAAgB,yBAAyB,IAAI,sBAAsB,CAMlE"}
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview BiorxivFullTextService — fetches a preprint's rendered full-text
|
|
3
3
|
* HTML page from the bioRxiv/medRxiv website (`www.{server}.org/content/{doi}v{N}.full`)
|
|
4
|
-
* and extracts readable Markdown via the framework HTML extractor.
|
|
4
|
+
* and extracts readable Markdown via the framework HTML extractor. The page is
|
|
5
|
+
* reshaped first (`prepareArticlePage`) so the extractor keeps inline citation
|
|
6
|
+
* labels and reference text, and the tables the page carries no cells for are
|
|
7
|
+
* returned by name with the URL that serves them.
|
|
5
8
|
*
|
|
6
9
|
* This is a DISTINCT fetch path from BiorxivApiService: it EXPECTS an HTML
|
|
7
10
|
* document (the rendered article page), so it never routes through the JSON
|
|
@@ -11,7 +14,7 @@
|
|
|
11
14
|
* the extractor.
|
|
12
15
|
*
|
|
13
16
|
* Extracted articles are cached in `ctx.state` under
|
|
14
|
-
* `fulltext/
|
|
17
|
+
* `fulltext/v3/{server}/{doi}/{version}` for {@link FULLTEXT_CACHE_TTL_SECONDS}
|
|
15
18
|
* seconds, so paging a long article with offset/limit costs one origin fetch and
|
|
16
19
|
* one extraction instead of one per chunk. `ctx.state` is tenant-scoped, which is
|
|
17
20
|
* the intended blast radius — no cross-tenant or process-wide sharing. Only
|
|
@@ -22,9 +25,10 @@
|
|
|
22
25
|
* @module services/biorxiv-fulltext/biorxiv-fulltext-service
|
|
23
26
|
*/
|
|
24
27
|
import { McpError } from '@cyanheads/mcp-ts-core/errors';
|
|
25
|
-
import { fetchWithTimeout, htmlExtractor
|
|
28
|
+
import { fetchWithTimeout, htmlExtractor } from '@cyanheads/mcp-ts-core/utils';
|
|
29
|
+
import { parseHTML } from 'linkedom';
|
|
26
30
|
import { getServerConfig } from '../../config/server-config.js';
|
|
27
|
-
import { parseRetryAfterSeconds, SERVER_VERSION } from '../shared.js';
|
|
31
|
+
import { parseRetryAfterSeconds, retryWithinBudget, SERVER_VERSION, } from '../shared.js';
|
|
28
32
|
/**
|
|
29
33
|
* bioRxiv/medRxiv render their article pages on the Highwire platform, whose
|
|
30
34
|
* main full-text container is `<div class="article fulltext-view">`. Defuddle's
|
|
@@ -57,14 +61,87 @@ const EXPECTED_STATUSES = [...DETERMINISTIC_UNAVAILABLE_STATUSES, 429];
|
|
|
57
61
|
*/
|
|
58
62
|
const FULLTEXT_CACHE_TTL_SECONDS = 3_600;
|
|
59
63
|
/**
|
|
60
|
-
* Cache key: `fulltext/
|
|
64
|
+
* Cache key: `fulltext/v3/{server}/{doi}/{version}`. Slash-delimited because the
|
|
61
65
|
* storage layer validates keys against `[a-zA-Z0-9_.\-/]` — a colon separator is
|
|
62
66
|
* rejected outright, and a DOI's own `/` and `.` are already inside the allowed
|
|
63
|
-
* set. The `
|
|
67
|
+
* set. The `v3` segment is the cached-value schema generation — bump it when
|
|
64
68
|
* {@link CachedArticle} changes so entries written by an older build are never
|
|
65
|
-
* read back under the new shape. (`v2` dropped the extractor's `wordCount
|
|
69
|
+
* read back under the new shape. (`v2` dropped the extractor's `wordCount`; `v3`
|
|
70
|
+
* added `omittedTables` and keeps citation labels and reference text.)
|
|
66
71
|
*/
|
|
67
|
-
const cacheKeyFor = (server, doi, version) => `fulltext/
|
|
72
|
+
const cacheKeyFor = (server, doi, version) => `fulltext/v3/${server}/${doi}/${version}`;
|
|
73
|
+
/**
|
|
74
|
+
* Rewrites the article container so the extractor keeps what its cleanup rules
|
|
75
|
+
* strip from Highwire pages, and lists the tables the page carries no cells for.
|
|
76
|
+
* Defuddle removes every `a[href^="#"]` whose class contains `ref` — every
|
|
77
|
+
* `a.xref-bibr` / `a.xref-fig` / `a.xref-table` label — leaving `()` behind; its
|
|
78
|
+
* footnote pass renumbers numeric citations (`[^1]–[^3]` for `1–16`); and its
|
|
79
|
+
* `metadata` partial selector removes each reference's `div.cit-metadata`. The
|
|
80
|
+
* framework extractor exposes none of those switches, and turning them off
|
|
81
|
+
* globally would re-admit the page chrome they exist to strip, so the page is
|
|
82
|
+
* reshaped here instead:
|
|
83
|
+
*
|
|
84
|
+
* - each cross-reference anchor becomes its own label text;
|
|
85
|
+
* - in the reference list, the `↵` backlinks go, every `.cit-extra` link-out but
|
|
86
|
+
* the DOI one goes (the OpenURL ones are an unexpanded `{openurl}` template,
|
|
87
|
+
* the PubMed ones often a truncated `access_num`), the DOI one becomes a
|
|
88
|
+
* `doi.org` link labelled with the DOI, and every `class` and `id` is dropped
|
|
89
|
+
* so no selector matches the entries;
|
|
90
|
+
* - every `div.table` loses its callout links ("View popup", "Download
|
|
91
|
+
* powerpoint"). One without a `<table>` holds only a caption and a link to the
|
|
92
|
+
* origin's expansion page, where the cells live: the caption stays in place,
|
|
93
|
+
* and the table is returned with that page's URL, under its printed label, or
|
|
94
|
+
* its `id` when it prints none. The cells are never fetched: each would cost
|
|
95
|
+
* a request to an origin that rate-limits after a handful.
|
|
96
|
+
*
|
|
97
|
+
* A page without the container is returned unchanged. An `href` that does not
|
|
98
|
+
* parse as a URL counts as no link: the page is upstream markup, and one bad
|
|
99
|
+
* link must not cost the caller the whole article.
|
|
100
|
+
*/
|
|
101
|
+
function prepareArticlePage(html, sourceUrl) {
|
|
102
|
+
const { document } = parseHTML(html);
|
|
103
|
+
const view = document.querySelector(FULLTEXT_CONTENT_SELECTOR);
|
|
104
|
+
if (!view)
|
|
105
|
+
return { html, omittedTables: [] };
|
|
106
|
+
for (const anchor of view.querySelectorAll('a[class^="xref-"]')) {
|
|
107
|
+
anchor.replaceWith(document.createTextNode(anchor.textContent ?? ''));
|
|
108
|
+
}
|
|
109
|
+
for (const refList of view.querySelectorAll('div.ref-list')) {
|
|
110
|
+
for (const backlink of refList.querySelectorAll('a.rev-xref-ref'))
|
|
111
|
+
backlink.remove();
|
|
112
|
+
for (const extra of refList.querySelectorAll('.cit-extra')) {
|
|
113
|
+
for (const link of extra.querySelectorAll('a')) {
|
|
114
|
+
const doi = link.classList.contains('cit-ref-sprinkles-doi')
|
|
115
|
+
? URL.parse(link.getAttribute('href') ?? '', sourceUrl)?.searchParams.get('access_num')
|
|
116
|
+
: null;
|
|
117
|
+
if (!doi) {
|
|
118
|
+
link.remove();
|
|
119
|
+
continue;
|
|
120
|
+
}
|
|
121
|
+
link.setAttribute('href', `https://doi.org/${doi}`);
|
|
122
|
+
link.textContent = doi;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
for (const el of refList.querySelectorAll('[class], [id]')) {
|
|
126
|
+
el.removeAttribute('class');
|
|
127
|
+
el.removeAttribute('id');
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const omittedTables = [];
|
|
131
|
+
for (const table of view.querySelectorAll('div.table')) {
|
|
132
|
+
const href = table.querySelector('a.table-expand-popup')?.getAttribute('href');
|
|
133
|
+
for (const callout of table.querySelectorAll('.table-callout-links'))
|
|
134
|
+
callout.remove();
|
|
135
|
+
const url = href ? URL.parse(href, sourceUrl) : null;
|
|
136
|
+
if (!url || table.querySelector('table'))
|
|
137
|
+
continue;
|
|
138
|
+
const label = table.querySelector('.table-label')?.textContent?.trim().replace(/[.:]$/, '') ||
|
|
139
|
+
table.getAttribute('id') ||
|
|
140
|
+
'Unlabelled table';
|
|
141
|
+
omittedTables.push({ label, url: url.href });
|
|
142
|
+
}
|
|
143
|
+
return { html: document.toString(), omittedTables };
|
|
144
|
+
}
|
|
68
145
|
/**
|
|
69
146
|
* Reads a cached extraction, treating any storage failure as a miss. A key the
|
|
70
147
|
* backend rejects, an unreachable provider, or an in-memory store at capacity
|
|
@@ -131,14 +208,15 @@ export class BiorxivFullTextService {
|
|
|
131
208
|
* Resolves to `{ kind: 'article', markdown, ... }` on success, or
|
|
132
209
|
* `{ kind: 'unavailable', reason }` when the page is blocked (Cloudflare 403 /
|
|
133
210
|
* challenge), rate-limited (429), missing, or yields no extractable text
|
|
134
|
-
* (PDF-only preprints). Transient failures (5xx, timeout, network) bubble
|
|
135
|
-
*
|
|
211
|
+
* (PDF-only preprints). Transient failures (5xx, timeout, network) bubble
|
|
212
|
+
* unchanged after retry — `ServiceUnavailable`, or a `Timeout` once `deadline`
|
|
213
|
+
* runs out — for the caller to classify.
|
|
136
214
|
*
|
|
137
215
|
* A successful extraction is served from the tenant-scoped `ctx.state` cache on
|
|
138
216
|
* subsequent calls for the same server/DOI/version, so offset paging does not
|
|
139
217
|
* refetch the origin per chunk.
|
|
140
218
|
*/
|
|
141
|
-
async fetchFullText(server, doi, version, ctx) {
|
|
219
|
+
async fetchFullText(server, doi, version, deadline, ctx) {
|
|
142
220
|
const encodedDoi = doi.split('/').map(encodeURIComponent).join('/');
|
|
143
221
|
const sourceUrl = `${this.webBaseUrls[server]}/content/${encodedDoi}v${version}.full`;
|
|
144
222
|
// ctx.state is tenant-scoped and throws without a tenant (HTTP + JWT whose
|
|
@@ -154,9 +232,9 @@ export class BiorxivFullTextService {
|
|
|
154
232
|
}
|
|
155
233
|
let html;
|
|
156
234
|
try {
|
|
157
|
-
html = await
|
|
158
|
-
const response = await fetchWithTimeout(sourceUrl, 30_000, ctx, {
|
|
159
|
-
signal
|
|
235
|
+
html = await retryWithinBudget(async ({ signal, remainingMs }) => {
|
|
236
|
+
const response = await fetchWithTimeout(sourceUrl, Math.min(30_000, remainingMs), ctx, {
|
|
237
|
+
signal,
|
|
160
238
|
headers: { 'User-Agent': this.userAgent, Accept: 'text/html' },
|
|
161
239
|
expectedStatuses: EXPECTED_STATUSES,
|
|
162
240
|
});
|
|
@@ -166,13 +244,15 @@ export class BiorxivFullTextService {
|
|
|
166
244
|
context: ctx,
|
|
167
245
|
baseDelayMs: 500,
|
|
168
246
|
signal: ctx.signal,
|
|
247
|
+
deadline,
|
|
169
248
|
});
|
|
170
249
|
}
|
|
171
250
|
catch (err) {
|
|
172
251
|
// fetchWithTimeout throws on non-2xx with data.status set. Deterministic
|
|
173
252
|
// access failures (403 block, 404 missing) and origin rate limiting (429)
|
|
174
|
-
// become an `unavailable` result; transient
|
|
175
|
-
// bubble unchanged
|
|
253
|
+
// become an `unavailable` result; transient failures (5xx / timeout /
|
|
254
|
+
// network / deadline expiry) bubble unchanged. A 429 the budget ran out
|
|
255
|
+
// after arrives in place of the expiry, and is still the origin's answer.
|
|
176
256
|
const errData = err instanceof McpError
|
|
177
257
|
? err.data
|
|
178
258
|
: undefined;
|
|
@@ -219,7 +299,8 @@ export class BiorxivFullTextService {
|
|
|
219
299
|
sourceUrl,
|
|
220
300
|
};
|
|
221
301
|
}
|
|
222
|
-
const
|
|
302
|
+
const page = prepareArticlePage(html, sourceUrl);
|
|
303
|
+
const extracted = await htmlExtractor.extract(page.html, {
|
|
223
304
|
url: sourceUrl,
|
|
224
305
|
format: 'markdown',
|
|
225
306
|
contentSelector: FULLTEXT_CONTENT_SELECTOR,
|
|
@@ -237,6 +318,7 @@ export class BiorxivFullTextService {
|
|
|
237
318
|
markdown,
|
|
238
319
|
...(extracted.title && { title: extracted.title }),
|
|
239
320
|
sourceUrl,
|
|
321
|
+
...(page.omittedTables.length > 0 && { omittedTables: page.omittedTables }),
|
|
240
322
|
};
|
|
241
323
|
// Success path only — an unavailable result must recover on a later call.
|
|
242
324
|
if (cacheKey) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"biorxiv-fulltext-service.js","sourceRoot":"","sources":["../../../src/services/biorxiv-fulltext/biorxiv-fulltext-service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"biorxiv-fulltext-service.js","sourceRoot":"","sources":["../../../src/services/biorxiv-fulltext/biorxiv-fulltext-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAIH,OAAO,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAE5D,OAAO,EAEL,sBAAsB,EACtB,iBAAiB,EACjB,cAAc,GACf,MAAM,sBAAsB,CAAC;AAE9B;;;;;;;GAOG;AACH,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;AAEnD;;;;;;GAMG;AACH,MAAM,kCAAkC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE9E;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,CAAC,GAAG,kCAAkC,EAAE,GAAG,CAAC,CAAC;AAEvE;;;;;GAKG;AACH,MAAM,0BAA0B,GAAG,KAAK,CAAC;AAEzC;;;;;;;;GAQG;AACH,MAAM,WAAW,GAAG,CAAC,MAAqB,EAAE,GAAW,EAAE,OAAe,EAAU,EAAE,CAClF,eAAe,MAAM,IAAI,GAAG,IAAI,OAAO,EAAE,CAAC;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,SAAS,kBAAkB,CACzB,IAAY,EACZ,SAAiB;IAEjB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;IAC/D,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAChE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,EAAE,CAAC;QAC5D,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,gBAAgB,CAAC,gBAAgB,CAAC;YAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrF,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,gBAAgB,CAAC,YAAY,CAAC,EAAE,CAAC;YAC3D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,uBAAuB,CAAC;oBAC1D,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,YAAY,CAAC,GAAG,CAAC,YAAY,CAAC;oBACvF,CAAC,CAAC,IAAI,CAAC;gBACT,IAAI,CAAC,GAAG,EAAE,CAAC;oBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,mBAAmB,GAAG,EAAE,CAAC,CAAC;gBACpD,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC;YACzB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,OAAO,CAAC,gBAAgB,CAAC,eAAe,CAAC,EAAE,CAAC;YAC3D,EAAE,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YAC5B,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAmB,EAAE,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,sBAAsB,CAAC,EAAE,YAAY,CAAC,MAAM,CAAC,CAAC;QAC/E,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,gBAAgB,CAAC,sBAAsB,CAAC;YAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACvF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACrD,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC;YAAE,SAAS;QACnD,MAAM,KAAK,GACT,KAAK,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7E,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC;YACxB,kBAAkB,CAAC;QACrB,aAAa,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,iBAAiB,CAAC,GAAY,EAAE,GAAW;IACxD,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAgB,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC;IAChE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,6DAA6D,EAAE;YAC7E,GAAG;YACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;QACH,OAAO;IACT,CAAC;AACH,CAAC;AAED,+FAA+F;AAC/F,KAAK,UAAU,kBAAkB,CAC/B,GAAY,EACZ,GAAW,EACX,OAAsB;IAEtB,IAAI,CAAC;QACH,MAAM,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,0BAA0B,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,sEAAsE,EAAE;YACtF,GAAG;YACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACxD,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG;IACxB,kCAAkC;IAClC,kBAAkB;IAClB,yBAAyB;IACzB,sBAAsB;IACtB,aAAa;IACb,wCAAwC;IACxC,kBAAkB;CACnB,CAAC;AAEF,SAAS,eAAe,CAAC,IAAY;IACnC,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;AACnE,CAAC;AAiDD,MAAM,OAAO,sBAAsB;IAChB,WAAW,CAAgC;IAC3C,SAAS,CAAS;IAEnC,YAAY,OAAkB,EAAE,QAAwB;QACtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG;YACjB,OAAO,EAAE,SAAS,CAAC,iBAAiB;YACpC,OAAO,EAAE,SAAS,CAAC,iBAAiB;SACrC,CAAC;QACF,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM;YAC/B,CAAC,CAAC,sBAAsB,cAAc,YAAY,SAAS,CAAC,MAAM,GAAG;YACrE,CAAC,CAAC,sBAAsB,cAAc,EAAE,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,aAAa,CACjB,MAAqB,EACrB,GAAW,EACX,OAAe,EACf,QAAsB,EACtB,GAAY;QAEZ,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpE,MAAM,SAAS,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,UAAU,IAAI,OAAO,OAAO,CAAC;QAEtF,2EAA2E;QAC3E,8EAA8E;QAC9E,4CAA4C;QAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;QAC5F,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;YACtD,IAAI,MAAM,EAAE,CAAC;gBACX,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;gBACvE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,EAAE,CAAC;YACxC,CAAC;QACH,CAAC;QAED,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,iBAAiB,CAC5B,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE;gBAChC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,GAAG,EAAE;oBACrF,MAAM;oBACN,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE;oBAC9D,gBAAgB,EAAE,iBAAiB;iBACpC,CAAC,CAAC;gBACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC/B,CAAC,EACD;gBACE,SAAS,EAAE,sCAAsC;gBACjD,OAAO,EAAE,GAAG;gBACZ,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ;aACT,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,yEAAyE;YACzE,0EAA0E;YAC1E,sEAAsE;YACtE,wEAAwE;YACxE,0EAA0E;YAC1E,MAAM,OAAO,GACX,GAAG,YAAY,QAAQ;gBACrB,CAAC,CAAE,GAAG,CAAC,IAA+D;gBACtE,CAAC,CAAC,SAAS,CAAC;YAChB,MAAM,MAAM,GAAG,OAAO,OAAO,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAChF,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,sEAAsE;gBACtE,uEAAuE;gBACvE,kCAAkC;gBAClC,MAAM,UAAU,GAAG,sBAAsB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;gBAC/D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,2CAA2C,EAAE;oBACxD,SAAS;oBACT,MAAM;oBACN,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,CAAC;iBAChD,CAAC,CAAC;gBACH,OAAO;oBACL,IAAI,EAAE,aAAa;oBACnB,MAAM,EAAE,cAAc;oBACtB,MAAM,EACJ,UAAU,KAAK,SAAS;wBACtB,CAAC,CAAC,6DAA6D;wBAC/D,CAAC,CAAC,8EAA8E,UAAU,uCAAuC;oBACrI,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,CAAC;oBAC/C,SAAS;iBACV,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,KAAK,SAAS,IAAI,kCAAkC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3E,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;gBACvE,OAAO;oBACL,IAAI,EAAE,aAAa;oBACnB,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,mDAAmD,MAAM,4FAA4F;oBAC7J,SAAS;iBACV,CAAC;YACJ,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;QAED,4EAA4E;QAC5E,uEAAuE;QACvE,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,sDAAsD,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;YACvF,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,SAAS;gBACjB,MAAM,EACJ,6IAA6I;gBAC/I,SAAS;aACV,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE;YACvD,GAAG,EAAE,SAAS;YACd,MAAM,EAAE,UAAU;YAClB,eAAe,EAAE,yBAAyB;SAC3C,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;gBACL,IAAI,EAAE,aAAa;gBACnB,MAAM,EAAE,OAAO;gBACf,MAAM,EACJ,yIAAyI;gBAC3I,SAAS;aACV,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAkB;YAC7B,QAAQ;YACR,GAAG,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;YAClD,SAAS;YACT,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC;SAC5E,CAAC;QACF,0EAA0E;QAC1E,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,kBAAkB,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE,CAAC;IACzC,CAAC;CACF;AAED,gFAAgF;AAEhF,IAAI,QAA4C,CAAC;AAEjD,MAAM,UAAU,0BAA0B,CAAC,MAAiB,EAAE,OAAuB;IACnF,QAAQ,GAAG,IAAI,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,yBAAyB;IACvC,IAAI,CAAC,QAAQ;QACX,MAAM,IAAI,KAAK,CACb,uFAAuF,CACxF,CAAC;IACJ,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* abstracts for records that fall back to EuropePMC metadata: the ranked search
|
|
6
6
|
* uses the `lite` result type, which never carries `abstractText`, so abstracts
|
|
7
7
|
* come from one DOI-keyed `core` query instead. All requests
|
|
8
|
-
* include a polite User-Agent and are retried with exponential backoff
|
|
8
|
+
* include a polite User-Agent and are retried with exponential backoff inside
|
|
9
|
+
* the calling tool's deadline, which bounds every attempt and backoff.
|
|
9
10
|
* Detects HTML error pages, retries an HTTP 200 body that carries no result
|
|
10
11
|
* list, and classifies an origin rate limit (HTTP 429) as its own retryable
|
|
11
12
|
* `rate_limited` condition carrying the parsed `Retry-After` wait — never the
|
|
@@ -15,6 +16,7 @@
|
|
|
15
16
|
import type { Context } from '@cyanheads/mcp-ts-core';
|
|
16
17
|
import type { AppConfig } from '@cyanheads/mcp-ts-core/config';
|
|
17
18
|
import type { StorageService } from '@cyanheads/mcp-ts-core/storage';
|
|
19
|
+
import { type CallDeadline } from '../shared.js';
|
|
18
20
|
import type { EuropePmcSearchResult } from './types.js';
|
|
19
21
|
export interface SearchOptions {
|
|
20
22
|
/** Optional author filter — ANDed into the query as an `AUTH:"…"` clause. */
|
|
@@ -45,8 +47,10 @@ export declare class EuropePmcService {
|
|
|
45
47
|
* A body with no result list is retried; if every attempt returns one, throws
|
|
46
48
|
* `ValidationError` (`invalid_cursor_mark`) when a cursor page other than `*`
|
|
47
49
|
* was requested, and `ServiceUnavailable` (`empty_response_body`) otherwise.
|
|
50
|
+
* A deadline that runs out first is a `Timeout` expiry either way: fewer
|
|
51
|
+
* attempts than the full ladder do not establish that the cursor is at fault.
|
|
48
52
|
*/
|
|
49
|
-
search(options: SearchOptions, ctx: Context): Promise<EuropePmcSearchResult>;
|
|
53
|
+
search(options: SearchOptions, deadline: CallDeadline, ctx: Context): Promise<EuropePmcSearchResult>;
|
|
50
54
|
/**
|
|
51
55
|
* Fetch the abstracts of preprints by DOI in one `resultType=core` query — the
|
|
52
56
|
* only EuropePMC result type that carries `abstractText`. The query is keyed
|
|
@@ -56,7 +60,7 @@ export declare class EuropePmcService {
|
|
|
56
60
|
* absent from the map. Throws on failure — classified like `search`, with an
|
|
57
61
|
* HTTP 429 as a retryable `rate_limited` error carrying `Retry-After`.
|
|
58
62
|
*/
|
|
59
|
-
getAbstracts(dois: readonly string[], ctx: Context): Promise<Map<string, string>>;
|
|
63
|
+
getAbstracts(dois: readonly string[], deadline: CallDeadline, ctx: Context): Promise<Map<string, string>>;
|
|
60
64
|
}
|
|
61
65
|
export declare function initEuropePmcService(config: AppConfig, storage: StorageService): void;
|
|
62
66
|
export declare function getEuropePmcService(): EuropePmcService;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"europe-pmc-service.d.ts","sourceRoot":"","sources":["../../../src/services/europe-pmc/europe-pmc-service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"europe-pmc-service.d.ts","sourceRoot":"","sources":["../../../src/services/europe-pmc/europe-pmc-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,+BAA+B,CAAC;AAO/D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gCAAgC,CAAC;AAGrE,OAAO,EACL,KAAK,YAAY,EAOlB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAEV,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AA0FpB,MAAM,WAAW,aAAa;IAC5B,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,mFAAmF;IACnF,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC7B;AAED,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IAEnC,YAAY,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,cAAc,EAMvD;IAED;;;;;;;;;;;OAWG;IACG,MAAM,CACV,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,qBAAqB,CAAC,CAmGhC;IAED;;;;;;;;OAQG;IACG,YAAY,CAChB,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,QAAQ,EAAE,YAAY,EACtB,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAsD9B;CACF;AAMD,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAErF;AAED,wBAAgB,mBAAmB,IAAI,gBAAgB,CAItD"}
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* abstracts for records that fall back to EuropePMC metadata: the ranked search
|
|
6
6
|
* uses the `lite` result type, which never carries `abstractText`, so abstracts
|
|
7
7
|
* come from one DOI-keyed `core` query instead. All requests
|
|
8
|
-
* include a polite User-Agent and are retried with exponential backoff
|
|
8
|
+
* include a polite User-Agent and are retried with exponential backoff inside
|
|
9
|
+
* the calling tool's deadline, which bounds every attempt and backoff.
|
|
9
10
|
* Detects HTML error pages, retries an HTTP 200 body that carries no result
|
|
10
11
|
* list, and classifies an origin rate limit (HTTP 429) as its own retryable
|
|
11
12
|
* `rate_limited` condition carrying the parsed `Retry-After` wait — never the
|
|
@@ -13,9 +14,9 @@
|
|
|
13
14
|
* @module services/europe-pmc/europe-pmc-service
|
|
14
15
|
*/
|
|
15
16
|
import { McpError, rateLimited, serviceUnavailable, validationError, } from '@cyanheads/mcp-ts-core/errors';
|
|
16
|
-
import { fetchWithTimeout
|
|
17
|
+
import { fetchWithTimeout } from '@cyanheads/mcp-ts-core/utils';
|
|
17
18
|
import { getServerConfig } from '../../config/server-config.js';
|
|
18
|
-
import { describeWait, detectHtmlError, normalizeUpstreamText, parseRetryAfterSeconds, SERVER_VERSION, } from '../shared.js';
|
|
19
|
+
import { describeWait, detectHtmlError, normalizeUpstreamText, parseRetryAfterSeconds, retryWithinBudget, SERVER_VERSION, } from '../shared.js';
|
|
19
20
|
/**
|
|
20
21
|
* The one status this service classifies itself rather than treating as a bug.
|
|
21
22
|
* Passing it to `fetchWithTimeout` only lowers the log severity from `error` to
|
|
@@ -68,6 +69,7 @@ function rethrowClassified(err, cursorMark) {
|
|
|
68
69
|
}
|
|
69
70
|
throw serviceUnavailable(`EuropePMC answered ${tries} with HTTP 200 and no result list.`, { reason: EMPTY_BODY_REASON, retryAttempts }, { cause: err });
|
|
70
71
|
}
|
|
72
|
+
// A 429 the budget ran out after arrives here in place of the expiry.
|
|
71
73
|
if (data?.status !== 429)
|
|
72
74
|
throw err;
|
|
73
75
|
const retryAfter = parseRetryAfterSeconds(data.retryAfter);
|
|
@@ -102,8 +104,10 @@ export class EuropePmcService {
|
|
|
102
104
|
* A body with no result list is retried; if every attempt returns one, throws
|
|
103
105
|
* `ValidationError` (`invalid_cursor_mark`) when a cursor page other than `*`
|
|
104
106
|
* was requested, and `ServiceUnavailable` (`empty_response_body`) otherwise.
|
|
107
|
+
* A deadline that runs out first is a `Timeout` expiry either way: fewer
|
|
108
|
+
* attempts than the full ladder do not establish that the cursor is at fault.
|
|
105
109
|
*/
|
|
106
|
-
async search(options, ctx) {
|
|
110
|
+
async search(options, deadline, ctx) {
|
|
107
111
|
const limit = Math.min(options.limit ?? 25, 100);
|
|
108
112
|
// Base terms: the free-text keyword query and/or an AUTH: author clause.
|
|
109
113
|
// At least one is guaranteed non-empty by the tool's input refinement.
|
|
@@ -117,9 +121,10 @@ export class EuropePmcService {
|
|
|
117
121
|
terms.push(`AUTH:"${author.replace(/"/g, '')}"`);
|
|
118
122
|
}
|
|
119
123
|
let q = terms.join(' AND ');
|
|
124
|
+
// A blank bound is absent, never an open `[X TO ]` clause.
|
|
120
125
|
if (options.dateFrom || options.dateTo) {
|
|
121
|
-
const from = options.dateFrom
|
|
122
|
-
const to = options.dateTo
|
|
126
|
+
const from = options.dateFrom || '1900-01-01';
|
|
127
|
+
const to = options.dateTo || '9999-12-31';
|
|
123
128
|
q += ` AND FIRST_PDATE:[${from} TO ${to}]`;
|
|
124
129
|
}
|
|
125
130
|
// Scope to preprints; narrow to the specific server publisher when requested.
|
|
@@ -147,9 +152,9 @@ export class EuropePmcService {
|
|
|
147
152
|
});
|
|
148
153
|
const url = `${this.baseUrl}/search?${params.toString()}&filter=${filterParam}`;
|
|
149
154
|
try {
|
|
150
|
-
return await
|
|
151
|
-
const response = await fetchWithTimeout(url, 20_000, ctx, {
|
|
152
|
-
signal
|
|
155
|
+
return await retryWithinBudget(async ({ signal, remainingMs }) => {
|
|
156
|
+
const response = await fetchWithTimeout(url, Math.min(20_000, remainingMs), ctx, {
|
|
157
|
+
signal,
|
|
153
158
|
headers: { 'User-Agent': this.userAgent },
|
|
154
159
|
expectedStatuses: EXPECTED_STATUSES,
|
|
155
160
|
});
|
|
@@ -187,6 +192,7 @@ export class EuropePmcService {
|
|
|
187
192
|
context: ctx,
|
|
188
193
|
baseDelayMs: 300,
|
|
189
194
|
signal: ctx.signal,
|
|
195
|
+
deadline,
|
|
190
196
|
});
|
|
191
197
|
}
|
|
192
198
|
catch (err) {
|
|
@@ -202,7 +208,7 @@ export class EuropePmcService {
|
|
|
202
208
|
* absent from the map. Throws on failure — classified like `search`, with an
|
|
203
209
|
* HTTP 429 as a retryable `rate_limited` error carrying `Retry-After`.
|
|
204
210
|
*/
|
|
205
|
-
async getAbstracts(dois, ctx) {
|
|
211
|
+
async getAbstracts(dois, deadline, ctx) {
|
|
206
212
|
const unique = [...new Set(dois.map((doi) => doi.toLowerCase()))];
|
|
207
213
|
// Quotes are stripped so a stray one cannot end the DOI phrase early.
|
|
208
214
|
const clause = unique.map((doi) => `DOI:"${doi.replace(/"/g, '')}"`).join(' OR ');
|
|
@@ -215,9 +221,9 @@ export class EuropePmcService {
|
|
|
215
221
|
});
|
|
216
222
|
const url = `${this.baseUrl}/search?${params.toString()}`;
|
|
217
223
|
try {
|
|
218
|
-
return await
|
|
219
|
-
const response = await fetchWithTimeout(url, 20_000, ctx, {
|
|
220
|
-
signal
|
|
224
|
+
return await retryWithinBudget(async ({ signal, remainingMs }) => {
|
|
225
|
+
const response = await fetchWithTimeout(url, Math.min(20_000, remainingMs), ctx, {
|
|
226
|
+
signal,
|
|
221
227
|
headers: { 'User-Agent': this.userAgent },
|
|
222
228
|
expectedStatuses: EXPECTED_STATUSES,
|
|
223
229
|
});
|
|
@@ -244,6 +250,7 @@ export class EuropePmcService {
|
|
|
244
250
|
context: ctx,
|
|
245
251
|
baseDelayMs: 300,
|
|
246
252
|
signal: ctx.signal,
|
|
253
|
+
deadline,
|
|
247
254
|
});
|
|
248
255
|
}
|
|
249
256
|
catch (err) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"europe-pmc-service.js","sourceRoot":"","sources":["../../../src/services/europe-pmc/europe-pmc-service.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"europe-pmc-service.js","sourceRoot":"","sources":["../../../src/services/europe-pmc/europe-pmc-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EACL,QAAQ,EACR,WAAW,EACX,kBAAkB,EAClB,eAAe,GAChB,MAAM,+BAA+B,CAAC;AAEvC,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAEL,YAAY,EACZ,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,iBAAiB,EACjB,cAAc,GACf,MAAM,sBAAsB,CAAC;AAO9B;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,CAAC,GAAG,CAAC,CAAC;AAEhC;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;AAEhD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,SAAS,iBAAiB,CAAC,GAAY,EAAE,UAAkB;IACzD,MAAM,IAAI,GACR,GAAG,YAAY,QAAQ;QACrB,CAAC,CAAE,GAAG,CAAC,IAES;QAChB,CAAC,CAAC,SAAS,CAAC;IAEhB,IAAI,IAAI,EAAE,MAAM,KAAK,iBAAiB,EAAE,CAAC;QACvC,0EAA0E;QAC1E,wDAAwD;QACxD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAuB,CAAC;QACnD,MAAM,KAAK,GAAG,GAAG,aAAa,uBAAuB,CAAC;QACtD,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YACvB,MAAM,eAAe,CACnB,0DAA0D,KAAK,iCAAiC,EAChG,EAAE,MAAM,EAAE,qBAAqB,EAAE,WAAW,EAAE,UAAU,EAAE,aAAa,EAAE,EACzE,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;QACJ,CAAC;QACD,MAAM,kBAAkB,CACtB,sBAAsB,KAAK,oCAAoC,EAC/D,EAAE,MAAM,EAAE,iBAAiB,EAAE,aAAa,EAAE,EAC5C,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;IACJ,CAAC;IAED,sEAAsE;IACtE,IAAI,IAAI,EAAE,MAAM,KAAK,GAAG;QAAE,MAAM,GAAG,CAAC;IAEpC,MAAM,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;IACtC,MAAM,WAAW,CACf,UAAU,KAAK,SAAS;QACtB,CAAC,CAAC,kDAAkD;QACpD,CAAC,CAAC,mEAAmE,UAAU,uCAAuC,EACxH;QACE,MAAM,EAAE,cAAc;QACtB,SAAS,EAAE,IAAI;QACf,GAAG,CAAC,UAAU,KAAK,SAAS,IAAI,EAAE,UAAU,EAAE,CAAC;QAC/C,QAAQ,EAAE;YACR,IAAI,EAAE,QAAQ,IAAI,qJAAqJ;SACxK;KACF,EACD,EAAE,KAAK,EAAE,GAAG,EAAE,CACf,CAAC;AACJ,CAAC;AAmBD,MAAM,OAAO,gBAAgB;IACV,OAAO,CAAS;IAChB,SAAS,CAAS;IAEnC,YAAY,OAAkB,EAAE,QAAwB;QACtD,MAAM,SAAS,GAAG,eAAe,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,gBAAgB,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,MAAM;YAC/B,CAAC,CAAC,sBAAsB,cAAc,YAAY,SAAS,CAAC,MAAM,GAAG;YACrE,CAAC,CAAC,sBAAsB,cAAc,EAAE,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,MAAM,CACV,OAAsB,EACtB,QAAsB,EACtB,GAAY;QAEZ,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;QAEjD,yEAAyE;QACzE,uEAAuE;QACvE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;QACtC,IAAI,MAAM,EAAE,CAAC;YACX,6EAA6E;YAC7E,KAAK,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAE5B,2DAA2D;QAC3D,IAAI,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC;YAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC;YAC1C,CAAC,IAAI,qBAAqB,IAAI,OAAO,EAAE,GAAG,CAAC;QAC7C,CAAC;QAED,8EAA8E;QAC9E,kFAAkF;QAClF,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,CAAC,IAAI,wBAAwB,CAAC;QAChC,CAAC;aAAM,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACxC,CAAC,IAAI,wBAAwB,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,CAAC,IAAI,+CAA+C,CAAC;QACvD,CAAC;QACD,MAAM,WAAW,GAAG,YAAY,CAAC;QAEjC,yEAAyE;QACzE,+EAA+E;QAC/E,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,KAAK,EAAE,CAAC;YACR,UAAU,EAAE,MAAM;YAClB,OAAO,EAAE,OAAO;YAChB,UAAU;YACV,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC;YACvB,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,WAAW,MAAM,CAAC,QAAQ,EAAE,WAAW,WAAW,EAAE,CAAC;QAEhF,IAAI,CAAC;YACH,OAAO,MAAM,iBAAiB,CAC5B,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE;gBAChC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,GAAG,EAAE;oBAC/E,MAAM;oBACN,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;oBACzC,gBAAgB,EAAE,iBAAiB;iBACpC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1B,MAAM,kBAAkB,CACtB,oEAAoE,EACpE,EAAE,GAAG,EAAE,CACR,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+B,CAAC;gBAC5D,iEAAiE;gBACjE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;oBACrB,MAAM,kBAAkB,CAAC,kDAAkD,EAAE;wBAC3E,MAAM,EAAE,iBAAiB;qBAC1B,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,EAAE,CAAC;qBAC3C,MAAM,CAAC,CAAC,GAAG,EAAuC,EAAE,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC;qBACrE,GAAG,CAAC,CAAC,GAAG,EAAmB,EAAE;oBAC5B,MAAM,KAAK,GAAG,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;oBAC/C,OAAO;wBACL,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,GAAG,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,CAAC;wBACvB,GAAG,CAAC,GAAG,CAAC,YAAY,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC;wBACtD,GAAG,CAAC,GAAG,CAAC,oBAAoB,IAAI,EAAE,aAAa,EAAE,GAAG,CAAC,oBAAoB,EAAE,CAAC;qBAC7E,CAAC;gBACJ,CAAC,CAAC,CAAC;gBACL,OAAO;oBACL,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM;oBACzC,OAAO;oBACP,kEAAkE;oBAClE,8CAA8C;oBAC9C,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC;iBACpE,CAAC;YACJ,CAAC,EACD;gBACE,SAAS,EAAE,yBAAyB;gBACpC,OAAO,EAAE,GAAG;gBACZ,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ;aACT,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,iBAAiB,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CAChB,IAAuB,EACvB,QAAsB,EACtB,GAAY;QAEZ,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QAClE,sEAAsE;QACtE,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,KAAK,EAAE,IAAI,MAAM,eAAe;YAChC,UAAU,EAAE,MAAM;YAClB,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;YAC/B,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO,WAAW,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QAE1D,IAAI,CAAC;YACH,OAAO,MAAM,iBAAiB,CAC5B,KAAK,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,EAAE;gBAChC,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,EAAE,GAAG,EAAE;oBAC/E,MAAM;oBACN,OAAO,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,SAAS,EAAE;oBACzC,gBAAgB,EAAE,iBAAiB;iBACpC,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1B,MAAM,kBAAkB,CACtB,oEAAoE,EACpE,EAAE,GAAG,EAAE,CACR,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA+B,CAAC;gBAC5D,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;oBACrB,MAAM,kBAAkB,CAAC,kDAAkD,EAAE;wBAC3E,MAAM,EAAE,iBAAiB;qBAC1B,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;gBAC5C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;oBAC/C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC;oBACnC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;oBACzD,IAAI,GAAG,IAAI,QAAQ,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC3E,CAAC;gBACD,OAAO,SAAS,CAAC;YACnB,CAAC,EACD;gBACE,SAAS,EAAE,+BAA+B;gBAC1C,OAAO,EAAE,GAAG;gBACZ,WAAW,EAAE,GAAG;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ;aACT,CACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6EAA6E;YAC7E,iBAAiB,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;CACF;AAED,gFAAgF;AAEhF,IAAI,QAAsC,CAAC;AAE3C,MAAM,UAAU,oBAAoB,CAAC,MAAiB,EAAE,OAAuB;IAC7E,QAAQ,GAAG,IAAI,gBAAgB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,IAAI,CAAC,QAAQ;QACX,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;IAC/F,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|