ai4l-library-dev 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +2172 -2162
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -46,22 +46,13 @@ function bool(name, def, env = process.env) {
|
|
|
46
46
|
return ["true", "1", "yes", "on"].includes(v) ? true : ["false", "0", "no", "off"].includes(v) ? false : def;
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
// src/env-load.ts
|
|
50
|
+
loadDotenv();
|
|
51
|
+
|
|
49
52
|
// src/index.ts
|
|
50
53
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
51
54
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
52
55
|
|
|
53
|
-
// src/tools.ts
|
|
54
|
-
import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
|
|
55
|
-
import { z } from "zod";
|
|
56
|
-
|
|
57
|
-
// src/entry.ts
|
|
58
|
-
import { copyFileSync, existsSync as existsSync2, mkdirSync, readdirSync, readFileSync as readFileSync2, renameSync, rmSync, writeFileSync } from "fs";
|
|
59
|
-
import { join as join2 } from "path";
|
|
60
|
-
import { basename, dirname } from "path";
|
|
61
|
-
|
|
62
|
-
// src/pubmed.ts
|
|
63
|
-
import { XMLParser } from "fast-xml-parser";
|
|
64
|
-
|
|
65
56
|
// src/contact.ts
|
|
66
57
|
var DEFAULT = "hello@forever-healthy.org";
|
|
67
58
|
var contactEmail = (env = process.env) => str("AI4L_LIBRARY_CONTACT_EMAIL", env) || DEFAULT;
|
|
@@ -109,197 +100,173 @@ async function fetchText(url, headers = {}) {
|
|
|
109
100
|
throw new Error(last);
|
|
110
101
|
}
|
|
111
102
|
|
|
112
|
-
// src/
|
|
113
|
-
var
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
123
|
-
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
124
|
-
function efetchUrl(pmid) {
|
|
125
|
-
const u = new URL(EFETCH);
|
|
126
|
-
u.searchParams.set("db", "pubmed");
|
|
127
|
-
u.searchParams.set("id", pmid);
|
|
128
|
-
u.searchParams.set("retmode", "xml");
|
|
129
|
-
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
130
|
-
u.searchParams.set("tool", "ai4l-library");
|
|
131
|
-
u.searchParams.set("email", contactEmail());
|
|
132
|
-
return u.toString();
|
|
103
|
+
// src/checks.ts
|
|
104
|
+
var GREEK = { "\u03B1": "alpha", "\u03B2": "beta", "\xDF": "beta", "\u03B3": "gamma", "\u03B4": "delta", "\u03B5": "epsilon", "\u03B6": "zeta", "\u03B7": "eta", "\u03B8": "theta", "\u03B9": "iota", "\u03BA": "kappa", "\u03BB": "lambda", "\u03BC": "mu", "\xB5": "mu", "\u03BD": "nu", "\u03BE": "xi", "\u03C0": "pi", "\u03C1": "rho", "\u03C3": "sigma", "\u03C4": "tau", "\u03C5": "upsilon", "\u03C6": "phi", "\u03C7": "chi", "\u03C8": "psi", "\u03C9": "omega", "\u0391": "alpha", "\u0392": "beta", "\u0393": "gamma", "\u0394": "delta", "\u03A9": "omega", "\u03A3": "sigma", "\u039A": "kappa", "\u039B": "lambda", "\u03A0": "pi", "\u03A6": "phi", "\u03A8": "psi" };
|
|
105
|
+
var greek = (s) => s.replace(/[\u0370-\u03ff\u00b5\u00df]/g, (c) => GREEK[c] ?? c);
|
|
106
|
+
var norm = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
107
|
+
var compact = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
108
|
+
function hasTitle(md, title) {
|
|
109
|
+
const t = norm(title);
|
|
110
|
+
if (!t) return true;
|
|
111
|
+
if (norm(md).includes(t.slice(0, 60))) return true;
|
|
112
|
+
return compact(md).includes(compact(title).slice(0, 50));
|
|
133
113
|
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
114
|
+
var WALL = [
|
|
115
|
+
/checking your browser/i,
|
|
116
|
+
/recaptcha/i,
|
|
117
|
+
/verify (that )?you are (a )?human/i,
|
|
118
|
+
/human verification/i,
|
|
119
|
+
/access denied/i,
|
|
120
|
+
/enable javascript/i,
|
|
121
|
+
/javascript is (disabled|required)/i,
|
|
122
|
+
/cloudflare/i,
|
|
123
|
+
/captcha/i,
|
|
124
|
+
/sign in to (continue|access)/i,
|
|
125
|
+
/purchase (access|this article|pdf)/i,
|
|
126
|
+
/subscribe to (read|access)/i,
|
|
127
|
+
/cookie.{0,40}(consent|wall)/i,
|
|
128
|
+
/403 forbidden/i,
|
|
129
|
+
/page not found/i
|
|
130
|
+
];
|
|
131
|
+
function isWallPage(text3) {
|
|
132
|
+
const short = text3.length < 4e3;
|
|
133
|
+
if (short && WALL.some((re) => re.test(text3))) return true;
|
|
134
|
+
const title = /<title>([^<]*)<\/title>/i.exec(text3)?.[1] ?? "";
|
|
135
|
+
return /checking your browser|recaptcha|just a moment|access denied|attention required|human verification/i.test(title);
|
|
136
|
+
}
|
|
137
|
+
var headings = (md) => md.split("\n").filter((l) => /^#{1,6}\s/.test(l)).map((l) => l.replace(/^#+\s*/, ""));
|
|
138
|
+
function proseWords(md) {
|
|
139
|
+
let n = 0;
|
|
140
|
+
for (const p of md.split(/\n\s*\n/).map((x) => x.trim()).filter((x) => x && !/^#{1,6}\s/.test(x))) {
|
|
141
|
+
const words2 = p.split(/\s+/).length;
|
|
142
|
+
if (words2 < 40 || /^(-|\*|\d+\.)\s/.test(p)) continue;
|
|
143
|
+
const linkChars = (p.match(/\[[^\]]*\]\([^)]*\)/g) ?? []).join("").length;
|
|
144
|
+
if (linkChars / p.length > 0.3) continue;
|
|
145
|
+
n += words2;
|
|
146
|
+
}
|
|
147
|
+
return n;
|
|
148
|
+
}
|
|
149
|
+
var MIN_PROSE = 800;
|
|
150
|
+
function hasMethodsAndResults(md) {
|
|
151
|
+
const hs = headings(md);
|
|
152
|
+
const has = (re) => hs.some((h) => re.test(h));
|
|
153
|
+
if (has(/method|material|procedure|design|experimental|patients? and|participants/i) && has(/result|finding|outcome/i)) return true;
|
|
154
|
+
const words2 = md.split(/\s+/).length;
|
|
155
|
+
if (words2 < 1200 || hs.length < 2) return false;
|
|
156
|
+
const SECTION = /^(abstract|summary|introduction|background|discussion|conclusions?|references|acknowledg|limitations|key points|funding|participants|statistical|supplementary|data availability|author contributions|competing interests|conflicts? of interest|disclosure)/i;
|
|
157
|
+
return hs.filter((h) => SECTION.test(h.trim())).length >= 2;
|
|
158
|
+
}
|
|
159
|
+
var hasDoi = (md, doi) => !!doi && compact(md).includes(compact(doi));
|
|
160
|
+
function checkFulltext(md, title, titleSource, doi, trusted = false) {
|
|
161
|
+
if (isWallPage(md)) return { ok: false, reason: "wall page" };
|
|
162
|
+
const inBody = hasTitle(md, title);
|
|
163
|
+
if (!inBody && !(titleSource && hasTitle(titleSource, title)) && !hasDoi(md, doi)) return { ok: false, reason: "title missing" };
|
|
164
|
+
if (!trusted && proseWords(md) < MIN_PROSE) return { ok: false, reason: "no methods/results" };
|
|
165
|
+
if (!hasMethodsAndResults(md)) return { ok: false, reason: "no methods/results" };
|
|
166
|
+
return { ok: true };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// src/html.ts
|
|
170
|
+
import TurndownService from "turndown";
|
|
171
|
+
import { gfm } from "@joplin/turndown-plugin-gfm";
|
|
172
|
+
var td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
|
|
173
|
+
td.use(gfm);
|
|
174
|
+
td.remove(["script", "style", "nav", "noscript", "iframe", "button", "input", "select", "textarea", "header", "footer", "aside"]);
|
|
175
|
+
var FURNITURE = /gdpr|cookie|onetrust|consent|advert|\bad-slot\b/i;
|
|
176
|
+
td.remove(((n) => ["div", "section"].includes(String(n.nodeName).toLowerCase()) && FURNITURE.test(`${n.className ?? ""} ${n.id ?? ""}`)));
|
|
177
|
+
td.addRule("drop-svg", { filter: (n) => n.nodeName.toLowerCase() === "svg", replacement: () => "" });
|
|
178
|
+
td.addRule("drop-images", { filter: ["img"], replacement: () => "" });
|
|
179
|
+
td.addRule("div-headings", {
|
|
180
|
+
filter: (n) => String(n.nodeName).toLowerCase() === "div" && /(^|\s)h[1-6](\s|$)/.test(String(n.className ?? "")),
|
|
181
|
+
replacement: (content, n) => {
|
|
182
|
+
const level = /(^|\s)h([1-6])(\s|$)/.exec(String(n.className))[2];
|
|
183
|
+
return `
|
|
184
|
+
|
|
185
|
+
${"#".repeat(Number(level))} ${content.trim()}
|
|
186
|
+
|
|
187
|
+
`;
|
|
139
188
|
}
|
|
189
|
+
});
|
|
190
|
+
function htmlToMarkdown(html) {
|
|
191
|
+
return td.turndown(html).replace(/\n{3,}/g, "\n\n").trim() + "\n";
|
|
140
192
|
}
|
|
193
|
+
|
|
194
|
+
// src/jats.ts
|
|
195
|
+
import { XMLParser } from "fast-xml-parser";
|
|
141
196
|
var parser = new XMLParser({
|
|
197
|
+
preserveOrder: true,
|
|
142
198
|
ignoreAttributes: false,
|
|
143
199
|
attributeNamePrefix: "@_",
|
|
144
200
|
parseTagValue: false,
|
|
201
|
+
trimValues: false,
|
|
145
202
|
processEntities: true,
|
|
146
|
-
|
|
147
|
-
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
203
|
+
htmlEntities: true
|
|
148
204
|
});
|
|
149
|
-
function text(n) {
|
|
150
|
-
if (n == null) return "";
|
|
151
|
-
if (typeof n === "object") return text(n["#text"]);
|
|
152
|
-
return String(n);
|
|
153
|
-
}
|
|
154
|
-
var plain = (n) => decodeEntities(text(n)).trim();
|
|
155
|
-
function inlineText(raw) {
|
|
156
|
-
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
157
|
-
}
|
|
158
|
-
function decodeEntities(s) {
|
|
159
|
-
return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
160
|
-
}
|
|
161
|
-
function parsePubmedXml(xml) {
|
|
162
|
-
const doc = parser.parse(xml);
|
|
163
|
-
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
164
|
-
if (!article) {
|
|
165
|
-
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
166
|
-
return book ? parseBook(book) : null;
|
|
167
|
-
}
|
|
168
|
-
const cit = article.MedlineCitation ?? {};
|
|
169
|
-
const art = cit.Article ?? {};
|
|
170
|
-
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
171
|
-
const idOf = (t) => {
|
|
172
|
-
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
173
|
-
return hit ? text(hit).trim() || null : null;
|
|
174
|
-
};
|
|
175
|
-
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
176
|
-
const authors = authorList.map((a) => {
|
|
177
|
-
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
178
|
-
const last = plain(a.LastName);
|
|
179
|
-
const ini = plain(a.Initials);
|
|
180
|
-
return [last, ini].filter(Boolean).join(" ");
|
|
181
|
-
}).filter(Boolean);
|
|
182
|
-
const first = authorList[0];
|
|
183
|
-
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
184
|
-
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
185
|
-
let year = text(pub.Year).trim() || null;
|
|
186
|
-
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
187
|
-
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
188
|
-
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
189
|
-
text: inlineText(text(a))
|
|
190
|
-
})).filter((a) => a.text);
|
|
191
|
-
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
192
|
-
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
193
|
-
if (!type) return [];
|
|
194
|
-
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText(text(c.RefSource)) }];
|
|
195
|
-
});
|
|
196
|
-
const funding = [];
|
|
197
|
-
for (const g of art.GrantList?.Grant ?? []) {
|
|
198
|
-
const a = plain(g.Agency);
|
|
199
|
-
if (a && !funding.includes(a)) funding.push(a);
|
|
200
|
-
}
|
|
201
|
-
return {
|
|
202
|
-
coi: inlineText(text(cit.CoiStatement)) || null,
|
|
203
|
-
funding,
|
|
204
|
-
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
205
|
-
pmcid: idOf("pmc"),
|
|
206
|
-
doi: idOf("doi"),
|
|
207
|
-
title: inlineText(text(art.ArticleTitle)),
|
|
208
|
-
authors,
|
|
209
|
-
firstAuthor,
|
|
210
|
-
journal: plain(art.Journal?.Title) || null,
|
|
211
|
-
year,
|
|
212
|
-
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
213
|
-
abstract,
|
|
214
|
-
corrections
|
|
215
|
-
};
|
|
216
|
-
}
|
|
217
|
-
function parseBook(b) {
|
|
218
|
-
const d = b.BookDocument ?? {};
|
|
219
|
-
const book = d.Book ?? {};
|
|
220
|
-
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
221
|
-
const idOf = (t) => {
|
|
222
|
-
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
223
|
-
return hit ? text(hit).trim() || null : null;
|
|
224
|
-
};
|
|
225
|
-
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
226
|
-
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
227
|
-
const people = pick?.Author ?? [];
|
|
228
|
-
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
229
|
-
const first = people[0];
|
|
230
|
-
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText(text(a)) })).filter((a) => a.text);
|
|
231
|
-
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
232
|
-
return {
|
|
233
|
-
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
234
|
-
pmcid: null,
|
|
235
|
-
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
236
|
-
title: inlineText(text(d.ArticleTitle)) || inlineText(text(book.BookTitle)),
|
|
237
|
-
authors,
|
|
238
|
-
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
239
|
-
journal: plain(book.Publisher?.PublisherName) || null,
|
|
240
|
-
year: text(book.PubDate?.Year).trim() || null,
|
|
241
|
-
publicationTypes: [...new Set(types)],
|
|
242
|
-
abstract,
|
|
243
|
-
corrections: [],
|
|
244
|
-
coi: null,
|
|
245
|
-
funding: []
|
|
246
|
-
};
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// src/spl.ts
|
|
250
|
-
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
251
|
-
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
252
|
-
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
253
|
-
async function fetchSpl(setid) {
|
|
254
|
-
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
255
|
-
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
256
|
-
return xml;
|
|
257
|
-
}
|
|
258
|
-
var parser2 = new XMLParser2({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
|
|
259
205
|
var tagOf = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
260
206
|
var kids = (n) => n[tagOf(n)] ?? [];
|
|
261
207
|
var attr = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
262
|
-
var
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
const
|
|
269
|
-
if (
|
|
208
|
+
var SKIP_INLINE = /* @__PURE__ */ new Set(["inline-graphic", "graphic", "private-char"]);
|
|
209
|
+
function formula(n) {
|
|
210
|
+
const all = [n, ...descendants(n)];
|
|
211
|
+
const tex = all.find((x) => tagOf(x) === "tex-math");
|
|
212
|
+
if (tex) {
|
|
213
|
+
let t = plainText(kids(tex)).trim();
|
|
214
|
+
const m = /\\begin\{document\}([\s\S]*?)\\end\{document\}/.exec(t);
|
|
215
|
+
if (m) t = m[1].trim();
|
|
216
|
+
t = t.replace(/^\$+|\$+$/g, "").trim();
|
|
217
|
+
return t ? `$${t}$` : "";
|
|
270
218
|
}
|
|
271
|
-
|
|
219
|
+
const mml = all.find((x) => /^(mml:)?math$/.test(tagOf(x)));
|
|
220
|
+
return mml ? plainText(kids(mml)).replace(/\s+/g, "") : plainText(kids(n));
|
|
272
221
|
}
|
|
273
|
-
function
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
222
|
+
function descendants(n) {
|
|
223
|
+
if ("#text" in n) return [];
|
|
224
|
+
const out = [];
|
|
225
|
+
for (const k of kids(n)) {
|
|
226
|
+
out.push(k, ...descendants(k));
|
|
227
|
+
}
|
|
228
|
+
return out;
|
|
229
|
+
}
|
|
230
|
+
function inlineText(nodes) {
|
|
231
|
+
let out = "";
|
|
232
|
+
for (const n of nodes) {
|
|
233
|
+
if ("#text" in n) {
|
|
234
|
+
out += String(n["#text"]);
|
|
235
|
+
continue;
|
|
279
236
|
}
|
|
280
237
|
const t = tagOf(n);
|
|
281
|
-
|
|
238
|
+
if (SKIP_INLINE.has(t)) continue;
|
|
239
|
+
const inner = () => inlineText(kids(n));
|
|
282
240
|
switch (t) {
|
|
283
|
-
case "
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
241
|
+
case "italic":
|
|
242
|
+
case "i":
|
|
243
|
+
out += `*${inner()}*`;
|
|
244
|
+
break;
|
|
245
|
+
case "bold":
|
|
246
|
+
case "b":
|
|
247
|
+
out += `**${inner()}**`;
|
|
287
248
|
break;
|
|
288
|
-
}
|
|
289
249
|
case "sup":
|
|
290
250
|
out += `^${inner()}^`;
|
|
291
251
|
break;
|
|
292
252
|
case "sub":
|
|
293
253
|
out += `~${inner()}~`;
|
|
294
254
|
break;
|
|
295
|
-
case "
|
|
255
|
+
case "break":
|
|
296
256
|
out += " ";
|
|
297
257
|
break;
|
|
298
|
-
case "
|
|
299
|
-
|
|
258
|
+
case "ext-link":
|
|
259
|
+
case "uri": {
|
|
260
|
+
const href = attr(n, "xlink:href") || attr(n, "href");
|
|
261
|
+
const text3 = inner().trim() || href;
|
|
262
|
+
out += href ? `[${text3}](${href})` : text3;
|
|
300
263
|
break;
|
|
301
|
-
|
|
302
|
-
case "
|
|
264
|
+
}
|
|
265
|
+
case "inline-formula":
|
|
266
|
+
case "tex-math":
|
|
267
|
+
case "mml:math":
|
|
268
|
+
case "alternatives":
|
|
269
|
+
out += formula(n);
|
|
303
270
|
break;
|
|
304
271
|
default:
|
|
305
272
|
out += inner();
|
|
@@ -307,33 +274,36 @@ function inline(nodes) {
|
|
|
307
274
|
}
|
|
308
275
|
return out;
|
|
309
276
|
}
|
|
310
|
-
function
|
|
311
|
-
|
|
312
|
-
const
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
277
|
+
function plainText(nodes) {
|
|
278
|
+
let out = "";
|
|
279
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : plainText(kids(n));
|
|
280
|
+
return out;
|
|
281
|
+
}
|
|
282
|
+
var squash = (s) => s.replace(/\s+/g, " ").trim();
|
|
283
|
+
var cell = (s) => squash(s).replace(/\|/g, "\\|");
|
|
284
|
+
function renderTable(table3) {
|
|
285
|
+
const rows = [];
|
|
286
|
+
const walk = (nodes) => {
|
|
287
|
+
for (const n of nodes) {
|
|
288
|
+
const t = tagOf(n);
|
|
289
|
+
if (t === "tr") rows.push(kids(n).filter((c) => ["td", "th"].includes(tagOf(c))).map((c) => cell(inlineText(kids(c)))));
|
|
290
|
+
else if (!("#text" in n)) walk(kids(n));
|
|
317
291
|
}
|
|
318
292
|
};
|
|
319
|
-
|
|
320
|
-
for (const x of kids(n)) {
|
|
321
|
-
const t = tagOf(x);
|
|
322
|
-
if (t === "thead") collect(kids(x), head);
|
|
323
|
-
else if (t === "tfoot") collect(kids(x), foot);
|
|
324
|
-
else if (t === "tr") body.push(row2(x));
|
|
325
|
-
else if (!("#text" in x) && t !== "caption") collect(kids(x), body);
|
|
326
|
-
}
|
|
327
|
-
const rows = [...head, ...body];
|
|
293
|
+
walk(kids(table3));
|
|
328
294
|
if (!rows.length) return "";
|
|
329
295
|
const width = Math.max(...rows.map((r) => r.length));
|
|
330
|
-
const
|
|
331
|
-
const
|
|
332
|
-
|
|
333
|
-
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
334
|
-
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
296
|
+
const pad = (r) => [...r, ...Array(width - r.length).fill("")];
|
|
297
|
+
const line = (r) => `| ${pad(r).join(" | ")} |`;
|
|
298
|
+
return [line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line)].join("\n");
|
|
335
299
|
}
|
|
336
|
-
function
|
|
300
|
+
function labelCaption(n) {
|
|
301
|
+
const label = n ? squash(plainText(kids(n).filter((c) => tagOf(c) === "label"))) : "";
|
|
302
|
+
const cap = kids(n).find((c) => tagOf(c) === "caption");
|
|
303
|
+
const capText = cap ? squash(inlineText(kids(cap))) : "";
|
|
304
|
+
return [label ? `**${label}.**` : "", capText].filter(Boolean).join(" ");
|
|
305
|
+
}
|
|
306
|
+
function renderBlocks(nodes, depth) {
|
|
337
307
|
const out = [];
|
|
338
308
|
for (const n of nodes) {
|
|
339
309
|
if ("#text" in n) {
|
|
@@ -344,328 +314,200 @@ function blocks(nodes, depth) {
|
|
|
344
314
|
const t = tagOf(n);
|
|
345
315
|
const c = kids(n);
|
|
346
316
|
switch (t) {
|
|
347
|
-
case "
|
|
348
|
-
|
|
317
|
+
case "sec":
|
|
318
|
+
case "app":
|
|
319
|
+
case "ack":
|
|
320
|
+
case "boxed-text":
|
|
321
|
+
case "notes":
|
|
322
|
+
case "glossary": {
|
|
323
|
+
const title = c.find((x) => tagOf(x) === "title");
|
|
324
|
+
if (title) out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash(inlineText(kids(title)))}`);
|
|
325
|
+
else if (t === "ack") out.push(`${"#".repeat(Math.min(depth + 1, 6))} Acknowledgements`);
|
|
326
|
+
out.push(...renderBlocks(c.filter((x) => tagOf(x) !== "title"), depth + 1));
|
|
327
|
+
break;
|
|
328
|
+
}
|
|
329
|
+
case "title":
|
|
330
|
+
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash(inlineText(c))}`);
|
|
331
|
+
break;
|
|
332
|
+
case "p": {
|
|
333
|
+
const BLOCK = /* @__PURE__ */ new Set(["table-wrap", "fig", "list", "disp-formula", "disp-quote", "boxed-text", "supplementary-material"]);
|
|
334
|
+
const inline2 = c.filter((x) => "#text" in x || !BLOCK.has(tagOf(x)));
|
|
335
|
+
const blocks2 = c.filter((x) => !("#text" in x) && BLOCK.has(tagOf(x)));
|
|
336
|
+
const s = squash(inlineText(inline2));
|
|
349
337
|
if (s) out.push(s);
|
|
350
|
-
out.push(...
|
|
338
|
+
out.push(...renderBlocks(blocks2, depth));
|
|
351
339
|
break;
|
|
352
340
|
}
|
|
353
341
|
case "list": {
|
|
354
|
-
const ordered =
|
|
342
|
+
const ordered = /order|alpha|roman|number/i.test(attr(n, "list-type"));
|
|
355
343
|
let i = 0;
|
|
356
|
-
for (const
|
|
344
|
+
for (const li of c.filter((x) => tagOf(x) === "list-item")) {
|
|
357
345
|
i++;
|
|
358
|
-
const body =
|
|
359
|
-
|
|
346
|
+
const body = renderBlocks(kids(li).filter((x) => tagOf(x) !== "label"), depth).join(" ");
|
|
347
|
+
out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
360
348
|
}
|
|
361
349
|
break;
|
|
362
350
|
}
|
|
363
|
-
case "table": {
|
|
364
|
-
const
|
|
365
|
-
if (
|
|
351
|
+
case "table-wrap": {
|
|
352
|
+
const head = labelCaption(n);
|
|
353
|
+
if (head) out.push(head);
|
|
354
|
+
for (const tbl of c.filter((x) => tagOf(x) === "table")) {
|
|
355
|
+
const md = renderTable(tbl);
|
|
356
|
+
if (md) out.push(md);
|
|
357
|
+
}
|
|
358
|
+
const foot = c.find((x) => tagOf(x) === "table-wrap-foot");
|
|
359
|
+
if (foot) out.push(...renderBlocks(kids(foot), depth));
|
|
366
360
|
break;
|
|
367
361
|
}
|
|
368
|
-
case "
|
|
369
|
-
|
|
370
|
-
|
|
362
|
+
case "fig": {
|
|
363
|
+
const head = labelCaption(n);
|
|
364
|
+
out.push(head || "*(figure)*");
|
|
371
365
|
break;
|
|
372
|
-
|
|
373
|
-
|
|
366
|
+
}
|
|
367
|
+
case "disp-formula": {
|
|
368
|
+
const s = squash(formula(n));
|
|
369
|
+
if (s) out.push(s);
|
|
374
370
|
break;
|
|
375
|
-
|
|
376
|
-
case "
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
371
|
+
}
|
|
372
|
+
case "disp-quote":
|
|
373
|
+
out.push(...renderBlocks(c, depth).map((l) => `> ${l}`));
|
|
374
|
+
break;
|
|
375
|
+
case "def-list":
|
|
376
|
+
for (const d of c.filter((x) => tagOf(x) === "def-item")) {
|
|
377
|
+
const term = d["def-item"].find((x) => tagOf(x) === "term");
|
|
378
|
+
const def = d["def-item"].find((x) => tagOf(x) === "def");
|
|
379
|
+
out.push(`**${squash(inlineText(term ? kids(term) : []))}**: ${def ? renderBlocks(kids(def), depth).join(" ") : ""}`);
|
|
380
|
+
}
|
|
381
|
+
break;
|
|
382
|
+
case "ref-list": {
|
|
383
|
+
const title = c.find((x) => tagOf(x) === "title");
|
|
384
|
+
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${title ? squash(inlineText(kids(title))) : "References"}`);
|
|
385
|
+
for (const r of c.filter((x) => tagOf(x) === "ref")) {
|
|
386
|
+
const rk = kids(r);
|
|
387
|
+
const label = squash(plainText(rk.filter((x) => tagOf(x) === "label")));
|
|
388
|
+
out.push(`- ${label ? `${label} ` : ""}${renderCitation(rk.filter((x) => tagOf(x) !== "label"))}`);
|
|
389
|
+
}
|
|
390
|
+
break;
|
|
391
|
+
}
|
|
392
|
+
case "supplementary-material":
|
|
393
|
+
case "graphic":
|
|
394
|
+
case "media":
|
|
395
|
+
case "fn-group":
|
|
396
|
+
case "label":
|
|
381
397
|
break;
|
|
398
|
+
default:
|
|
399
|
+
out.push(...renderBlocks(c, depth));
|
|
382
400
|
}
|
|
383
401
|
}
|
|
384
402
|
return out;
|
|
385
403
|
}
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
404
|
+
function renderCitation(nodes) {
|
|
405
|
+
const alt = nodes.find((x) => tagOf(x) === "citation-alternatives");
|
|
406
|
+
if (alt) return renderCitation(kids(alt));
|
|
407
|
+
const mixed = nodes.find((x) => ["mixed-citation", "citation", "nlm-citation"].includes(tagOf(x)) && !kids(x).some((y) => tagOf(y) === "person-group"));
|
|
408
|
+
if (mixed) return squash(inlineText(kids(mixed)));
|
|
409
|
+
const el = nodes.find((x) => ["element-citation", "mixed-citation", "citation", "nlm-citation"].includes(tagOf(x)));
|
|
410
|
+
if (!el) return squash(inlineText(nodes));
|
|
411
|
+
const f = kids(el);
|
|
412
|
+
const get = (tag) => f.filter((x) => tagOf(x) === tag).map((x) => squash(inlineText(kids(x))));
|
|
413
|
+
const names = f.filter((x) => tagOf(x) === "person-group").flatMap((g) => kids(g)).map((x) => {
|
|
414
|
+
const t = tagOf(x);
|
|
415
|
+
if (t === "etal") return "et al";
|
|
416
|
+
if (t === "collab") return squash(plainText(kids(x)));
|
|
417
|
+
if (t === "name" || t === "string-name") {
|
|
418
|
+
const sur = squash(plainText(kids(x).filter((y) => tagOf(y) === "surname")));
|
|
419
|
+
const giv = squash(plainText(kids(x).filter((y) => tagOf(y) === "given-names")));
|
|
420
|
+
return [sur, giv].filter(Boolean).join(" ");
|
|
421
|
+
}
|
|
422
|
+
return "";
|
|
423
|
+
}).filter(Boolean);
|
|
424
|
+
const vol = get("volume")[0], issue = get("issue")[0], fp = get("fpage")[0], lp = get("lpage")[0];
|
|
425
|
+
const pages = fp ? lp ? `${fp}-${lp}` : fp : get("elocation-id")[0];
|
|
426
|
+
const where = [get("source")[0], get("year")[0] ? `${get("year")[0]};` : "", vol ? `${vol}${issue ? `(${issue})` : ""}` : "", pages ? `:${pages}` : ""].filter(Boolean).join("").replace(/;$/, "");
|
|
427
|
+
const ids = f.filter((x) => tagOf(x) === "pub-id").map((x) => `${attr(x, "pub-id-type")}:${squash(plainText(kids(x)))}`);
|
|
428
|
+
return [names.join(", "), [...get("article-title"), ...get("chapter-title")][0], where, ...ids].filter(Boolean).join(". ").replace(/\.\./g, ".");
|
|
429
|
+
}
|
|
430
|
+
var FN_LABELS = {
|
|
431
|
+
"conflict": "Conflicts of interest",
|
|
432
|
+
"COI-statement": "Conflicts of interest",
|
|
433
|
+
"coi-statement": "Conflicts of interest",
|
|
434
|
+
"financial-disclosure": "Financial disclosure",
|
|
435
|
+
"supported-by": "Funding",
|
|
436
|
+
"funding": "Funding",
|
|
437
|
+
"equal": "Author note",
|
|
438
|
+
"con": "Contributions",
|
|
439
|
+
"contributed-by": "Contributions",
|
|
440
|
+
"other": ""
|
|
392
441
|
};
|
|
393
|
-
function
|
|
394
|
-
const code = child(n, "code");
|
|
395
|
-
const c = code ? attr(code, "code") : "";
|
|
396
|
-
if (SKIP_CODES.has(c)) return [];
|
|
397
|
-
const title = child(n, "title");
|
|
398
|
-
let head = title ? squash(plain2(kids(title))) : "";
|
|
399
|
-
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
400
|
-
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
442
|
+
function declarations(meta, back) {
|
|
401
443
|
const out = [];
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
444
|
+
const fg = meta ? findDeep(kids(meta), "funding-group") : void 0;
|
|
445
|
+
if (fg) {
|
|
446
|
+
const statements = kids(fg).filter((x) => tagOf(x) === "funding-statement").map((x) => squash(inlineText(kids(x)))).filter(Boolean);
|
|
447
|
+
const awards = kids(fg).filter((x) => tagOf(x) === "award-group").map((ag) => {
|
|
448
|
+
const src = kids(ag).filter((x) => tagOf(x) === "funding-source").map((x) => squash(plainText(kids(x)))).filter(Boolean);
|
|
449
|
+
const ids = kids(ag).filter((x) => tagOf(x) === "award-id").map((x) => squash(plainText(kids(x)))).filter(Boolean);
|
|
450
|
+
return [src.join(", "), ids.length ? `(${ids.join(", ")})` : ""].filter(Boolean).join(" ");
|
|
451
|
+
}).filter(Boolean);
|
|
452
|
+
const text3 = statements.length ? statements.join(" ") : awards.join("; ");
|
|
453
|
+
if (text3) out.push(`Funding: ${text3}`);
|
|
454
|
+
}
|
|
455
|
+
const fns = [];
|
|
456
|
+
const notes = meta ? findDeep(kids(meta), "author-notes") : void 0;
|
|
457
|
+
if (notes) fns.push(...kids(notes).filter((x) => tagOf(x) === "fn"));
|
|
458
|
+
if (back) for (const g of kids(back).filter((x) => tagOf(x) === "fn-group")) fns.push(...kids(g).filter((x) => tagOf(x) === "fn"));
|
|
459
|
+
for (const fn of fns) {
|
|
460
|
+
const text3 = renderBlocks(kids(fn).filter((x) => tagOf(x) !== "label"), 1).join(" ").trim();
|
|
461
|
+
if (!text3) continue;
|
|
462
|
+
const label = FN_LABELS[attr(fn, "fn-type")] ?? "";
|
|
463
|
+
out.push(label && !new RegExp(`^[*_]*${label.split(" ")[0]}`, "i").test(text3) ? `${label}: ${text3}` : text3);
|
|
464
|
+
}
|
|
405
465
|
return out;
|
|
406
466
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
const
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
const version = attr(child(d, "versionNumber") ?? {}, "value") || null;
|
|
415
|
-
const effective = isoDate(attr(child(d, "effectiveTime") ?? {}, "value"));
|
|
416
|
-
const named = (nodes) => {
|
|
417
|
-
for (const n of nodes) {
|
|
418
|
-
if ("#text" in n) continue;
|
|
419
|
-
if (tagOf(n) === "manufacturedProduct") {
|
|
420
|
-
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
421
|
-
if (child(inner2, "name")) return inner2;
|
|
422
|
-
}
|
|
423
|
-
const h = named(kids(n));
|
|
424
|
-
if (h) return h;
|
|
425
|
-
}
|
|
426
|
-
return void 0;
|
|
427
|
-
};
|
|
428
|
-
const inner = named(k);
|
|
429
|
-
const brand = inner ? squash(inline(kids(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
430
|
-
const genericNode = inner ? findDeep(kids(inner), "genericMedicine") : void 0;
|
|
431
|
-
const generic = genericNode ? squash(inline(kids(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
432
|
-
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
433
|
-
const form = formNode ? attr(formNode, "displayName").toLowerCase() || null : null;
|
|
434
|
-
const org = findDeep(k, "representedOrganization");
|
|
435
|
-
const labeler = org ? squash(inline(kids(child(org, "name") ?? { name: [] }))) || null : null;
|
|
436
|
-
const body = findDeep(k, "structuredBody");
|
|
437
|
-
const sections2 = body ? blocks(kids(body), 2) : [];
|
|
438
|
-
const docTitle = child(d, "title");
|
|
439
|
-
const lines = [];
|
|
440
|
-
let cur = "";
|
|
441
|
-
for (const x of docTitle ? kids(docTitle) : []) {
|
|
442
|
-
if (!("#text" in x) && tagOf(x) === "br") {
|
|
443
|
-
lines.push(cur);
|
|
444
|
-
cur = "";
|
|
445
|
-
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
467
|
+
function find(nodes, path) {
|
|
468
|
+
let cur = nodes;
|
|
469
|
+
let hit;
|
|
470
|
+
for (const tag of path) {
|
|
471
|
+
hit = cur.find((x) => tagOf(x) === tag);
|
|
472
|
+
if (!hit) return void 0;
|
|
473
|
+
cur = kids(hit);
|
|
446
474
|
}
|
|
447
|
-
|
|
448
|
-
const highlights = lines.map(squash).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
449
|
-
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
450
|
-
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
451
|
-
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
452
|
-
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
475
|
+
return hit;
|
|
453
476
|
}
|
|
454
|
-
function
|
|
455
|
-
|
|
477
|
+
function findDeep(nodes, tag) {
|
|
478
|
+
for (const n of nodes) {
|
|
479
|
+
if ("#text" in n) continue;
|
|
480
|
+
if (tagOf(n) === tag) return n;
|
|
481
|
+
const hit = findDeep(kids(n), tag);
|
|
482
|
+
if (hit) return hit;
|
|
483
|
+
}
|
|
484
|
+
return void 0;
|
|
456
485
|
}
|
|
457
|
-
function
|
|
458
|
-
const
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
AVAILABLE: "Available",
|
|
483
|
-
NO_LONGER_AVAILABLE: "No longer available",
|
|
484
|
-
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
485
|
-
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
486
|
-
WITHHELD: "Withheld"
|
|
487
|
-
};
|
|
488
|
-
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
489
|
-
var words = (v) => {
|
|
490
|
-
const s = String(v ?? "");
|
|
491
|
-
if (!s) return "";
|
|
492
|
-
if (STATUS[s]) return STATUS[s];
|
|
493
|
-
if (PHASE[s]) return PHASE[s];
|
|
494
|
-
const w = s.toLowerCase().replace(/_/g, " ");
|
|
495
|
-
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
496
|
-
};
|
|
497
|
-
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
498
|
-
var list = (a) => Array.isArray(a) ? a : [];
|
|
499
|
-
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
500
|
-
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
501
|
-
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
502
|
-
function parseStudy(json2) {
|
|
503
|
-
let d;
|
|
504
|
-
try {
|
|
505
|
-
d = JSON.parse(json2);
|
|
506
|
-
} catch {
|
|
507
|
-
return null;
|
|
508
|
-
}
|
|
509
|
-
const ps = d?.protocolSection;
|
|
510
|
-
if (!ps) return null;
|
|
511
|
-
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
512
|
-
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
513
|
-
const nctid = String(idm.nctId ?? "").toUpperCase();
|
|
514
|
-
const out = [];
|
|
515
|
-
const sec = (title, lines) => {
|
|
516
|
-
const body = lines.filter((l) => !!l && l.trim() !== "");
|
|
517
|
-
if (body.length) out.push(`## ${title}`, ...body);
|
|
518
|
-
};
|
|
519
|
-
const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
|
|
520
|
-
sec("Identification", [
|
|
521
|
-
idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
|
|
522
|
-
idm.acronym ? `Acronym: ${idm.acronym}` : null,
|
|
523
|
-
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
524
|
-
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
525
|
-
]);
|
|
526
|
-
sec("Status", [
|
|
527
|
-
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
528
|
-
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
529
|
-
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
530
|
-
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
531
|
-
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
532
|
-
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
533
|
-
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
534
|
-
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
535
|
-
]);
|
|
536
|
-
const rp = sp.responsibleParty;
|
|
537
|
-
sec("Sponsor and collaborators", [
|
|
538
|
-
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
539
|
-
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
540
|
-
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
541
|
-
]);
|
|
542
|
-
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
543
|
-
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
544
|
-
const di = dm.designInfo ?? {};
|
|
545
|
-
const mi = di.maskingInfo ?? {};
|
|
546
|
-
sec("Design", [
|
|
547
|
-
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
548
|
-
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
549
|
-
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
550
|
-
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
551
|
-
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
552
|
-
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
553
|
-
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
554
|
-
]);
|
|
555
|
-
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
556
|
-
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
557
|
-
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
558
|
-
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
559
|
-
sec("Outcomes", [
|
|
560
|
-
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
561
|
-
...list(om.primaryOutcomes).map(outcome),
|
|
562
|
-
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
563
|
-
...list(om.secondaryOutcomes).map(outcome),
|
|
564
|
-
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
565
|
-
...list(om.otherOutcomes).map(outcome)
|
|
566
|
-
]);
|
|
567
|
-
sec("Eligibility", [
|
|
568
|
-
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
569
|
-
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
570
|
-
]);
|
|
571
|
-
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
572
|
-
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
573
|
-
const refs = list(rf.references);
|
|
574
|
-
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
575
|
-
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
576
|
-
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
577
|
-
return {
|
|
578
|
-
nctid,
|
|
579
|
-
title: para(idm.briefTitle) || nctid,
|
|
580
|
-
acronym: idm.acronym ?? null,
|
|
581
|
-
sponsor: sp.leadSponsor?.name ?? null,
|
|
582
|
-
firstPosted: st.studyFirstPostDateStruct?.date ?? null,
|
|
583
|
-
lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
|
|
584
|
-
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
585
|
-
hasResults: !!d.hasResults,
|
|
586
|
-
publications,
|
|
587
|
-
sections: out
|
|
588
|
-
};
|
|
589
|
-
}
|
|
590
|
-
function results(rs) {
|
|
591
|
-
const out = ["## Results"];
|
|
592
|
-
const pf = rs.participantFlowModule;
|
|
593
|
-
if (pf) {
|
|
594
|
-
out.push("### Participant flow");
|
|
595
|
-
const groups = list(pf.groups);
|
|
596
|
-
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
597
|
-
if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
|
|
598
|
-
for (const p of list(pf.periods)) {
|
|
599
|
-
const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
|
|
600
|
-
const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
|
|
601
|
-
for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
|
|
602
|
-
out.push(table2(head, rows));
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
const bc = rs.baselineCharacteristicsModule;
|
|
606
|
-
if (bc) {
|
|
607
|
-
out.push("### Baseline characteristics");
|
|
608
|
-
const groups = list(bc.groups);
|
|
609
|
-
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
610
|
-
for (const m of list(bc.measures)) {
|
|
611
|
-
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
612
|
-
const rows = [];
|
|
613
|
-
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
614
|
-
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
615
|
-
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
616
|
-
})]);
|
|
617
|
-
out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
const om = rs.outcomeMeasuresModule;
|
|
621
|
-
if (om) {
|
|
622
|
-
out.push("### Outcome measures");
|
|
623
|
-
for (const m of list(om.outcomeMeasures)) {
|
|
624
|
-
const groups = list(m.groups);
|
|
625
|
-
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
626
|
-
out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
|
|
627
|
-
if (m.description) out.push(para(m.description));
|
|
628
|
-
out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
|
|
629
|
-
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
630
|
-
const rows = [];
|
|
631
|
-
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
632
|
-
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
633
|
-
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
634
|
-
})]);
|
|
635
|
-
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
636
|
-
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
637
|
-
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
638
|
-
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
const ae = rs.adverseEventsModule;
|
|
642
|
-
if (ae) {
|
|
643
|
-
out.push("### Adverse events");
|
|
644
|
-
const groups = list(ae.eventGroups);
|
|
645
|
-
out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
|
|
646
|
-
const summary = [
|
|
647
|
-
["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
|
|
648
|
-
["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
|
|
649
|
-
["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
|
|
650
|
-
].filter((r) => r.slice(1).some(Boolean));
|
|
651
|
-
out.push(table2(["", ...groups.map((g) => g.title)], summary));
|
|
652
|
-
const events = (title, evs) => {
|
|
653
|
-
if (!evs.length) return;
|
|
654
|
-
out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
|
|
655
|
-
const s = list(e.stats).find((x) => x.groupId === g.id);
|
|
656
|
-
return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
|
|
657
|
-
})])));
|
|
658
|
-
};
|
|
659
|
-
events("Serious adverse events", list(ae.seriousEvents));
|
|
660
|
-
events("Other adverse events", list(ae.otherEvents));
|
|
661
|
-
}
|
|
662
|
-
return out;
|
|
663
|
-
}
|
|
664
|
-
function headLineStudy(s) {
|
|
665
|
-
return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
|
|
666
|
-
}
|
|
667
|
-
function renderStudyMd(s) {
|
|
668
|
-
const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
|
|
486
|
+
function parseJats(xml) {
|
|
487
|
+
const doc = parser.parse(xml);
|
|
488
|
+
const article = find(doc, ["pmc-articleset", "article"]) ?? find(doc, ["article"]) ?? findDeep(doc, "article");
|
|
489
|
+
const a = article ? kids(article) : [];
|
|
490
|
+
const front = find(a, ["front"]);
|
|
491
|
+
const meta = front ? find(kids(front), ["article-meta"]) ?? front : void 0;
|
|
492
|
+
const titleNode = meta ? find(kids(meta), ["title-group", "article-title"]) ?? findDeep(kids(meta), "article-title") : void 0;
|
|
493
|
+
const abstractNode = meta ? kids(meta).find((x) => tagOf(x) === "abstract" && !attr(x, "abstract-type")) ?? findDeep(kids(meta), "abstract") : void 0;
|
|
494
|
+
const body = find(a, ["body"]);
|
|
495
|
+
const back = find(a, ["back"]);
|
|
496
|
+
return {
|
|
497
|
+
title: titleNode ? squash(inlineText(kids(titleNode))) : "",
|
|
498
|
+
abstract: abstractNode ? renderBlocks(kids(abstractNode).filter((x) => tagOf(x) !== "title"), 2) : [],
|
|
499
|
+
body: body ? renderBlocks(kids(body), 1) : [],
|
|
500
|
+
declarations: declarations(meta, back),
|
|
501
|
+
back: back ? renderBlocks(kids(back), 1) : [],
|
|
502
|
+
hasBody: !!body && kids(body).length > 0
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
function renderFulltextMd(doc, headLine2) {
|
|
506
|
+
const parts = [`# ${doc.title}`, "", headLine2, ""];
|
|
507
|
+
if (doc.abstract.length) parts.push("## Abstract", "", ...doc.abstract.flatMap((b) => [b, ""]));
|
|
508
|
+
parts.push(...doc.body.flatMap((b) => [b, ""]));
|
|
509
|
+
if (doc.declarations.length) parts.push("## Declarations", "", ...doc.declarations.flatMap((b) => [b, ""]));
|
|
510
|
+
parts.push(...doc.back.flatMap((b) => [b, ""]));
|
|
669
511
|
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
670
512
|
}
|
|
671
513
|
|
|
@@ -953,1788 +795,1990 @@ var UNRESOLVED_GLYPH = /[\u0001-\u0008\u000b\u000c\u000e-\u001f]/g;
|
|
|
953
795
|
var countUnresolved = (text3) => (text3.match(UNRESOLVED_GLYPH) ?? []).length;
|
|
954
796
|
var looksLikePdf = (bytes) => bytes.subarray(0, 5).toString("latin1") === "%PDF-";
|
|
955
797
|
|
|
956
|
-
// src/
|
|
957
|
-
|
|
798
|
+
// src/browser.ts
|
|
799
|
+
import { existsSync as existsSync3, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
|
|
800
|
+
import { join as join3 } from "path";
|
|
801
|
+
import { chromium } from "playwright-core";
|
|
802
|
+
|
|
803
|
+
// src/layout.ts
|
|
804
|
+
import { createHash } from "crypto";
|
|
805
|
+
import { existsSync as existsSync2 } from "fs";
|
|
806
|
+
import { join as join2, resolve as resolve2 } from "path";
|
|
807
|
+
var AI4L_LIBRARY_DIR = (() => {
|
|
808
|
+
const env = process.env.AI4L_LIBRARY_DIR;
|
|
809
|
+
if (env && env.trim()) return resolve2(env.trim());
|
|
810
|
+
if (existsSync2("/library")) return "/library";
|
|
811
|
+
return resolve2("library");
|
|
812
|
+
})();
|
|
813
|
+
var doiSlug = (doi) => doi.toLowerCase().replace(/[^a-z0-9.-]+/g, "_");
|
|
814
|
+
function entryDirRel(type, num2) {
|
|
815
|
+
if (type === "doi") {
|
|
816
|
+
const h = createHash("sha1").update(num2).digest("hex");
|
|
817
|
+
return `${type}/${h.slice(0, 2)}/${h.slice(0, 4)}/${num2}`;
|
|
818
|
+
}
|
|
819
|
+
if (!/^\d+$/.test(num2)) return `${type}/${num2.slice(0, 2)}/${num2.slice(0, 4)}/${num2}`;
|
|
820
|
+
const l1 = num2.length > 6 ? num2.slice(0, -6) : "0";
|
|
821
|
+
const l2 = num2.length > 4 ? num2.slice(0, -4) : "0";
|
|
822
|
+
return `${type}/${l1}/${l2}/${num2}`;
|
|
823
|
+
}
|
|
824
|
+
function entryPaths(libraryDir, type, num2) {
|
|
825
|
+
if (type === "doi") num2 = doiSlug(num2);
|
|
826
|
+
const entry = `${type}-${num2}`;
|
|
827
|
+
const entryDir = join2(libraryDir, entryDirRel(type, num2));
|
|
958
828
|
return {
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
setid: null,
|
|
967
|
-
nctid: null,
|
|
968
|
-
publications: [],
|
|
969
|
-
pmcid: null,
|
|
970
|
-
doi: null,
|
|
971
|
-
title: null,
|
|
972
|
-
first_author: null,
|
|
973
|
-
year: null,
|
|
974
|
-
publication_types: [],
|
|
975
|
-
retracted: false,
|
|
976
|
-
corrections: [],
|
|
977
|
-
coi: null,
|
|
978
|
-
funding: [],
|
|
979
|
-
citable: true,
|
|
980
|
-
comment: null,
|
|
981
|
-
has_abstract: null,
|
|
982
|
-
level: null,
|
|
983
|
-
version: null,
|
|
984
|
-
retrieved_by: null,
|
|
985
|
-
retrieved_date: null,
|
|
986
|
-
pubmed_checked: null,
|
|
987
|
-
attempts: [],
|
|
988
|
-
files: [],
|
|
989
|
-
glyphs_unresolved: null
|
|
829
|
+
entry,
|
|
830
|
+
entryDir,
|
|
831
|
+
xml: join2(entryDir, `${entry}-abstract.xml`),
|
|
832
|
+
md: join2(entryDir, `${entry}-abstract.md`),
|
|
833
|
+
meta: join2(entryDir, `${entry}-meta.md`),
|
|
834
|
+
fulltextMd: join2(entryDir, `${entry}-fulltext.md`),
|
|
835
|
+
fulltextRaw: (format) => join2(entryDir, `${entry}-fulltext.${format}`)
|
|
990
836
|
};
|
|
991
837
|
}
|
|
992
|
-
var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
993
|
-
function headLine(r) {
|
|
994
|
-
return [
|
|
995
|
-
r.authors.join(", ") || null,
|
|
996
|
-
r.journal,
|
|
997
|
-
r.year,
|
|
998
|
-
r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
|
|
999
|
-
r.doi ? `DOI ${r.doi}` : null,
|
|
1000
|
-
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
1001
|
-
].filter(Boolean).join(" \xB7 ");
|
|
1002
|
-
}
|
|
1003
|
-
function renderAbstractMd(r) {
|
|
1004
|
-
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
1005
|
-
return `# ${r.title}
|
|
1006
|
-
|
|
1007
|
-
${headLine(r)}
|
|
1008
|
-
|
|
1009
|
-
## Abstract
|
|
1010
838
|
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
}
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
return lines[1] ?? "";
|
|
1018
|
-
} catch {
|
|
1019
|
-
return "";
|
|
839
|
+
// src/browser.ts
|
|
840
|
+
function browserConfig(env = process.env) {
|
|
841
|
+
const out = { browsers: [], headless: "auto" };
|
|
842
|
+
for (let i = 1; i <= 9; i++) {
|
|
843
|
+
const v = str(`AI4L_LIBRARY_BROWSER_${i}`, env);
|
|
844
|
+
if (v && !out.browsers.includes(v)) out.browsers.push(v);
|
|
1020
845
|
}
|
|
846
|
+
const h = str("AI4L_LIBRARY_BROWSER_HEADLESS", env).toLowerCase();
|
|
847
|
+
if (h === "true" || h === "false") out.headless = h === "true";
|
|
848
|
+
return out;
|
|
1021
849
|
}
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
pmcid: r.pmcid,
|
|
1029
|
-
doi: r.doi,
|
|
1030
|
-
title: r.title,
|
|
1031
|
-
first_author: r.firstAuthor,
|
|
1032
|
-
year: r.year,
|
|
1033
|
-
publication_types: r.publicationTypes,
|
|
1034
|
-
retracted: isRetracted(r),
|
|
1035
|
-
corrections: r.corrections,
|
|
1036
|
-
coi: r.coi,
|
|
1037
|
-
funding: r.funding,
|
|
1038
|
-
has_abstract: r.abstract.length > 0,
|
|
1039
|
-
level: "abstract",
|
|
1040
|
-
version: null,
|
|
1041
|
-
files,
|
|
1042
|
-
retrieved_by: "pubmed",
|
|
1043
|
-
retrieved_date: today(),
|
|
1044
|
-
converted_by: converter,
|
|
1045
|
-
pubmed_checked: today(),
|
|
1046
|
-
attempts: []
|
|
1047
|
-
};
|
|
1048
|
-
}
|
|
1049
|
-
var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
|
|
1050
|
-
var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
|
|
1051
|
-
function parseCorrection(v) {
|
|
1052
|
-
const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
|
|
1053
|
-
if (!m) return null;
|
|
1054
|
-
let source;
|
|
850
|
+
var INSTALL_HINT = "npx playwright-core install chromium";
|
|
851
|
+
var isRemote = (endpoint) => /^wss?:\/\//i.test(endpoint);
|
|
852
|
+
var isCdp = (endpoint) => /^cdp:\/\//i.test(endpoint);
|
|
853
|
+
var cdpHttp = (endpoint) => endpoint.replace(/^cdp:\/\//i, "http://");
|
|
854
|
+
var rungKind = (endpoint) => isCdp(endpoint) ? "chrome" : isRemote(endpoint) ? "server" : "local";
|
|
855
|
+
function localChromium() {
|
|
1055
856
|
try {
|
|
1056
|
-
|
|
857
|
+
const p = chromium.executablePath();
|
|
858
|
+
return existsSync3(p) ? p : null;
|
|
1057
859
|
} catch {
|
|
1058
860
|
return null;
|
|
1059
861
|
}
|
|
1060
|
-
return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
|
|
1061
|
-
}
|
|
1062
|
-
function renderMeta(m) {
|
|
1063
|
-
const lines = [
|
|
1064
|
-
["pmid", m.pmid],
|
|
1065
|
-
...m.setid ? [["setid", m.setid]] : [],
|
|
1066
|
-
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
1067
|
-
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
1068
|
-
["pmcid", m.pmcid ?? ""],
|
|
1069
|
-
["doi", m.doi ?? ""],
|
|
1070
|
-
["title", JSON.stringify(m.title)],
|
|
1071
|
-
["first_author", m.first_author ?? ""],
|
|
1072
|
-
["year", m.year ?? ""],
|
|
1073
|
-
["publication_types", m.publication_types.join("; ")],
|
|
1074
|
-
["retracted", m.retracted ? "yes" : "no"],
|
|
1075
|
-
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
1076
|
-
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
1077
|
-
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
1078
|
-
// a JSON array: a statement may hold semicolons
|
|
1079
|
-
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
1080
|
-
["level", m.level],
|
|
1081
|
-
["version", m.version ?? ""],
|
|
1082
|
-
["files", m.files.join(", ")],
|
|
1083
|
-
["retrieved_by", m.retrieved_by],
|
|
1084
|
-
["retrieved_date", m.retrieved_date],
|
|
1085
|
-
["converted_by", m.converted_by],
|
|
1086
|
-
["pubmed_checked", m.pubmed_checked],
|
|
1087
|
-
["attempts", m.attempts.join(", ")]
|
|
1088
|
-
];
|
|
1089
|
-
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
1090
862
|
}
|
|
1091
|
-
function
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
fm[k] = v;
|
|
1099
|
-
(all[k] ??= []).push(v);
|
|
863
|
+
function browserLabel(endpoint) {
|
|
864
|
+
if (rungKind(endpoint) === "local") return "local";
|
|
865
|
+
try {
|
|
866
|
+
const h = new URL(isCdp(endpoint) ? cdpHttp(endpoint) : endpoint).hostname;
|
|
867
|
+
return isCdp(endpoint) ? `chrome@${h}` : h;
|
|
868
|
+
} catch {
|
|
869
|
+
return endpoint;
|
|
1100
870
|
}
|
|
1101
|
-
return { fm, all };
|
|
1102
871
|
}
|
|
1103
|
-
var
|
|
1104
|
-
function
|
|
1105
|
-
if (
|
|
1106
|
-
|
|
1107
|
-
let cur = "", depth = 0;
|
|
1108
|
-
for (const ch of v) {
|
|
1109
|
-
if (ch === "(") depth++;
|
|
1110
|
-
else if (ch === ")") depth = Math.max(0, depth - 1);
|
|
1111
|
-
if (ch === "," && depth === 0) {
|
|
1112
|
-
out.push(cur);
|
|
1113
|
-
cur = "";
|
|
1114
|
-
} else cur += ch;
|
|
1115
|
-
}
|
|
1116
|
-
out.push(cur);
|
|
1117
|
-
const merged = [];
|
|
1118
|
-
for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
|
|
1119
|
-
if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
|
|
1120
|
-
else merged.push(piece);
|
|
1121
|
-
}
|
|
1122
|
-
return merged;
|
|
872
|
+
var hasDisplay = () => !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
|
|
873
|
+
function browserHeadless(endpoint, cfg = browserConfig()) {
|
|
874
|
+
if (isCdp(endpoint)) return false;
|
|
875
|
+
return cfg.headless === "auto" ? isRemote(endpoint) ? false : !hasDisplay() : cfg.headless;
|
|
1123
876
|
}
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
else out.push(piece);
|
|
1133
|
-
}
|
|
1134
|
-
return out;
|
|
877
|
+
function browserRungs(cfg = browserConfig()) {
|
|
878
|
+
return cfg.browsers.map((endpoint) => ({
|
|
879
|
+
endpoint,
|
|
880
|
+
label: browserLabel(endpoint),
|
|
881
|
+
kind: rungKind(endpoint),
|
|
882
|
+
available: rungKind(endpoint) !== "local" || localChromium() !== null,
|
|
883
|
+
headless: browserHeadless(endpoint, cfg)
|
|
884
|
+
}));
|
|
1135
885
|
}
|
|
1136
|
-
|
|
886
|
+
var CHALLENGE = /just a moment|nur einen moment|attention required|verify you are human|human verification|checking your browser|access denied/i;
|
|
887
|
+
var CONNECT_TIMEOUT_MS = 2e4;
|
|
888
|
+
var NAV_TIMEOUT_MS = 2e4;
|
|
889
|
+
var CHALLENGE_WAIT_MS = 8e3;
|
|
890
|
+
var DOWNLOAD_WAIT_MS = 1e4;
|
|
891
|
+
var IDLE_CLOSE_MS = 12e4;
|
|
892
|
+
var ctxs = /* @__PURE__ */ new Map();
|
|
893
|
+
var idleTimer = null;
|
|
894
|
+
function profileDir() {
|
|
895
|
+
const dir = join3(AI4L_LIBRARY_DIR, ".browser");
|
|
896
|
+
mkdirSync(join3(dir, "Default"), { recursive: true });
|
|
897
|
+
const prefs = join3(dir, "Default", "Preferences");
|
|
898
|
+
let p = {};
|
|
1137
899
|
try {
|
|
1138
|
-
|
|
900
|
+
p = JSON.parse(readFileSync2(prefs, "utf8"));
|
|
1139
901
|
} catch {
|
|
1140
|
-
return false;
|
|
1141
902
|
}
|
|
903
|
+
p.plugins = { ...p.plugins ?? {}, always_open_pdf_externally: true };
|
|
904
|
+
p.download = { ...p.download ?? {}, prompt_for_download: false };
|
|
905
|
+
writeFileSync(prefs, JSON.stringify(p));
|
|
906
|
+
return dir;
|
|
1142
907
|
}
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
908
|
+
var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
|
|
909
|
+
var LOCAL_WINDOW_ARGS = [...LAUNCH_ARGS, "--window-position=4000,4000"];
|
|
910
|
+
var TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "Europe/Berlin";
|
|
911
|
+
var CONTEXT_OPTS = { acceptDownloads: true, viewport: null, locale: "en-US", timezoneId: TIMEZONE };
|
|
912
|
+
function stateFile(endpoint) {
|
|
913
|
+
const dir = join3(AI4L_LIBRARY_DIR, ".browser");
|
|
914
|
+
mkdirSync(dir, { recursive: true });
|
|
915
|
+
return join3(dir, `${browserLabel(endpoint).replace(/[^A-Za-z0-9.-]+/g, "_")}.json`);
|
|
1149
916
|
}
|
|
1150
|
-
function
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
917
|
+
async function context(endpoint, cfg) {
|
|
918
|
+
let p = ctxs.get(endpoint);
|
|
919
|
+
if (!p) {
|
|
920
|
+
p = (async () => {
|
|
921
|
+
const headless = browserHeadless(endpoint, cfg);
|
|
922
|
+
const drop = () => {
|
|
923
|
+
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
924
|
+
};
|
|
925
|
+
if (isCdp(endpoint)) {
|
|
926
|
+
const browser = await chromium.connectOverCDP(cdpHttp(endpoint), { timeout: CONNECT_TIMEOUT_MS });
|
|
927
|
+
browser.on("disconnected", drop);
|
|
928
|
+
const ctx2 = browser.contexts()[0] ?? await browser.newContext(CONTEXT_OPTS);
|
|
929
|
+
return { ctx: ctx2, browser };
|
|
930
|
+
}
|
|
931
|
+
if (isRemote(endpoint)) {
|
|
932
|
+
const browser = await chromium.connect(endpoint, {
|
|
933
|
+
timeout: CONNECT_TIMEOUT_MS,
|
|
934
|
+
headers: { "x-playwright-launch-options": JSON.stringify({ headless, args: LAUNCH_ARGS, ignoreDefaultArgs: ["--enable-automation"] }) }
|
|
935
|
+
});
|
|
936
|
+
browser.on("disconnected", drop);
|
|
937
|
+
const state = stateFile(endpoint);
|
|
938
|
+
const ctx2 = await browser.newContext({ ...CONTEXT_OPTS, ...existsSync3(state) ? { storageState: state } : {} });
|
|
939
|
+
return { ctx: ctx2, browser };
|
|
940
|
+
}
|
|
941
|
+
if (!localChromium()) throw new Error(`no local Chromium (${INSTALL_HINT})`);
|
|
942
|
+
const ctx = await chromium.launchPersistentContext(profileDir(), {
|
|
943
|
+
channel: "chromium",
|
|
944
|
+
headless,
|
|
945
|
+
...CONTEXT_OPTS,
|
|
946
|
+
args: headless ? LAUNCH_ARGS : LOCAL_WINDOW_ARGS,
|
|
947
|
+
ignoreDefaultArgs: ["--enable-automation"]
|
|
948
|
+
});
|
|
949
|
+
ctx.on("close", drop);
|
|
950
|
+
return { ctx, browser: null };
|
|
951
|
+
})();
|
|
952
|
+
ctxs.set(endpoint, p);
|
|
953
|
+
p.catch(() => {
|
|
954
|
+
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
955
|
+
});
|
|
1156
956
|
}
|
|
957
|
+
return (await p).ctx;
|
|
1157
958
|
}
|
|
1158
|
-
function
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
959
|
+
function touchIdle() {
|
|
960
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
961
|
+
idleTimer = setTimeout(() => {
|
|
962
|
+
void closeBrowsers();
|
|
963
|
+
}, IDLE_CLOSE_MS);
|
|
964
|
+
idleTimer.unref();
|
|
965
|
+
}
|
|
966
|
+
async function closeBrowsers() {
|
|
967
|
+
const open = [...ctxs.entries()];
|
|
968
|
+
ctxs.clear();
|
|
969
|
+
for (const [endpoint, p] of open) {
|
|
1162
970
|
try {
|
|
1163
|
-
|
|
971
|
+
const { ctx, browser } = await p;
|
|
972
|
+
if (isCdp(endpoint)) {
|
|
973
|
+
await browser?.close().catch(() => void 0);
|
|
974
|
+
continue;
|
|
975
|
+
}
|
|
976
|
+
if (browser) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
977
|
+
await ctx.close().catch(() => void 0);
|
|
978
|
+
await browser?.close().catch(() => void 0);
|
|
1164
979
|
} catch {
|
|
1165
980
|
}
|
|
1166
981
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
982
|
+
}
|
|
983
|
+
async function browserFetch(url, endpoint, cfg = browserConfig()) {
|
|
984
|
+
const ctx = await context(endpoint, cfg);
|
|
985
|
+
touchIdle();
|
|
986
|
+
const page = await ctx.newPage();
|
|
987
|
+
try {
|
|
988
|
+
let pdfBest = null;
|
|
989
|
+
const pdfReads = [];
|
|
990
|
+
let status, contentType = "";
|
|
991
|
+
page.on("response", (r) => {
|
|
992
|
+
if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) {
|
|
993
|
+
status = r.status();
|
|
994
|
+
contentType = r.headers()["content-type"] ?? "";
|
|
1177
995
|
}
|
|
1178
|
-
|
|
1179
|
-
|
|
996
|
+
if (r.status() !== 200 || !/application\/pdf/i.test(r.headers()["content-type"] ?? "")) return;
|
|
997
|
+
pdfReads.push(r.body().then((b) => {
|
|
998
|
+
if (b.subarray(0, 5).toString("latin1") === "%PDF-" && (!pdfBest || b.length > pdfBest.length)) pdfBest = b;
|
|
999
|
+
}).catch(() => void 0));
|
|
1000
|
+
});
|
|
1001
|
+
let download2 = null;
|
|
1002
|
+
const dl = page.waitForEvent("download", { timeout: NAV_TIMEOUT_MS + CHALLENGE_WAIT_MS + DOWNLOAD_WAIT_MS }).then((d) => {
|
|
1003
|
+
download2 = d;
|
|
1004
|
+
return d;
|
|
1005
|
+
}).catch(() => null);
|
|
1006
|
+
let navError = null;
|
|
1007
|
+
const nav = page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).then((resp) => {
|
|
1008
|
+
if (status === void 0) {
|
|
1009
|
+
status = resp?.status();
|
|
1010
|
+
contentType = resp?.headers()["content-type"] ?? "";
|
|
1011
|
+
}
|
|
1012
|
+
}).catch((e) => {
|
|
1013
|
+
navError = (e instanceof Error ? e.message : String(e)).split("\n")[0].slice(0, 80);
|
|
1014
|
+
});
|
|
1015
|
+
await Promise.race([nav, dl]);
|
|
1016
|
+
const pause = (ms) => Promise.race([dl, new Promise((r) => setTimeout(r, ms))]);
|
|
1017
|
+
if (download2) {
|
|
1018
|
+
} else if (navError && !/download is starting|net::ERR_ABORTED/i.test(navError)) {
|
|
1019
|
+
await pause(2e3);
|
|
1020
|
+
if (!download2) throw new Error(`navigation: ${navError}`);
|
|
1021
|
+
} else {
|
|
1022
|
+
await pause(2e3);
|
|
1023
|
+
const t0 = Date.now();
|
|
1024
|
+
while (!download2 && Date.now() - t0 < CHALLENGE_WAIT_MS) {
|
|
1025
|
+
const title2 = await page.title().catch(() => "");
|
|
1026
|
+
if (!CHALLENGE.test(title2)) break;
|
|
1027
|
+
await pause(1e3);
|
|
1028
|
+
}
|
|
1029
|
+
if (!download2) await pause(DOWNLOAD_WAIT_MS / 5);
|
|
1030
|
+
}
|
|
1031
|
+
if (download2) {
|
|
1032
|
+
const d = download2;
|
|
1033
|
+
return { bytes: readFileSync2(await d.path()), contentType: "application/pdf", url: d.url(), via: "download" };
|
|
1034
|
+
}
|
|
1035
|
+
const title = await page.title().catch(() => "");
|
|
1036
|
+
if (CHALLENGE.test(title)) throw new Error(`challenge not cleared (${title.slice(0, 40)})`);
|
|
1037
|
+
if (status && status >= 400) throw new Error(`HTTP ${status}`);
|
|
1038
|
+
const docType = await page.evaluate(() => document.contentType).catch(() => "");
|
|
1039
|
+
const inViewer = /application\/pdf/i.test(contentType) || /application\/pdf/i.test(docType);
|
|
1040
|
+
if (inViewer) {
|
|
1041
|
+
const t1 = Date.now();
|
|
1042
|
+
while (Date.now() - t1 < DOWNLOAD_WAIT_MS) {
|
|
1043
|
+
await Promise.all(pdfReads);
|
|
1044
|
+
if (pdfBest && pdfBest.length > 4096) break;
|
|
1045
|
+
if (!pdfReads.length && Date.now() - t1 > 3e3) break;
|
|
1046
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1047
|
+
}
|
|
1048
|
+
if (pdfBest) return { bytes: pdfBest, contentType: "application/pdf", url: page.url(), via: "response" };
|
|
1049
|
+
}
|
|
1050
|
+
if (inViewer) {
|
|
1051
|
+
const r = await ctx.request.get(page.url(), { timeout: NAV_TIMEOUT_MS });
|
|
1052
|
+
if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
|
|
1053
|
+
return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
|
|
1054
|
+
}
|
|
1055
|
+
const html = await page.content();
|
|
1056
|
+
return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
|
|
1057
|
+
} finally {
|
|
1058
|
+
await page.close().catch(() => void 0);
|
|
1059
|
+
if (isRemote(endpoint)) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
1180
1060
|
}
|
|
1181
|
-
return {
|
|
1182
|
-
pmid: fm.pmid ?? "",
|
|
1183
|
-
setid: opt(fm.setid),
|
|
1184
|
-
nctid: opt(fm.nctid),
|
|
1185
|
-
publications: list2(fm.publications),
|
|
1186
|
-
pmcid: opt(fm.pmcid),
|
|
1187
|
-
doi: opt(fm.doi),
|
|
1188
|
-
title,
|
|
1189
|
-
first_author: opt(fm.first_author),
|
|
1190
|
-
year: opt(fm.year),
|
|
1191
|
-
publication_types: splitPublicationTypes(fm.publication_types),
|
|
1192
|
-
retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
|
|
1193
|
-
corrections,
|
|
1194
|
-
coi,
|
|
1195
|
-
funding,
|
|
1196
|
-
has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
|
|
1197
|
-
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
1198
|
-
version: opt(fm.version),
|
|
1199
|
-
files: list2(fm.files),
|
|
1200
|
-
retrieved_by: fm.retrieved_by ?? "",
|
|
1201
|
-
retrieved_date: fm.retrieved_date ?? "",
|
|
1202
|
-
converted_by: fm.converted_by ?? "",
|
|
1203
|
-
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
1204
|
-
attempts: splitAttempts(fm.attempts)
|
|
1205
|
-
};
|
|
1206
|
-
}
|
|
1207
|
-
function writeAtomic(file, data) {
|
|
1208
|
-
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
1209
|
-
writeFileSync(tmp, data);
|
|
1210
|
-
renameSync(tmp, file);
|
|
1211
1061
|
}
|
|
1212
|
-
|
|
1213
|
-
|
|
1062
|
+
|
|
1063
|
+
// src/unlock.ts
|
|
1064
|
+
var ENDPOINT = "https://api.brightdata.com/request";
|
|
1065
|
+
var TIMEOUT_MS2 = 6e4;
|
|
1066
|
+
var unlockerConfigured = () => !!process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
1067
|
+
async function unlock(url, fetchImpl = fetch) {
|
|
1068
|
+
const token = process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
1069
|
+
if (!token) throw new Error("unlocker not configured");
|
|
1070
|
+
const zone = process.env.BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
|
|
1071
|
+
const res = await fetchImpl(ENDPOINT, {
|
|
1072
|
+
method: "POST",
|
|
1073
|
+
headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
|
|
1074
|
+
body: JSON.stringify({ zone, url, format: "raw" }),
|
|
1075
|
+
signal: AbortSignal.timeout(TIMEOUT_MS2)
|
|
1076
|
+
});
|
|
1077
|
+
if (!res.ok) {
|
|
1078
|
+
const why = res.headers.get("x-brd-err-msg") ?? res.headers.get("x-brd-error");
|
|
1079
|
+
throw new Error(`unlocker HTTP ${res.status}${why ? `: ${why}` : ""}`);
|
|
1080
|
+
}
|
|
1081
|
+
return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
|
|
1214
1082
|
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
const m = readMeta(from);
|
|
1222
|
-
const copy = { ...m, files: m.files.map(rename) };
|
|
1223
|
-
writeFileSync(join2(tmp, basename(to.meta)), renderMeta(copy));
|
|
1224
|
-
mkdirSync(dirname(to.entryDir), { recursive: true });
|
|
1225
|
-
if (existsSync2(to.entryDir)) renameSync(to.entryDir, old);
|
|
1226
|
-
renameSync(tmp, to.entryDir);
|
|
1227
|
-
rmSync(old, { recursive: true, force: true });
|
|
1228
|
-
return copy;
|
|
1083
|
+
|
|
1084
|
+
// src/tiers/tier.ts
|
|
1085
|
+
function oaVersion(v) {
|
|
1086
|
+
if (v === "acceptedVersion") return "accepted_manuscript";
|
|
1087
|
+
if (v === "submittedVersion") return "preprint";
|
|
1088
|
+
return "published";
|
|
1229
1089
|
}
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1090
|
+
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
1091
|
+
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
1092
|
+
|
|
1093
|
+
// src/download.ts
|
|
1094
|
+
var hostOf = (url) => {
|
|
1095
|
+
try {
|
|
1096
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
1097
|
+
} catch {
|
|
1098
|
+
return url;
|
|
1236
1099
|
}
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
];
|
|
1242
|
-
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
1243
|
-
}
|
|
1244
|
-
function statusFromMeta(p, m, base) {
|
|
1245
|
-
const full = existsSync2(p.fulltextMd) ? readFileSync2(p.fulltextMd, "utf8") : null;
|
|
1246
|
-
const cit = citability(m);
|
|
1247
|
-
if (full?.includes(`
|
|
1248
|
-
${CORRECTED_HEADING}
|
|
1249
|
-
`) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
|
|
1250
|
-
return {
|
|
1251
|
-
...base,
|
|
1252
|
-
entry: p.entry,
|
|
1253
|
-
entry_dir: p.entryDir,
|
|
1254
|
-
pmid: m.pmid || null,
|
|
1255
|
-
setid: m.setid,
|
|
1256
|
-
nctid: m.nctid,
|
|
1257
|
-
publications: m.publications,
|
|
1258
|
-
pmcid: m.pmcid,
|
|
1259
|
-
doi: m.doi,
|
|
1260
|
-
title: m.title || null,
|
|
1261
|
-
first_author: m.first_author,
|
|
1262
|
-
year: m.year,
|
|
1263
|
-
publication_types: m.publication_types,
|
|
1264
|
-
retracted: m.retracted,
|
|
1265
|
-
corrections: m.corrections,
|
|
1266
|
-
coi: m.coi,
|
|
1267
|
-
funding: m.funding,
|
|
1268
|
-
...cit,
|
|
1269
|
-
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
1270
|
-
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
1271
|
-
// there is abstract text).
|
|
1272
|
-
has_abstract: m.has_abstract,
|
|
1273
|
-
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
1274
|
-
version: m.version,
|
|
1275
|
-
retrieved_by: m.retrieved_by || null,
|
|
1276
|
-
retrieved_date: m.retrieved_date || null,
|
|
1277
|
-
pubmed_checked: m.pubmed_checked || null,
|
|
1278
|
-
attempts: m.attempts,
|
|
1279
|
-
files: m.files,
|
|
1280
|
-
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
1281
|
-
};
|
|
1100
|
+
};
|
|
1101
|
+
var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
|
|
1102
|
+
function order(candidates) {
|
|
1103
|
+
const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
|
|
1104
|
+
return [...candidates].sort((a, b) => rank(a) - rank(b));
|
|
1282
1105
|
}
|
|
1283
|
-
function
|
|
1284
|
-
|
|
1106
|
+
async function download(tier, p, candidates, ctx) {
|
|
1107
|
+
const usable = candidates.filter((c) => !NCBI.test(c.url));
|
|
1108
|
+
if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
|
|
1109
|
+
const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
|
|
1110
|
+
if (!ordered.length) return miss(tier, "only copies already tried");
|
|
1111
|
+
const reasons = [];
|
|
1112
|
+
for (const c of ordered) {
|
|
1113
|
+
ctx.tried.add(c.url);
|
|
1114
|
+
let r = await tryOne(p, c, tier);
|
|
1115
|
+
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
1116
|
+
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
1117
|
+
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
1118
|
+
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
1119
|
+
for (const b of browserRungs()) {
|
|
1120
|
+
if (r.ok || c.headers || c.service || !b.available || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
1121
|
+
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
1122
|
+
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
1123
|
+
}
|
|
1124
|
+
if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
1125
|
+
r = await tryOne(p, target, tier, "unlocker");
|
|
1126
|
+
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
1127
|
+
if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
|
|
1128
|
+
}
|
|
1129
|
+
if (r.ok) return r;
|
|
1130
|
+
reasons.push(`${r.reason}: ${hostOf(c.url)}`);
|
|
1131
|
+
}
|
|
1132
|
+
return miss(tier, reasons.join("; "));
|
|
1285
1133
|
}
|
|
1286
|
-
|
|
1287
|
-
|
|
1134
|
+
var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
|
|
1135
|
+
async function tryOne(p, c, tier, via = "plain") {
|
|
1136
|
+
let got;
|
|
1137
|
+
try {
|
|
1138
|
+
got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
|
|
1139
|
+
} catch (e) {
|
|
1140
|
+
const body = e?.body;
|
|
1141
|
+
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
1142
|
+
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
1143
|
+
}
|
|
1144
|
+
const { bytes, contentType } = got;
|
|
1145
|
+
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
1146
|
+
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
1147
|
+
const r = await convertBytes(p, bytes, contentType);
|
|
1148
|
+
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
1149
|
+
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
1288
1150
|
}
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
...m,
|
|
1302
|
-
pmcid: r.pmcid ?? m.pmcid,
|
|
1303
|
-
doi: r.doi ?? m.doi,
|
|
1304
|
-
publication_types: r.publicationTypes,
|
|
1305
|
-
retracted: isRetracted(r),
|
|
1306
|
-
corrections: r.corrections,
|
|
1307
|
-
coi: r.coi ?? m.coi,
|
|
1308
|
-
funding: r.funding.length ? r.funding : m.funding,
|
|
1309
|
-
// the paper's own statements stay when PubMed has none
|
|
1310
|
-
pubmed_checked: today()
|
|
1311
|
-
};
|
|
1312
|
-
writeMeta(p, next);
|
|
1313
|
-
return next;
|
|
1151
|
+
var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
|
|
1152
|
+
function serviceMessage(bytes, contentType = "") {
|
|
1153
|
+
const text3 = bytes.toString("utf8").trim();
|
|
1154
|
+
if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
|
|
1155
|
+
try {
|
|
1156
|
+
const j = JSON.parse(text3);
|
|
1157
|
+
const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
|
|
1158
|
+
if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
|
|
1159
|
+
} catch {
|
|
1160
|
+
}
|
|
1161
|
+
const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
1162
|
+
return (visible || text3).slice(0, 160);
|
|
1314
1163
|
}
|
|
1315
|
-
function
|
|
1316
|
-
const
|
|
1317
|
-
|
|
1164
|
+
function describe(reason, format, md) {
|
|
1165
|
+
const words2 = md.split(/\s+/).filter(Boolean).length;
|
|
1166
|
+
const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
|
|
1167
|
+
const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
|
|
1168
|
+
if (reason === "no methods/results") return `no article body (${seen})`;
|
|
1169
|
+
if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
|
|
1170
|
+
return reason;
|
|
1318
1171
|
}
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
if (h) {
|
|
1330
|
-
section2 = h[1];
|
|
1331
|
-
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
1332
|
-
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
1333
|
-
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
1334
|
-
continue;
|
|
1172
|
+
async function convertBytes(p, bytes, contentType = "") {
|
|
1173
|
+
let format;
|
|
1174
|
+
let md;
|
|
1175
|
+
if (looksLikePdf(bytes)) {
|
|
1176
|
+
format = "pdf";
|
|
1177
|
+
const stats = {};
|
|
1178
|
+
try {
|
|
1179
|
+
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats);
|
|
1180
|
+
} catch (e) {
|
|
1181
|
+
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
1335
1182
|
}
|
|
1336
|
-
if (
|
|
1337
|
-
|
|
1338
|
-
const
|
|
1339
|
-
if (
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1183
|
+
if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
1184
|
+
} else {
|
|
1185
|
+
const text3 = bytes.toString("utf8");
|
|
1186
|
+
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
1187
|
+
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
1188
|
+
format = "xml";
|
|
1189
|
+
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
1190
|
+
if (!doc.title) doc.title = p.title;
|
|
1191
|
+
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
1192
|
+
md = renderFulltextMd(doc, p.headLine);
|
|
1193
|
+
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
1194
|
+
format = "html";
|
|
1195
|
+
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
1196
|
+
md = `# ${p.title}
|
|
1197
|
+
|
|
1198
|
+
${p.headLine}
|
|
1199
|
+
|
|
1200
|
+
${htmlToMarkdown(body)}`;
|
|
1201
|
+
} else {
|
|
1202
|
+
return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
|
|
1351
1203
|
}
|
|
1352
1204
|
}
|
|
1353
|
-
|
|
1205
|
+
const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
|
|
1206
|
+
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
1207
|
+
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
1208
|
+
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
1209
|
+
return { ok: true, format, md };
|
|
1210
|
+
}
|
|
1211
|
+
var ELSEVIER_TAGS = [
|
|
1212
|
+
[/ja:article\b/g, "article"],
|
|
1213
|
+
[/ja:body\b/g, "body"],
|
|
1214
|
+
[/ja:head\b/g, "front"],
|
|
1215
|
+
[/ce:sections\b/g, "sec-group"],
|
|
1216
|
+
[/ce:section-title\b/g, "title"],
|
|
1217
|
+
[/ce:section\b/g, "sec"],
|
|
1218
|
+
[/ce:simple-para\b/g, "p"],
|
|
1219
|
+
[/ce:para\b/g, "p"],
|
|
1220
|
+
[/ce:abstract-sec\b/g, "sec"],
|
|
1221
|
+
[/ce:abstract\b/g, "abstract"],
|
|
1222
|
+
[/ce:title\b/g, "article-title"],
|
|
1223
|
+
[/ce:italic\b/g, "italic"],
|
|
1224
|
+
[/ce:bold\b/g, "bold"],
|
|
1225
|
+
[/ce:sup\b/g, "sup"],
|
|
1226
|
+
[/ce:inf\b/g, "sub"],
|
|
1227
|
+
[/ce:cross-refs?\b/g, "xref"],
|
|
1228
|
+
[/ce:list-item\b/g, "list-item"],
|
|
1229
|
+
[/ce:list\b/g, "list"],
|
|
1230
|
+
[/ce:label\b/g, "label"],
|
|
1231
|
+
[/ce:caption\b/g, "caption"],
|
|
1232
|
+
[/ce:table\b/g, "table-wrap"],
|
|
1233
|
+
[/ce:figure\b/g, "fig"],
|
|
1234
|
+
[/ce:bib-reference\b/g, "ref"],
|
|
1235
|
+
[/ce:bibliography\b/g, "ref-list"],
|
|
1236
|
+
[/sb:reference\b/g, "mixed-citation"],
|
|
1237
|
+
[/ce:acknowledgment\b/g, "ack"],
|
|
1238
|
+
[/ce:inter-ref\b/g, "ext-link"]
|
|
1239
|
+
];
|
|
1240
|
+
function elsevierToJats(xml) {
|
|
1241
|
+
let out = xml;
|
|
1242
|
+
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
1243
|
+
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
1354
1244
|
}
|
|
1355
|
-
function withCorrectedBlock(md, sentences) {
|
|
1356
|
-
const base = md.split(`
|
|
1357
|
-
${CORRECTED_HEADING}
|
|
1358
|
-
`)[0].trimEnd();
|
|
1359
|
-
if (!sentences.length) return base + "\n";
|
|
1360
|
-
return `${base}
|
|
1361
|
-
|
|
1362
|
-
${CORRECTED_HEADING}
|
|
1363
|
-
|
|
1364
|
-
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
1365
1245
|
|
|
1366
|
-
|
|
1367
|
-
|
|
1246
|
+
// src/tiers/core.ts
|
|
1247
|
+
var WINDOW_MS = 6e4;
|
|
1248
|
+
var PER_WINDOW = 10;
|
|
1249
|
+
var MAX_WAIT_MS = 65e3;
|
|
1250
|
+
var TIMEOUT_MS3 = 1e4;
|
|
1251
|
+
var calls = [];
|
|
1252
|
+
async function pace() {
|
|
1253
|
+
const now = Date.now();
|
|
1254
|
+
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
1255
|
+
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
1256
|
+
calls.push(Date.now());
|
|
1368
1257
|
}
|
|
1369
|
-
function
|
|
1370
|
-
|
|
1371
|
-
let
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1258
|
+
async function coreFetch(url, key) {
|
|
1259
|
+
const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
|
|
1260
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
1261
|
+
await pace();
|
|
1262
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
|
|
1263
|
+
if (res.ok) return await res.text();
|
|
1264
|
+
const body = await res.text().catch(() => "");
|
|
1265
|
+
if (res.status === 429 && attempt === 0) {
|
|
1266
|
+
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
1267
|
+
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
1268
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
1269
|
+
continue;
|
|
1270
|
+
}
|
|
1271
|
+
throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
|
|
1376
1272
|
}
|
|
1377
|
-
|
|
1378
|
-
const md = readFileSync2(p.fulltextMd, "utf8");
|
|
1379
|
-
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
1380
|
-
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
1273
|
+
throw new Error("HTTP 429");
|
|
1381
1274
|
}
|
|
1382
|
-
function
|
|
1383
|
-
const
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1275
|
+
function coreTier(apiKey) {
|
|
1276
|
+
const key = apiKey?.trim();
|
|
1277
|
+
return {
|
|
1278
|
+
name: "core",
|
|
1279
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
1280
|
+
async run(p, ctx) {
|
|
1281
|
+
if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
|
|
1282
|
+
let data;
|
|
1283
|
+
try {
|
|
1284
|
+
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
1285
|
+
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
1286
|
+
} catch (e) {
|
|
1287
|
+
return miss(this.name, errMsg(e));
|
|
1288
|
+
}
|
|
1289
|
+
const candidates = coreCandidates(data);
|
|
1290
|
+
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
1291
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1292
|
+
return download(this.name, p, candidates, ctx);
|
|
1293
|
+
}
|
|
1399
1294
|
};
|
|
1400
|
-
writeMeta(p, next);
|
|
1401
|
-
return next;
|
|
1402
1295
|
}
|
|
1403
|
-
function
|
|
1404
|
-
|
|
1405
|
-
const
|
|
1406
|
-
const
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1296
|
+
function coreCandidates(d) {
|
|
1297
|
+
const out = [];
|
|
1298
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1299
|
+
const add = (url, pdf, host) => {
|
|
1300
|
+
if (!url || seen.has(url)) return;
|
|
1301
|
+
seen.add(url);
|
|
1302
|
+
out.push({ url, version: "published", source: "core", host, pdf });
|
|
1303
|
+
};
|
|
1304
|
+
for (const w of d.results ?? []) {
|
|
1305
|
+
add(w.downloadUrl, true, "core");
|
|
1306
|
+
for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
|
|
1307
|
+
for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
|
|
1308
|
+
}
|
|
1309
|
+
return out;
|
|
1410
1310
|
}
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
attempts: []
|
|
1440
|
-
};
|
|
1441
|
-
writeMeta(p, meta);
|
|
1442
|
-
return meta;
|
|
1443
|
-
}
|
|
1444
|
-
function writeStudyEntry(p, s, json2, converter) {
|
|
1445
|
-
mkdirSync(p.entryDir, { recursive: true });
|
|
1446
|
-
const raw = p.fulltextRaw("json");
|
|
1447
|
-
writeAtomic(raw, json2);
|
|
1448
|
-
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
1449
|
-
const meta = {
|
|
1450
|
-
pmid: "",
|
|
1451
|
-
setid: null,
|
|
1452
|
-
nctid: s.nctid,
|
|
1453
|
-
publications: s.publications,
|
|
1454
|
-
pmcid: null,
|
|
1455
|
-
doi: null,
|
|
1456
|
-
title: s.title,
|
|
1457
|
-
first_author: s.sponsor,
|
|
1458
|
-
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
1459
|
-
publication_types: ["Trial Registration"],
|
|
1460
|
-
retracted: false,
|
|
1461
|
-
corrections: [],
|
|
1462
|
-
coi: null,
|
|
1463
|
-
funding: [],
|
|
1464
|
-
has_abstract: false,
|
|
1465
|
-
level: "full_text",
|
|
1466
|
-
version: s.lastUpdate,
|
|
1467
|
-
files: [basename(raw), basename(p.fulltextMd)],
|
|
1468
|
-
retrieved_by: "ctgov",
|
|
1469
|
-
retrieved_date: today(),
|
|
1470
|
-
converted_by: converter,
|
|
1471
|
-
pubmed_checked: today(),
|
|
1472
|
-
attempts: []
|
|
1311
|
+
|
|
1312
|
+
// src/tiers/europepmc.ts
|
|
1313
|
+
var europepmc = {
|
|
1314
|
+
name: "europepmc",
|
|
1315
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
1316
|
+
async run(p, ctx) {
|
|
1317
|
+
let data;
|
|
1318
|
+
try {
|
|
1319
|
+
const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
|
|
1320
|
+
data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
|
|
1321
|
+
} catch (e) {
|
|
1322
|
+
return miss(this.name, errMsg(e));
|
|
1323
|
+
}
|
|
1324
|
+
const rec = data.resultList?.result?.[0];
|
|
1325
|
+
if (!rec) return miss(this.name, "not in europe pmc");
|
|
1326
|
+
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
1327
|
+
if (!candidates.length) return miss(this.name, "no free copy");
|
|
1328
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1329
|
+
return download(this.name, p, candidates, ctx);
|
|
1330
|
+
}
|
|
1331
|
+
};
|
|
1332
|
+
function europepmcCandidates(r, pmcid) {
|
|
1333
|
+
const out = [];
|
|
1334
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1335
|
+
const add = (url, pdf, host, version) => {
|
|
1336
|
+
if (!url || seen.has(url)) return;
|
|
1337
|
+
seen.add(url);
|
|
1338
|
+
out.push({ url, version, source: "europepmc", host, pdf });
|
|
1473
1339
|
};
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1340
|
+
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
1341
|
+
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
1342
|
+
if (id && r.inEPMC === "Y") {
|
|
1343
|
+
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
1344
|
+
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
1345
|
+
}
|
|
1346
|
+
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
1347
|
+
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
1348
|
+
if (u.documentStyle === "abs") continue;
|
|
1349
|
+
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
1350
|
+
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
1351
|
+
}
|
|
1352
|
+
return out;
|
|
1485
1353
|
}
|
|
1486
1354
|
|
|
1487
|
-
// src/
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
var
|
|
1491
|
-
var
|
|
1492
|
-
function
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
var WALL = [
|
|
1499
|
-
/checking your browser/i,
|
|
1500
|
-
/recaptcha/i,
|
|
1501
|
-
/verify (that )?you are (a )?human/i,
|
|
1502
|
-
/human verification/i,
|
|
1503
|
-
/access denied/i,
|
|
1504
|
-
/enable javascript/i,
|
|
1505
|
-
/javascript is (disabled|required)/i,
|
|
1506
|
-
/cloudflare/i,
|
|
1507
|
-
/captcha/i,
|
|
1508
|
-
/sign in to (continue|access)/i,
|
|
1509
|
-
/purchase (access|this article|pdf)/i,
|
|
1510
|
-
/subscribe to (read|access)/i,
|
|
1511
|
-
/cookie.{0,40}(consent|wall)/i,
|
|
1512
|
-
/403 forbidden/i,
|
|
1513
|
-
/page not found/i
|
|
1514
|
-
];
|
|
1515
|
-
function isWallPage(text3) {
|
|
1516
|
-
const short = text3.length < 4e3;
|
|
1517
|
-
if (short && WALL.some((re) => re.test(text3))) return true;
|
|
1518
|
-
const title = /<title>([^<]*)<\/title>/i.exec(text3)?.[1] ?? "";
|
|
1519
|
-
return /checking your browser|recaptcha|just a moment|access denied|attention required|human verification/i.test(title);
|
|
1520
|
-
}
|
|
1521
|
-
var headings = (md) => md.split("\n").filter((l) => /^#{1,6}\s/.test(l)).map((l) => l.replace(/^#+\s*/, ""));
|
|
1522
|
-
function proseWords(md) {
|
|
1523
|
-
let n = 0;
|
|
1524
|
-
for (const p of md.split(/\n\s*\n/).map((x) => x.trim()).filter((x) => x && !/^#{1,6}\s/.test(x))) {
|
|
1525
|
-
const words2 = p.split(/\s+/).length;
|
|
1526
|
-
if (words2 < 40 || /^(-|\*|\d+\.)\s/.test(p)) continue;
|
|
1527
|
-
const linkChars = (p.match(/\[[^\]]*\]\([^)]*\)/g) ?? []).join("").length;
|
|
1528
|
-
if (linkChars / p.length > 0.3) continue;
|
|
1529
|
-
n += words2;
|
|
1355
|
+
// src/tiers/inbox.ts
|
|
1356
|
+
import { existsSync as existsSync4, readdirSync, readFileSync as readFileSync3, unlinkSync } from "fs";
|
|
1357
|
+
import { join as join4 } from "path";
|
|
1358
|
+
var INBOX_DIR = join4(AI4L_LIBRARY_DIR, "inbox");
|
|
1359
|
+
var EXT = { ".pdf": "application/pdf", ".xml": "text/xml", ".html": "text/html", ".htm": "text/html" };
|
|
1360
|
+
function findInboxFile(dir, pmid) {
|
|
1361
|
+
if (!existsSync4(dir)) return null;
|
|
1362
|
+
const re = new RegExp(`(^|[^0-9])${pmid}([^0-9]|$)`);
|
|
1363
|
+
for (const name of readdirSync(dir).sort()) {
|
|
1364
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
1365
|
+
if (ext in EXT && re.test(name.slice(0, -ext.length))) return join4(dir, name);
|
|
1530
1366
|
}
|
|
1531
|
-
return
|
|
1532
|
-
}
|
|
1533
|
-
var MIN_PROSE = 800;
|
|
1534
|
-
function hasMethodsAndResults(md) {
|
|
1535
|
-
const hs = headings(md);
|
|
1536
|
-
const has = (re) => hs.some((h) => re.test(h));
|
|
1537
|
-
if (has(/method|material|procedure|design|experimental|patients? and|participants/i) && has(/result|finding|outcome/i)) return true;
|
|
1538
|
-
const words2 = md.split(/\s+/).length;
|
|
1539
|
-
if (words2 < 1200 || hs.length < 2) return false;
|
|
1540
|
-
const SECTION = /^(abstract|summary|introduction|background|discussion|conclusions?|references|acknowledg|limitations|key points|funding|participants|statistical|supplementary|data availability|author contributions|competing interests|conflicts? of interest|disclosure)/i;
|
|
1541
|
-
return hs.filter((h) => SECTION.test(h.trim())).length >= 2;
|
|
1367
|
+
return null;
|
|
1542
1368
|
}
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1369
|
+
function inboxTier(dir = INBOX_DIR) {
|
|
1370
|
+
return {
|
|
1371
|
+
name: "inbox",
|
|
1372
|
+
always: true,
|
|
1373
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
1374
|
+
async run(p, ctx) {
|
|
1375
|
+
const file = findInboxFile(dir, p.pmid);
|
|
1376
|
+
if (!file) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
|
|
1377
|
+
const name = file.slice(file.lastIndexOf("/") + 1);
|
|
1378
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
1379
|
+
const bytes = readFileSync3(file);
|
|
1380
|
+
const r = await convertBytes(p, bytes, EXT[ext]);
|
|
1381
|
+
if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
|
|
1382
|
+
ctx.onTry?.(`${name} ok, removed from the inbox`);
|
|
1383
|
+
unlinkSync(file);
|
|
1384
|
+
return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
|
|
1385
|
+
}
|
|
1386
|
+
};
|
|
1551
1387
|
}
|
|
1388
|
+
var inbox = inboxTier();
|
|
1552
1389
|
|
|
1553
|
-
// src/
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
const
|
|
1567
|
-
return
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
`;
|
|
1390
|
+
// src/tiers/openalex.ts
|
|
1391
|
+
var openalex = {
|
|
1392
|
+
name: "openalex",
|
|
1393
|
+
applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
|
|
1394
|
+
async run(p, ctx) {
|
|
1395
|
+
const mail = contactEmail();
|
|
1396
|
+
const key = p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`;
|
|
1397
|
+
let data;
|
|
1398
|
+
try {
|
|
1399
|
+
data = JSON.parse(await fetchText(`https://api.openalex.org/works/${encodeURIComponent(key)}?mailto=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
1400
|
+
} catch (e) {
|
|
1401
|
+
return miss(this.name, errMsg(e));
|
|
1402
|
+
}
|
|
1403
|
+
const candidates = openalexCandidates(data);
|
|
1404
|
+
if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
|
|
1405
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1406
|
+
return download(this.name, p, candidates, ctx);
|
|
1572
1407
|
}
|
|
1573
|
-
}
|
|
1574
|
-
function
|
|
1575
|
-
|
|
1408
|
+
};
|
|
1409
|
+
function openalexCandidates(d) {
|
|
1410
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
1411
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1412
|
+
const out = [];
|
|
1413
|
+
for (const l of locs) {
|
|
1414
|
+
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
1415
|
+
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
1416
|
+
seen.add(url);
|
|
1417
|
+
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
return out;
|
|
1576
1421
|
}
|
|
1577
1422
|
|
|
1578
|
-
// src/
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
});
|
|
1589
|
-
var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
1590
|
-
var kids2 = (n) => n[tagOf2(n)] ?? [];
|
|
1591
|
-
var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
1592
|
-
var SKIP_INLINE = /* @__PURE__ */ new Set(["inline-graphic", "graphic", "private-char"]);
|
|
1593
|
-
function formula(n) {
|
|
1594
|
-
const all = [n, ...descendants(n)];
|
|
1595
|
-
const tex = all.find((x) => tagOf2(x) === "tex-math");
|
|
1596
|
-
if (tex) {
|
|
1597
|
-
let t = plainText(kids2(tex)).trim();
|
|
1598
|
-
const m = /\\begin\{document\}([\s\S]*?)\\end\{document\}/.exec(t);
|
|
1599
|
-
if (m) t = m[1].trim();
|
|
1600
|
-
t = t.replace(/^\$+|\$+$/g, "").trim();
|
|
1601
|
-
return t ? `$${t}$` : "";
|
|
1602
|
-
}
|
|
1603
|
-
const mml = all.find((x) => /^(mml:)?math$/.test(tagOf2(x)));
|
|
1604
|
-
return mml ? plainText(kids2(mml)).replace(/\s+/g, "") : plainText(kids2(n));
|
|
1423
|
+
// src/tiers/pmc.ts
|
|
1424
|
+
function pmcEfetchUrl(pmcid) {
|
|
1425
|
+
const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
|
|
1426
|
+
u.searchParams.set("db", "pmc");
|
|
1427
|
+
u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
|
|
1428
|
+
u.searchParams.set("retmode", "xml");
|
|
1429
|
+
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
1430
|
+
u.searchParams.set("tool", "ai4l-library");
|
|
1431
|
+
u.searchParams.set("email", contactEmail());
|
|
1432
|
+
return u.toString();
|
|
1605
1433
|
}
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1434
|
+
var pmc = {
|
|
1435
|
+
name: "pmc",
|
|
1436
|
+
applies: (p) => p.pmcid ? true : "no pmcid",
|
|
1437
|
+
async run(p) {
|
|
1438
|
+
let xml;
|
|
1439
|
+
try {
|
|
1440
|
+
xml = await fetchText(pmcEfetchUrl(p.pmcid));
|
|
1441
|
+
} catch (e) {
|
|
1442
|
+
return miss(this.name, errMsg(e));
|
|
1443
|
+
}
|
|
1444
|
+
const doc = parseJats(xml);
|
|
1445
|
+
if (!doc.hasBody) return miss(this.name, "no body");
|
|
1446
|
+
const md = renderFulltextMd(doc, p.headLine);
|
|
1447
|
+
const check = checkFulltext(md, p.title);
|
|
1448
|
+
if (!check.ok) return miss(this.name, check.reason);
|
|
1449
|
+
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
1611
1450
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1451
|
+
};
|
|
1452
|
+
|
|
1453
|
+
// src/tiers/tdm.ts
|
|
1454
|
+
var CROSSREF = "https://api.crossref.org/works/";
|
|
1455
|
+
var TOKENED = [
|
|
1456
|
+
{ host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
|
|
1457
|
+
{ host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
|
|
1458
|
+
];
|
|
1459
|
+
var hostOf2 = (url) => {
|
|
1460
|
+
try {
|
|
1461
|
+
return new URL(url).hostname;
|
|
1462
|
+
} catch {
|
|
1463
|
+
return "";
|
|
1464
|
+
}
|
|
1465
|
+
};
|
|
1466
|
+
function tdmCandidates(work, tokens = {}, env = process.env) {
|
|
1467
|
+
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
1468
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1469
|
+
const candidates = [];
|
|
1470
|
+
const missingTokens = /* @__PURE__ */ new Set();
|
|
1471
|
+
for (const l of links) {
|
|
1472
|
+
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
1473
|
+
if (seen.has(l.URL)) {
|
|
1474
|
+
const prev = candidates.find((c) => c.url === l.URL);
|
|
1475
|
+
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
1619
1476
|
continue;
|
|
1620
1477
|
}
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
case "bold":
|
|
1630
|
-
case "b":
|
|
1631
|
-
out += `**${inner()}**`;
|
|
1632
|
-
break;
|
|
1633
|
-
case "sup":
|
|
1634
|
-
out += `^${inner()}^`;
|
|
1635
|
-
break;
|
|
1636
|
-
case "sub":
|
|
1637
|
-
out += `~${inner()}~`;
|
|
1638
|
-
break;
|
|
1639
|
-
case "break":
|
|
1640
|
-
out += " ";
|
|
1641
|
-
break;
|
|
1642
|
-
case "ext-link":
|
|
1643
|
-
case "uri": {
|
|
1644
|
-
const href = attr2(n, "xlink:href") || attr2(n, "href");
|
|
1645
|
-
const text3 = inner().trim() || href;
|
|
1646
|
-
out += href ? `[${text3}](${href})` : text3;
|
|
1647
|
-
break;
|
|
1478
|
+
seen.add(l.URL);
|
|
1479
|
+
const rule = TOKENED.find((r) => r.host.test(hostOf2(l.URL)));
|
|
1480
|
+
let headers;
|
|
1481
|
+
if (rule) {
|
|
1482
|
+
const token = tokens[rule.key]?.trim() || env[rule.env]?.trim();
|
|
1483
|
+
if (!token) {
|
|
1484
|
+
missingTokens.add(rule.key);
|
|
1485
|
+
continue;
|
|
1648
1486
|
}
|
|
1649
|
-
|
|
1650
|
-
case "tex-math":
|
|
1651
|
-
case "mml:math":
|
|
1652
|
-
case "alternatives":
|
|
1653
|
-
out += formula(n);
|
|
1654
|
-
break;
|
|
1655
|
-
default:
|
|
1656
|
-
out += inner();
|
|
1487
|
+
headers = { [rule.header]: token };
|
|
1657
1488
|
}
|
|
1489
|
+
if (/text\/plain/.test(ct)) continue;
|
|
1490
|
+
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf2(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
1658
1491
|
}
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
let out = "";
|
|
1663
|
-
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : plainText(kids2(n));
|
|
1664
|
-
return out;
|
|
1492
|
+
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
1493
|
+
candidates.sort((a, b) => rank(a) - rank(b));
|
|
1494
|
+
return { candidates, missingTokens: [...missingTokens] };
|
|
1665
1495
|
}
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1496
|
+
function tdmTier(tokens = {}) {
|
|
1497
|
+
return {
|
|
1498
|
+
name: "tdm",
|
|
1499
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
1500
|
+
async run(p, ctx) {
|
|
1501
|
+
let work;
|
|
1502
|
+
try {
|
|
1503
|
+
work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
|
|
1504
|
+
} catch (e) {
|
|
1505
|
+
return miss(this.name, errMsg(e));
|
|
1506
|
+
}
|
|
1507
|
+
const { candidates, missingTokens } = tdmCandidates(work, tokens);
|
|
1508
|
+
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
1509
|
+
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
1510
|
+
return download(this.name, p, candidates, ctx);
|
|
1675
1511
|
}
|
|
1676
1512
|
};
|
|
1677
|
-
walk(kids2(table3));
|
|
1678
|
-
if (!rows.length) return "";
|
|
1679
|
-
const width = Math.max(...rows.map((r) => r.length));
|
|
1680
|
-
const pad = (r) => [...r, ...Array(width - r.length).fill("")];
|
|
1681
|
-
const line = (r) => `| ${pad(r).join(" | ")} |`;
|
|
1682
|
-
return [line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line)].join("\n");
|
|
1683
|
-
}
|
|
1684
|
-
function labelCaption(n) {
|
|
1685
|
-
const label = n ? squash2(plainText(kids2(n).filter((c) => tagOf2(c) === "label"))) : "";
|
|
1686
|
-
const cap = kids2(n).find((c) => tagOf2(c) === "caption");
|
|
1687
|
-
const capText = cap ? squash2(inlineText2(kids2(cap))) : "";
|
|
1688
|
-
return [label ? `**${label}.**` : "", capText].filter(Boolean).join(" ");
|
|
1689
1513
|
}
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1514
|
+
var tdm = tdmTier();
|
|
1515
|
+
|
|
1516
|
+
// src/tiers/unpaywall.ts
|
|
1517
|
+
var unpaywall = {
|
|
1518
|
+
name: "unpaywall",
|
|
1519
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
1520
|
+
async run(p, ctx) {
|
|
1521
|
+
const mail = contactEmail();
|
|
1522
|
+
let data;
|
|
1523
|
+
try {
|
|
1524
|
+
data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
1525
|
+
} catch (e) {
|
|
1526
|
+
return miss(this.name, errMsg(e));
|
|
1697
1527
|
}
|
|
1698
|
-
const
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
}
|
|
1713
|
-
case "title":
|
|
1714
|
-
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash2(inlineText2(c))}`);
|
|
1715
|
-
break;
|
|
1716
|
-
case "p": {
|
|
1717
|
-
const BLOCK = /* @__PURE__ */ new Set(["table-wrap", "fig", "list", "disp-formula", "disp-quote", "boxed-text", "supplementary-material"]);
|
|
1718
|
-
const inline2 = c.filter((x) => "#text" in x || !BLOCK.has(tagOf2(x)));
|
|
1719
|
-
const blocks2 = c.filter((x) => !("#text" in x) && BLOCK.has(tagOf2(x)));
|
|
1720
|
-
const s = squash2(inlineText2(inline2));
|
|
1721
|
-
if (s) out.push(s);
|
|
1722
|
-
out.push(...renderBlocks(blocks2, depth));
|
|
1723
|
-
break;
|
|
1724
|
-
}
|
|
1725
|
-
case "list": {
|
|
1726
|
-
const ordered = /order|alpha|roman|number/i.test(attr2(n, "list-type"));
|
|
1727
|
-
let i = 0;
|
|
1728
|
-
for (const li of c.filter((x) => tagOf2(x) === "list-item")) {
|
|
1729
|
-
i++;
|
|
1730
|
-
const body = renderBlocks(kids2(li).filter((x) => tagOf2(x) !== "label"), depth).join(" ");
|
|
1731
|
-
out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
1732
|
-
}
|
|
1733
|
-
break;
|
|
1734
|
-
}
|
|
1735
|
-
case "table-wrap": {
|
|
1736
|
-
const head = labelCaption(n);
|
|
1737
|
-
if (head) out.push(head);
|
|
1738
|
-
for (const tbl of c.filter((x) => tagOf2(x) === "table")) {
|
|
1739
|
-
const md = renderTable(tbl);
|
|
1740
|
-
if (md) out.push(md);
|
|
1741
|
-
}
|
|
1742
|
-
const foot = c.find((x) => tagOf2(x) === "table-wrap-foot");
|
|
1743
|
-
if (foot) out.push(...renderBlocks(kids2(foot), depth));
|
|
1744
|
-
break;
|
|
1745
|
-
}
|
|
1746
|
-
case "fig": {
|
|
1747
|
-
const head = labelCaption(n);
|
|
1748
|
-
out.push(head || "*(figure)*");
|
|
1749
|
-
break;
|
|
1750
|
-
}
|
|
1751
|
-
case "disp-formula": {
|
|
1752
|
-
const s = squash2(formula(n));
|
|
1753
|
-
if (s) out.push(s);
|
|
1754
|
-
break;
|
|
1755
|
-
}
|
|
1756
|
-
case "disp-quote":
|
|
1757
|
-
out.push(...renderBlocks(c, depth).map((l) => `> ${l}`));
|
|
1758
|
-
break;
|
|
1759
|
-
case "def-list":
|
|
1760
|
-
for (const d of c.filter((x) => tagOf2(x) === "def-item")) {
|
|
1761
|
-
const term = d["def-item"].find((x) => tagOf2(x) === "term");
|
|
1762
|
-
const def = d["def-item"].find((x) => tagOf2(x) === "def");
|
|
1763
|
-
out.push(`**${squash2(inlineText2(term ? kids2(term) : []))}**: ${def ? renderBlocks(kids2(def), depth).join(" ") : ""}`);
|
|
1764
|
-
}
|
|
1765
|
-
break;
|
|
1766
|
-
case "ref-list": {
|
|
1767
|
-
const title = c.find((x) => tagOf2(x) === "title");
|
|
1768
|
-
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${title ? squash2(inlineText2(kids2(title))) : "References"}`);
|
|
1769
|
-
for (const r of c.filter((x) => tagOf2(x) === "ref")) {
|
|
1770
|
-
const rk = kids2(r);
|
|
1771
|
-
const label = squash2(plainText(rk.filter((x) => tagOf2(x) === "label")));
|
|
1772
|
-
out.push(`- ${label ? `${label} ` : ""}${renderCitation(rk.filter((x) => tagOf2(x) !== "label"))}`);
|
|
1773
|
-
}
|
|
1774
|
-
break;
|
|
1775
|
-
}
|
|
1776
|
-
case "supplementary-material":
|
|
1777
|
-
case "graphic":
|
|
1778
|
-
case "media":
|
|
1779
|
-
case "fn-group":
|
|
1780
|
-
case "label":
|
|
1781
|
-
break;
|
|
1782
|
-
default:
|
|
1783
|
-
out.push(...renderBlocks(c, depth));
|
|
1528
|
+
const candidates = unpaywallCandidates(data);
|
|
1529
|
+
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
1530
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1531
|
+
return download(this.name, p, candidates, ctx);
|
|
1532
|
+
}
|
|
1533
|
+
};
|
|
1534
|
+
function unpaywallCandidates(d) {
|
|
1535
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
|
|
1536
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1537
|
+
const out = [];
|
|
1538
|
+
for (const l of locs) {
|
|
1539
|
+
for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
|
|
1540
|
+
if (!url || seen.has(url)) continue;
|
|
1541
|
+
seen.add(url);
|
|
1542
|
+
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
1784
1543
|
}
|
|
1785
1544
|
}
|
|
1786
1545
|
return out;
|
|
1787
1546
|
}
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
return [sur, giv].filter(Boolean).join(" ");
|
|
1805
|
-
}
|
|
1806
|
-
return "";
|
|
1807
|
-
}).filter(Boolean);
|
|
1808
|
-
const vol = get("volume")[0], issue = get("issue")[0], fp = get("fpage")[0], lp = get("lpage")[0];
|
|
1809
|
-
const pages = fp ? lp ? `${fp}-${lp}` : fp : get("elocation-id")[0];
|
|
1810
|
-
const where = [get("source")[0], get("year")[0] ? `${get("year")[0]};` : "", vol ? `${vol}${issue ? `(${issue})` : ""}` : "", pages ? `:${pages}` : ""].filter(Boolean).join("").replace(/;$/, "");
|
|
1811
|
-
const ids = f.filter((x) => tagOf2(x) === "pub-id").map((x) => `${attr2(x, "pub-id-type")}:${squash2(plainText(kids2(x)))}`);
|
|
1812
|
-
return [names.join(", "), [...get("article-title"), ...get("chapter-title")][0], where, ...ids].filter(Boolean).join(". ").replace(/\.\./g, ".");
|
|
1813
|
-
}
|
|
1814
|
-
var FN_LABELS = {
|
|
1815
|
-
"conflict": "Conflicts of interest",
|
|
1816
|
-
"COI-statement": "Conflicts of interest",
|
|
1817
|
-
"coi-statement": "Conflicts of interest",
|
|
1818
|
-
"financial-disclosure": "Financial disclosure",
|
|
1819
|
-
"supported-by": "Funding",
|
|
1820
|
-
"funding": "Funding",
|
|
1821
|
-
"equal": "Author note",
|
|
1822
|
-
"con": "Contributions",
|
|
1823
|
-
"contributed-by": "Contributions",
|
|
1824
|
-
"other": ""
|
|
1825
|
-
};
|
|
1826
|
-
function declarations(meta, back) {
|
|
1827
|
-
const out = [];
|
|
1828
|
-
const fg = meta ? findDeep2(kids2(meta), "funding-group") : void 0;
|
|
1829
|
-
if (fg) {
|
|
1830
|
-
const statements = kids2(fg).filter((x) => tagOf2(x) === "funding-statement").map((x) => squash2(inlineText2(kids2(x)))).filter(Boolean);
|
|
1831
|
-
const awards = kids2(fg).filter((x) => tagOf2(x) === "award-group").map((ag) => {
|
|
1832
|
-
const src = kids2(ag).filter((x) => tagOf2(x) === "funding-source").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
|
|
1833
|
-
const ids = kids2(ag).filter((x) => tagOf2(x) === "award-id").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
|
|
1834
|
-
return [src.join(", "), ids.length ? `(${ids.join(", ")})` : ""].filter(Boolean).join(" ");
|
|
1835
|
-
}).filter(Boolean);
|
|
1836
|
-
const text3 = statements.length ? statements.join(" ") : awards.join("; ");
|
|
1837
|
-
if (text3) out.push(`Funding: ${text3}`);
|
|
1838
|
-
}
|
|
1839
|
-
const fns = [];
|
|
1840
|
-
const notes = meta ? findDeep2(kids2(meta), "author-notes") : void 0;
|
|
1841
|
-
if (notes) fns.push(...kids2(notes).filter((x) => tagOf2(x) === "fn"));
|
|
1842
|
-
if (back) for (const g of kids2(back).filter((x) => tagOf2(x) === "fn-group")) fns.push(...kids2(g).filter((x) => tagOf2(x) === "fn"));
|
|
1843
|
-
for (const fn of fns) {
|
|
1844
|
-
const text3 = renderBlocks(kids2(fn).filter((x) => tagOf2(x) !== "label"), 1).join(" ").trim();
|
|
1845
|
-
if (!text3) continue;
|
|
1846
|
-
const label = FN_LABELS[attr2(fn, "fn-type")] ?? "";
|
|
1847
|
-
out.push(label && !new RegExp(`^[*_]*${label.split(" ")[0]}`, "i").test(text3) ? `${label}: ${text3}` : text3);
|
|
1848
|
-
}
|
|
1849
|
-
return out;
|
|
1850
|
-
}
|
|
1851
|
-
function find(nodes, path) {
|
|
1852
|
-
let cur = nodes;
|
|
1853
|
-
let hit;
|
|
1854
|
-
for (const tag of path) {
|
|
1855
|
-
hit = cur.find((x) => tagOf2(x) === tag);
|
|
1856
|
-
if (!hit) return void 0;
|
|
1857
|
-
cur = kids2(hit);
|
|
1858
|
-
}
|
|
1859
|
-
return hit;
|
|
1860
|
-
}
|
|
1861
|
-
function findDeep2(nodes, tag) {
|
|
1862
|
-
for (const n of nodes) {
|
|
1863
|
-
if ("#text" in n) continue;
|
|
1864
|
-
if (tagOf2(n) === tag) return n;
|
|
1865
|
-
const hit = findDeep2(kids2(n), tag);
|
|
1866
|
-
if (hit) return hit;
|
|
1867
|
-
}
|
|
1868
|
-
return void 0;
|
|
1869
|
-
}
|
|
1870
|
-
function parseJats(xml) {
|
|
1871
|
-
const doc = parser3.parse(xml);
|
|
1872
|
-
const article = find(doc, ["pmc-articleset", "article"]) ?? find(doc, ["article"]) ?? findDeep2(doc, "article");
|
|
1873
|
-
const a = article ? kids2(article) : [];
|
|
1874
|
-
const front = find(a, ["front"]);
|
|
1875
|
-
const meta = front ? find(kids2(front), ["article-meta"]) ?? front : void 0;
|
|
1876
|
-
const titleNode = meta ? find(kids2(meta), ["title-group", "article-title"]) ?? findDeep2(kids2(meta), "article-title") : void 0;
|
|
1877
|
-
const abstractNode = meta ? kids2(meta).find((x) => tagOf2(x) === "abstract" && !attr2(x, "abstract-type")) ?? findDeep2(kids2(meta), "abstract") : void 0;
|
|
1878
|
-
const body = find(a, ["body"]);
|
|
1879
|
-
const back = find(a, ["back"]);
|
|
1547
|
+
|
|
1548
|
+
// src/tiers/index.ts
|
|
1549
|
+
var BUILTIN = [
|
|
1550
|
+
{ name: "inbox", make: () => inbox },
|
|
1551
|
+
{ name: "pmc", make: () => pmc },
|
|
1552
|
+
{ name: "europepmc", make: () => europepmc },
|
|
1553
|
+
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
1554
|
+
{ name: "unpaywall", make: () => unpaywall },
|
|
1555
|
+
{ name: "openalex", make: () => openalex },
|
|
1556
|
+
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
1557
|
+
];
|
|
1558
|
+
|
|
1559
|
+
// src/tiers/url.ts
|
|
1560
|
+
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
1561
|
+
function urlTier(name, template, headers) {
|
|
1562
|
+
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
1880
1563
|
return {
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1564
|
+
name,
|
|
1565
|
+
applies(p) {
|
|
1566
|
+
const missing = used.filter((k) => !p[k]);
|
|
1567
|
+
return missing.length ? `no ${missing.join("/")}` : true;
|
|
1568
|
+
},
|
|
1569
|
+
async run(p, ctx) {
|
|
1570
|
+
if (!used.length) return miss(name, "template has no placeholder");
|
|
1571
|
+
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
1572
|
+
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
1573
|
+
}
|
|
1887
1574
|
};
|
|
1888
1575
|
}
|
|
1889
|
-
function renderFulltextMd(doc, headLine2) {
|
|
1890
|
-
const parts = [`# ${doc.title}`, "", headLine2, ""];
|
|
1891
|
-
if (doc.abstract.length) parts.push("## Abstract", "", ...doc.abstract.flatMap((b) => [b, ""]));
|
|
1892
|
-
parts.push(...doc.body.flatMap((b) => [b, ""]));
|
|
1893
|
-
if (doc.declarations.length) parts.push("## Declarations", "", ...doc.declarations.flatMap((b) => [b, ""]));
|
|
1894
|
-
parts.push(...doc.back.flatMap((b) => [b, ""]));
|
|
1895
|
-
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1896
|
-
}
|
|
1897
|
-
|
|
1898
|
-
// src/browser.ts
|
|
1899
|
-
import { existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
1900
|
-
import { join as join4 } from "path";
|
|
1901
|
-
import { chromium } from "playwright-core";
|
|
1902
1576
|
|
|
1903
|
-
// src/
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1577
|
+
// src/config.ts
|
|
1578
|
+
var VARIABLES = [
|
|
1579
|
+
"AI4L_LIBRARY_DIR",
|
|
1580
|
+
"AI4L_LIBRARY_ENV",
|
|
1581
|
+
"AI4L_LIBRARY_TIERS",
|
|
1582
|
+
"AI4L_LIBRARY_CONTACT_EMAIL",
|
|
1583
|
+
"AI4L_LIBRARY_ELSEVIER_API_KEY",
|
|
1584
|
+
"AI4L_LIBRARY_WILEY_TDM_TOKEN",
|
|
1585
|
+
"AI4L_LIBRARY_CORE_API_KEY",
|
|
1586
|
+
"AI4L_LIBRARY_PROXY_URL",
|
|
1587
|
+
"AI4L_LIBRARY_PROXY_HEADERS",
|
|
1588
|
+
"AI4L_LIBRARY_BROWSER_1",
|
|
1589
|
+
"AI4L_LIBRARY_BROWSER_2",
|
|
1590
|
+
"AI4L_LIBRARY_BROWSER_3",
|
|
1591
|
+
"AI4L_LIBRARY_BROWSER_HEADLESS",
|
|
1592
|
+
"AI4L_LIBRARY_VERIFY_URL",
|
|
1593
|
+
"AI4L_LIBRARY_VERIFY_API_KEY",
|
|
1594
|
+
"AI4L_LIBRARY_VERIFY_MODEL",
|
|
1595
|
+
"AI4L_LIBRARY_VERIFY_THRESHOLD",
|
|
1596
|
+
"AI4L_LIBRARY_VERIFY_TIMEOUT_SECS",
|
|
1597
|
+
"AI4L_LIBRARY_RETRY_AFTER_DAYS",
|
|
1598
|
+
"AI4L_LIBRARY_RETRY_BLOCKED",
|
|
1599
|
+
"AI4L_LIBRARY_REFRESH_AFTER_DAYS",
|
|
1600
|
+
"NCBI_API_KEY",
|
|
1601
|
+
"BRIGHTDATA_API_TOKEN",
|
|
1602
|
+
"BRIGHTDATA_ZONE"
|
|
1603
|
+
];
|
|
1604
|
+
var sources = (env = process.env) => Object.fromEntries(VARIABLES.map((v) => [v, sourceOf(v, env)]));
|
|
1605
|
+
function settings(env = process.env) {
|
|
1928
1606
|
return {
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
md: join3(entryDir, `${entry}-abstract.md`),
|
|
1933
|
-
meta: join3(entryDir, `${entry}-meta.md`),
|
|
1934
|
-
fulltextMd: join3(entryDir, `${entry}-fulltext.md`),
|
|
1935
|
-
fulltextRaw: (format) => join3(entryDir, `${entry}-fulltext.${format}`)
|
|
1607
|
+
retry_after_days: num("AI4L_LIBRARY_RETRY_AFTER_DAYS", 1, env),
|
|
1608
|
+
retry_blocked: bool("AI4L_LIBRARY_RETRY_BLOCKED", false, env),
|
|
1609
|
+
refresh_after_days: num("AI4L_LIBRARY_REFRESH_AFTER_DAYS", 30, env)
|
|
1936
1610
|
};
|
|
1937
1611
|
}
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
const
|
|
1944
|
-
if (
|
|
1612
|
+
function proxyHeaders(spec) {
|
|
1613
|
+
const out = {};
|
|
1614
|
+
for (const part of spec.split(";")) {
|
|
1615
|
+
const i = part.indexOf(":");
|
|
1616
|
+
if (i < 0) continue;
|
|
1617
|
+
const name = part.slice(0, i).trim(), value = part.slice(i + 1).trim();
|
|
1618
|
+
if (name && value) out[name] = value;
|
|
1945
1619
|
}
|
|
1946
|
-
const h = str("AI4L_LIBRARY_BROWSER_HEADLESS", env).toLowerCase();
|
|
1947
|
-
if (h === "true" || h === "false") out.headless = h === "true";
|
|
1948
1620
|
return out;
|
|
1949
1621
|
}
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1622
|
+
function resolveTiers(env = process.env) {
|
|
1623
|
+
const problems = [];
|
|
1624
|
+
const available = /* @__PURE__ */ new Map();
|
|
1625
|
+
const tokens = { elsevier: str("AI4L_LIBRARY_ELSEVIER_API_KEY", env) || void 0, wiley: str("AI4L_LIBRARY_WILEY_TDM_TOKEN", env) || void 0 };
|
|
1626
|
+
for (const t of BUILTIN) available.set(t.name, t.make({ tokens, api_key: str("AI4L_LIBRARY_CORE_API_KEY", env) || void 0 }));
|
|
1627
|
+
const proxyUrl = str("AI4L_LIBRARY_PROXY_URL", env);
|
|
1628
|
+
if (proxyUrl) available.set("proxy", urlTier("proxy", proxyUrl, proxyHeaders(str("AI4L_LIBRARY_PROXY_HEADERS", env))));
|
|
1629
|
+
const listed = str("AI4L_LIBRARY_TIERS", env);
|
|
1630
|
+
const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
|
|
1631
|
+
const tiers = [], seen = /* @__PURE__ */ new Set();
|
|
1632
|
+
for (const name of order2) {
|
|
1633
|
+
if (seen.has(name)) continue;
|
|
1634
|
+
seen.add(name);
|
|
1635
|
+
const t = available.get(name);
|
|
1636
|
+
if (t) tiers.push(t);
|
|
1637
|
+
else if (name === "proxy") problems.push('AI4L_LIBRARY_TIERS: "proxy" listed, but AI4L_LIBRARY_PROXY_URL is unset');
|
|
1638
|
+
else problems.push(`AI4L_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when AI4L_LIBRARY_PROXY_URL is set)`);
|
|
1639
|
+
}
|
|
1640
|
+
const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
|
|
1641
|
+
return { tiers, all, source: sourceOf("AI4L_LIBRARY_TIERS", env), problems };
|
|
1642
|
+
}
|
|
1643
|
+
|
|
1644
|
+
// src/log.ts
|
|
1645
|
+
import { appendFileSync, mkdirSync as mkdirSync2, renameSync, statSync } from "fs";
|
|
1646
|
+
import { hostname } from "os";
|
|
1647
|
+
import { join as join5 } from "path";
|
|
1648
|
+
var HOST = hostname();
|
|
1649
|
+
var LOG_FILE = join5(AI4L_LIBRARY_DIR, "library.log");
|
|
1650
|
+
function stamp() {
|
|
1651
|
+
const d = /* @__PURE__ */ new Date();
|
|
1652
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
1653
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
1654
|
+
}
|
|
1655
|
+
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
1656
|
+
function rotateIfLarge(file, max = LOG_MAX_BYTES) {
|
|
1657
|
+
let size = 0;
|
|
1956
1658
|
try {
|
|
1957
|
-
|
|
1958
|
-
return existsSync4(p) ? p : null;
|
|
1659
|
+
size = statSync(file).size;
|
|
1959
1660
|
} catch {
|
|
1960
|
-
return
|
|
1661
|
+
return false;
|
|
1961
1662
|
}
|
|
1962
|
-
|
|
1963
|
-
function browserLabel(endpoint) {
|
|
1964
|
-
if (rungKind(endpoint) === "local") return "local";
|
|
1663
|
+
if (size <= max) return false;
|
|
1965
1664
|
try {
|
|
1966
|
-
|
|
1967
|
-
return
|
|
1665
|
+
renameSync(file, `${file}.1`);
|
|
1666
|
+
return true;
|
|
1968
1667
|
} catch {
|
|
1969
|
-
return
|
|
1668
|
+
return false;
|
|
1970
1669
|
}
|
|
1971
1670
|
}
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
}
|
|
1977
|
-
function browserRungs(cfg = browserConfig()) {
|
|
1978
|
-
return cfg.browsers.map((endpoint) => ({
|
|
1979
|
-
endpoint,
|
|
1980
|
-
label: browserLabel(endpoint),
|
|
1981
|
-
kind: rungKind(endpoint),
|
|
1982
|
-
available: rungKind(endpoint) !== "local" || localChromium() !== null,
|
|
1983
|
-
headless: browserHeadless(endpoint, cfg)
|
|
1984
|
-
}));
|
|
1985
|
-
}
|
|
1986
|
-
var CHALLENGE = /just a moment|nur einen moment|attention required|verify you are human|human verification|checking your browser|access denied/i;
|
|
1987
|
-
var CONNECT_TIMEOUT_MS = 2e4;
|
|
1988
|
-
var NAV_TIMEOUT_MS = 2e4;
|
|
1989
|
-
var CHALLENGE_WAIT_MS = 8e3;
|
|
1990
|
-
var DOWNLOAD_WAIT_MS = 1e4;
|
|
1991
|
-
var IDLE_CLOSE_MS = 12e4;
|
|
1992
|
-
var ctxs = /* @__PURE__ */ new Map();
|
|
1993
|
-
var idleTimer = null;
|
|
1994
|
-
function profileDir() {
|
|
1995
|
-
const dir = join4(AI4L_LIBRARY_DIR, ".browser");
|
|
1996
|
-
mkdirSync2(join4(dir, "Default"), { recursive: true });
|
|
1997
|
-
const prefs = join4(dir, "Default", "Preferences");
|
|
1998
|
-
let p = {};
|
|
1671
|
+
function log(tool, id, message) {
|
|
1672
|
+
const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
1673
|
+
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
|
|
1674
|
+
`;
|
|
1999
1675
|
try {
|
|
2000
|
-
|
|
1676
|
+
mkdirSync2(AI4L_LIBRARY_DIR, { recursive: true });
|
|
1677
|
+
rotateIfLarge(LOG_FILE);
|
|
1678
|
+
appendFileSync(LOG_FILE, line);
|
|
2001
1679
|
} catch {
|
|
2002
1680
|
}
|
|
2003
|
-
p.plugins = { ...p.plugins ?? {}, always_open_pdf_externally: true };
|
|
2004
|
-
p.download = { ...p.download ?? {}, prompt_for_download: false };
|
|
2005
|
-
writeFileSync2(prefs, JSON.stringify(p));
|
|
2006
|
-
return dir;
|
|
2007
|
-
}
|
|
2008
|
-
var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
|
|
2009
|
-
var LOCAL_WINDOW_ARGS = [...LAUNCH_ARGS, "--window-position=4000,4000"];
|
|
2010
|
-
var TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "Europe/Berlin";
|
|
2011
|
-
var CONTEXT_OPTS = { acceptDownloads: true, viewport: null, locale: "en-US", timezoneId: TIMEZONE };
|
|
2012
|
-
function stateFile(endpoint) {
|
|
2013
|
-
const dir = join4(AI4L_LIBRARY_DIR, ".browser");
|
|
2014
|
-
mkdirSync2(dir, { recursive: true });
|
|
2015
|
-
return join4(dir, `${browserLabel(endpoint).replace(/[^A-Za-z0-9.-]+/g, "_")}.json`);
|
|
2016
|
-
}
|
|
2017
|
-
async function context(endpoint, cfg) {
|
|
2018
|
-
let p = ctxs.get(endpoint);
|
|
2019
|
-
if (!p) {
|
|
2020
|
-
p = (async () => {
|
|
2021
|
-
const headless = browserHeadless(endpoint, cfg);
|
|
2022
|
-
const drop = () => {
|
|
2023
|
-
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
2024
|
-
};
|
|
2025
|
-
if (isCdp(endpoint)) {
|
|
2026
|
-
const browser = await chromium.connectOverCDP(cdpHttp(endpoint), { timeout: CONNECT_TIMEOUT_MS });
|
|
2027
|
-
browser.on("disconnected", drop);
|
|
2028
|
-
const ctx2 = browser.contexts()[0] ?? await browser.newContext(CONTEXT_OPTS);
|
|
2029
|
-
return { ctx: ctx2, browser };
|
|
2030
|
-
}
|
|
2031
|
-
if (isRemote(endpoint)) {
|
|
2032
|
-
const browser = await chromium.connect(endpoint, {
|
|
2033
|
-
timeout: CONNECT_TIMEOUT_MS,
|
|
2034
|
-
headers: { "x-playwright-launch-options": JSON.stringify({ headless, args: LAUNCH_ARGS, ignoreDefaultArgs: ["--enable-automation"] }) }
|
|
2035
|
-
});
|
|
2036
|
-
browser.on("disconnected", drop);
|
|
2037
|
-
const state = stateFile(endpoint);
|
|
2038
|
-
const ctx2 = await browser.newContext({ ...CONTEXT_OPTS, ...existsSync4(state) ? { storageState: state } : {} });
|
|
2039
|
-
return { ctx: ctx2, browser };
|
|
2040
|
-
}
|
|
2041
|
-
if (!localChromium()) throw new Error(`no local Chromium (${INSTALL_HINT})`);
|
|
2042
|
-
const ctx = await chromium.launchPersistentContext(profileDir(), {
|
|
2043
|
-
channel: "chromium",
|
|
2044
|
-
headless,
|
|
2045
|
-
...CONTEXT_OPTS,
|
|
2046
|
-
args: headless ? LAUNCH_ARGS : LOCAL_WINDOW_ARGS,
|
|
2047
|
-
ignoreDefaultArgs: ["--enable-automation"]
|
|
2048
|
-
});
|
|
2049
|
-
ctx.on("close", drop);
|
|
2050
|
-
return { ctx, browser: null };
|
|
2051
|
-
})();
|
|
2052
|
-
ctxs.set(endpoint, p);
|
|
2053
|
-
p.catch(() => {
|
|
2054
|
-
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
2055
|
-
});
|
|
2056
|
-
}
|
|
2057
|
-
return (await p).ctx;
|
|
2058
|
-
}
|
|
2059
|
-
function touchIdle() {
|
|
2060
|
-
if (idleTimer) clearTimeout(idleTimer);
|
|
2061
|
-
idleTimer = setTimeout(() => {
|
|
2062
|
-
void closeBrowsers();
|
|
2063
|
-
}, IDLE_CLOSE_MS);
|
|
2064
|
-
idleTimer.unref();
|
|
2065
|
-
}
|
|
2066
|
-
async function closeBrowsers() {
|
|
2067
|
-
const open = [...ctxs.entries()];
|
|
2068
|
-
ctxs.clear();
|
|
2069
|
-
for (const [endpoint, p] of open) {
|
|
2070
|
-
try {
|
|
2071
|
-
const { ctx, browser } = await p;
|
|
2072
|
-
if (isCdp(endpoint)) {
|
|
2073
|
-
await browser?.close().catch(() => void 0);
|
|
2074
|
-
continue;
|
|
2075
|
-
}
|
|
2076
|
-
if (browser) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
2077
|
-
await ctx.close().catch(() => void 0);
|
|
2078
|
-
await browser?.close().catch(() => void 0);
|
|
2079
|
-
} catch {
|
|
2080
|
-
}
|
|
2081
|
-
}
|
|
2082
|
-
}
|
|
2083
|
-
async function browserFetch(url, endpoint, cfg = browserConfig()) {
|
|
2084
|
-
const ctx = await context(endpoint, cfg);
|
|
2085
|
-
touchIdle();
|
|
2086
|
-
const page = await ctx.newPage();
|
|
2087
|
-
try {
|
|
2088
|
-
let pdfBest = null;
|
|
2089
|
-
const pdfReads = [];
|
|
2090
|
-
let status, contentType = "";
|
|
2091
|
-
page.on("response", (r) => {
|
|
2092
|
-
if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) {
|
|
2093
|
-
status = r.status();
|
|
2094
|
-
contentType = r.headers()["content-type"] ?? "";
|
|
2095
|
-
}
|
|
2096
|
-
if (r.status() !== 200 || !/application\/pdf/i.test(r.headers()["content-type"] ?? "")) return;
|
|
2097
|
-
pdfReads.push(r.body().then((b) => {
|
|
2098
|
-
if (b.subarray(0, 5).toString("latin1") === "%PDF-" && (!pdfBest || b.length > pdfBest.length)) pdfBest = b;
|
|
2099
|
-
}).catch(() => void 0));
|
|
2100
|
-
});
|
|
2101
|
-
let download2 = null;
|
|
2102
|
-
const dl = page.waitForEvent("download", { timeout: NAV_TIMEOUT_MS + CHALLENGE_WAIT_MS + DOWNLOAD_WAIT_MS }).then((d) => {
|
|
2103
|
-
download2 = d;
|
|
2104
|
-
return d;
|
|
2105
|
-
}).catch(() => null);
|
|
2106
|
-
let navError = null;
|
|
2107
|
-
const nav = page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).then((resp) => {
|
|
2108
|
-
if (status === void 0) {
|
|
2109
|
-
status = resp?.status();
|
|
2110
|
-
contentType = resp?.headers()["content-type"] ?? "";
|
|
2111
|
-
}
|
|
2112
|
-
}).catch((e) => {
|
|
2113
|
-
navError = (e instanceof Error ? e.message : String(e)).split("\n")[0].slice(0, 80);
|
|
2114
|
-
});
|
|
2115
|
-
await Promise.race([nav, dl]);
|
|
2116
|
-
const pause = (ms) => Promise.race([dl, new Promise((r) => setTimeout(r, ms))]);
|
|
2117
|
-
if (download2) {
|
|
2118
|
-
} else if (navError && !/download is starting|net::ERR_ABORTED/i.test(navError)) {
|
|
2119
|
-
await pause(2e3);
|
|
2120
|
-
if (!download2) throw new Error(`navigation: ${navError}`);
|
|
2121
|
-
} else {
|
|
2122
|
-
await pause(2e3);
|
|
2123
|
-
const t0 = Date.now();
|
|
2124
|
-
while (!download2 && Date.now() - t0 < CHALLENGE_WAIT_MS) {
|
|
2125
|
-
const title2 = await page.title().catch(() => "");
|
|
2126
|
-
if (!CHALLENGE.test(title2)) break;
|
|
2127
|
-
await pause(1e3);
|
|
2128
|
-
}
|
|
2129
|
-
if (!download2) await pause(DOWNLOAD_WAIT_MS / 5);
|
|
2130
|
-
}
|
|
2131
|
-
if (download2) {
|
|
2132
|
-
const d = download2;
|
|
2133
|
-
return { bytes: readFileSync3(await d.path()), contentType: "application/pdf", url: d.url(), via: "download" };
|
|
2134
|
-
}
|
|
2135
|
-
const title = await page.title().catch(() => "");
|
|
2136
|
-
if (CHALLENGE.test(title)) throw new Error(`challenge not cleared (${title.slice(0, 40)})`);
|
|
2137
|
-
if (status && status >= 400) throw new Error(`HTTP ${status}`);
|
|
2138
|
-
const docType = await page.evaluate(() => document.contentType).catch(() => "");
|
|
2139
|
-
const inViewer = /application\/pdf/i.test(contentType) || /application\/pdf/i.test(docType);
|
|
2140
|
-
if (inViewer) {
|
|
2141
|
-
const t1 = Date.now();
|
|
2142
|
-
while (Date.now() - t1 < DOWNLOAD_WAIT_MS) {
|
|
2143
|
-
await Promise.all(pdfReads);
|
|
2144
|
-
if (pdfBest && pdfBest.length > 4096) break;
|
|
2145
|
-
if (!pdfReads.length && Date.now() - t1 > 3e3) break;
|
|
2146
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
2147
|
-
}
|
|
2148
|
-
if (pdfBest) return { bytes: pdfBest, contentType: "application/pdf", url: page.url(), via: "response" };
|
|
2149
|
-
}
|
|
2150
|
-
if (inViewer) {
|
|
2151
|
-
const r = await ctx.request.get(page.url(), { timeout: NAV_TIMEOUT_MS });
|
|
2152
|
-
if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
|
|
2153
|
-
return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
|
|
2154
|
-
}
|
|
2155
|
-
const html = await page.content();
|
|
2156
|
-
return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
|
|
2157
|
-
} finally {
|
|
2158
|
-
await page.close().catch(() => void 0);
|
|
2159
|
-
if (isRemote(endpoint)) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
2160
|
-
}
|
|
2161
1681
|
}
|
|
2162
1682
|
|
|
2163
|
-
// src/
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
var unlockerConfigured = () => !!process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
2167
|
-
async function unlock(url, fetchImpl = fetch) {
|
|
2168
|
-
const token = process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
2169
|
-
if (!token) throw new Error("unlocker not configured");
|
|
2170
|
-
const zone = process.env.BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
|
|
2171
|
-
const res = await fetchImpl(ENDPOINT, {
|
|
2172
|
-
method: "POST",
|
|
2173
|
-
headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
|
|
2174
|
-
body: JSON.stringify({ zone, url, format: "raw" }),
|
|
2175
|
-
signal: AbortSignal.timeout(TIMEOUT_MS2)
|
|
2176
|
-
});
|
|
2177
|
-
if (!res.ok) throw new Error(`unlocker HTTP ${res.status}`);
|
|
2178
|
-
return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
|
|
2179
|
-
}
|
|
1683
|
+
// src/tools.ts
|
|
1684
|
+
import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
|
|
1685
|
+
import { z } from "zod";
|
|
2180
1686
|
|
|
2181
|
-
// src/
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
return "published";
|
|
2186
|
-
}
|
|
2187
|
-
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
2188
|
-
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
1687
|
+
// src/entry.ts
|
|
1688
|
+
import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readdirSync as readdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
|
|
1689
|
+
import { join as join6 } from "path";
|
|
1690
|
+
import { basename, dirname } from "path";
|
|
2189
1691
|
|
|
2190
|
-
// src/
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
1692
|
+
// src/pubmed.ts
|
|
1693
|
+
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
1694
|
+
var CORRECTION_TYPES = {
|
|
1695
|
+
ErratumIn: "erratum_in",
|
|
1696
|
+
RetractionIn: "retraction_in",
|
|
1697
|
+
PartialRetractionIn: "partial_retraction_in",
|
|
1698
|
+
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
1699
|
+
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
1700
|
+
UpdateIn: "update_in"
|
|
2197
1701
|
};
|
|
2198
|
-
var
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
ctx.tried.add(c.url);
|
|
2211
|
-
let r = await tryOne(p, c, tier);
|
|
2212
|
-
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
2213
|
-
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
2214
|
-
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
2215
|
-
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
2216
|
-
for (const b of browserRungs()) {
|
|
2217
|
-
if (r.ok || c.headers || c.service || !b.available || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
2218
|
-
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
2219
|
-
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
2220
|
-
}
|
|
2221
|
-
if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
2222
|
-
r = await tryOne(p, target, tier, "unlocker");
|
|
2223
|
-
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
2224
|
-
}
|
|
2225
|
-
if (r.ok) return r;
|
|
2226
|
-
reasons.push(`${r.reason}: ${hostOf(c.url)}`);
|
|
2227
|
-
}
|
|
2228
|
-
return miss(tier, reasons.join("; "));
|
|
1702
|
+
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
1703
|
+
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
1704
|
+
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
1705
|
+
function efetchUrl(pmid) {
|
|
1706
|
+
const u = new URL(EFETCH);
|
|
1707
|
+
u.searchParams.set("db", "pubmed");
|
|
1708
|
+
u.searchParams.set("id", pmid);
|
|
1709
|
+
u.searchParams.set("retmode", "xml");
|
|
1710
|
+
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
1711
|
+
u.searchParams.set("tool", "ai4l-library");
|
|
1712
|
+
u.searchParams.set("email", contactEmail());
|
|
1713
|
+
return u.toString();
|
|
2229
1714
|
}
|
|
2230
|
-
|
|
2231
|
-
async function tryOne(p, c, tier, via = "plain") {
|
|
2232
|
-
let got;
|
|
1715
|
+
async function fetchPubmedXml(pmid) {
|
|
2233
1716
|
try {
|
|
2234
|
-
|
|
1717
|
+
return await fetchText(efetchUrl(pmid));
|
|
2235
1718
|
} catch (e) {
|
|
2236
|
-
|
|
2237
|
-
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
2238
|
-
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
1719
|
+
throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2239
1720
|
}
|
|
2240
|
-
const { bytes, contentType } = got;
|
|
2241
|
-
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
2242
|
-
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
2243
|
-
const r = await convertBytes(p, bytes, contentType);
|
|
2244
|
-
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
2245
|
-
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
2246
1721
|
}
|
|
2247
|
-
var
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
return (
|
|
1722
|
+
var parser2 = new XMLParser2({
|
|
1723
|
+
ignoreAttributes: false,
|
|
1724
|
+
attributeNamePrefix: "@_",
|
|
1725
|
+
parseTagValue: false,
|
|
1726
|
+
processEntities: true,
|
|
1727
|
+
stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
|
|
1728
|
+
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
1729
|
+
});
|
|
1730
|
+
function text(n) {
|
|
1731
|
+
if (n == null) return "";
|
|
1732
|
+
if (typeof n === "object") return text(n["#text"]);
|
|
1733
|
+
return String(n);
|
|
2259
1734
|
}
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
|
|
2264
|
-
if (reason === "no methods/results") return `no article body (${seen})`;
|
|
2265
|
-
if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
|
|
2266
|
-
return reason;
|
|
1735
|
+
var plain = (n) => decodeEntities(text(n)).trim();
|
|
1736
|
+
function inlineText2(raw) {
|
|
1737
|
+
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
2267
1738
|
}
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
2278
|
-
}
|
|
2279
|
-
if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
2280
|
-
} else {
|
|
2281
|
-
const text3 = bytes.toString("utf8");
|
|
2282
|
-
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
2283
|
-
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
2284
|
-
format = "xml";
|
|
2285
|
-
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
2286
|
-
if (!doc.title) doc.title = p.title;
|
|
2287
|
-
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
2288
|
-
md = renderFulltextMd(doc, p.headLine);
|
|
2289
|
-
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
2290
|
-
format = "html";
|
|
2291
|
-
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
2292
|
-
md = `# ${p.title}
|
|
2293
|
-
|
|
2294
|
-
${p.headLine}
|
|
2295
|
-
|
|
2296
|
-
${htmlToMarkdown(body)}`;
|
|
2297
|
-
} else {
|
|
2298
|
-
return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
|
|
2299
|
-
}
|
|
1739
|
+
function decodeEntities(s) {
|
|
1740
|
+
return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
1741
|
+
}
|
|
1742
|
+
function parsePubmedXml(xml) {
|
|
1743
|
+
const doc = parser2.parse(xml);
|
|
1744
|
+
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
1745
|
+
if (!article) {
|
|
1746
|
+
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
1747
|
+
return book ? parseBook(book) : null;
|
|
2300
1748
|
}
|
|
2301
|
-
const
|
|
2302
|
-
const
|
|
2303
|
-
const
|
|
2304
|
-
|
|
2305
|
-
|
|
1749
|
+
const cit = article.MedlineCitation ?? {};
|
|
1750
|
+
const art = cit.Article ?? {};
|
|
1751
|
+
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
1752
|
+
const idOf = (t) => {
|
|
1753
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1754
|
+
return hit ? text(hit).trim() || null : null;
|
|
1755
|
+
};
|
|
1756
|
+
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
1757
|
+
const authors = authorList.map((a) => {
|
|
1758
|
+
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
1759
|
+
const last = plain(a.LastName);
|
|
1760
|
+
const ini = plain(a.Initials);
|
|
1761
|
+
return [last, ini].filter(Boolean).join(" ");
|
|
1762
|
+
}).filter(Boolean);
|
|
1763
|
+
const first = authorList[0];
|
|
1764
|
+
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
1765
|
+
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
1766
|
+
let year = text(pub.Year).trim() || null;
|
|
1767
|
+
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
1768
|
+
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
1769
|
+
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
1770
|
+
text: inlineText2(text(a))
|
|
1771
|
+
})).filter((a) => a.text);
|
|
1772
|
+
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
1773
|
+
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
1774
|
+
if (!type) return [];
|
|
1775
|
+
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
|
|
1776
|
+
});
|
|
1777
|
+
const funding = [];
|
|
1778
|
+
for (const g of art.GrantList?.Grant ?? []) {
|
|
1779
|
+
const a = plain(g.Agency);
|
|
1780
|
+
if (a && !funding.includes(a)) funding.push(a);
|
|
1781
|
+
}
|
|
1782
|
+
return {
|
|
1783
|
+
coi: inlineText2(text(cit.CoiStatement)) || null,
|
|
1784
|
+
funding,
|
|
1785
|
+
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
1786
|
+
pmcid: idOf("pmc"),
|
|
1787
|
+
doi: idOf("doi"),
|
|
1788
|
+
title: inlineText2(text(art.ArticleTitle)),
|
|
1789
|
+
authors,
|
|
1790
|
+
firstAuthor,
|
|
1791
|
+
journal: plain(art.Journal?.Title) || null,
|
|
1792
|
+
year,
|
|
1793
|
+
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
1794
|
+
abstract,
|
|
1795
|
+
corrections
|
|
1796
|
+
};
|
|
2306
1797
|
}
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
[
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
[
|
|
2316
|
-
[
|
|
2317
|
-
[
|
|
2318
|
-
[
|
|
2319
|
-
[
|
|
2320
|
-
[
|
|
2321
|
-
[
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
]
|
|
2336
|
-
|
|
2337
|
-
let out = xml;
|
|
2338
|
-
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
2339
|
-
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
1798
|
+
function parseBook(b) {
|
|
1799
|
+
const d = b.BookDocument ?? {};
|
|
1800
|
+
const book = d.Book ?? {};
|
|
1801
|
+
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
1802
|
+
const idOf = (t) => {
|
|
1803
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1804
|
+
return hit ? text(hit).trim() || null : null;
|
|
1805
|
+
};
|
|
1806
|
+
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
1807
|
+
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
1808
|
+
const people = pick?.Author ?? [];
|
|
1809
|
+
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
1810
|
+
const first = people[0];
|
|
1811
|
+
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
|
|
1812
|
+
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
1813
|
+
return {
|
|
1814
|
+
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
1815
|
+
pmcid: null,
|
|
1816
|
+
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
1817
|
+
title: inlineText2(text(d.ArticleTitle)) || inlineText2(text(book.BookTitle)),
|
|
1818
|
+
authors,
|
|
1819
|
+
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
1820
|
+
journal: plain(book.Publisher?.PublisherName) || null,
|
|
1821
|
+
year: text(book.PubDate?.Year).trim() || null,
|
|
1822
|
+
publicationTypes: [...new Set(types)],
|
|
1823
|
+
abstract,
|
|
1824
|
+
corrections: [],
|
|
1825
|
+
coi: null,
|
|
1826
|
+
funding: []
|
|
1827
|
+
};
|
|
2340
1828
|
}
|
|
2341
1829
|
|
|
2342
|
-
// src/
|
|
2343
|
-
|
|
2344
|
-
var
|
|
2345
|
-
var
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
2351
|
-
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
2352
|
-
calls.push(Date.now());
|
|
1830
|
+
// src/spl.ts
|
|
1831
|
+
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
1832
|
+
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
1833
|
+
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
1834
|
+
async function fetchSpl(setid) {
|
|
1835
|
+
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
1836
|
+
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
1837
|
+
return xml;
|
|
2353
1838
|
}
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
1839
|
+
var parser3 = new XMLParser3({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
|
|
1840
|
+
var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
1841
|
+
var kids2 = (n) => n[tagOf2(n)] ?? [];
|
|
1842
|
+
var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
1843
|
+
var squash2 = (s) => s.replace(/\s+/g, " ").trim();
|
|
1844
|
+
var child = (n, tag) => kids2(n).find((x) => tagOf2(x) === tag);
|
|
1845
|
+
function findDeep2(nodes, tag) {
|
|
1846
|
+
for (const n of nodes) {
|
|
1847
|
+
if ("#text" in n) continue;
|
|
1848
|
+
if (tagOf2(n) === tag) return n;
|
|
1849
|
+
const h = findDeep2(kids2(n), tag);
|
|
1850
|
+
if (h) return h;
|
|
1851
|
+
}
|
|
1852
|
+
return void 0;
|
|
1853
|
+
}
|
|
1854
|
+
function inline(nodes) {
|
|
1855
|
+
let out = "";
|
|
1856
|
+
for (const n of nodes) {
|
|
1857
|
+
if ("#text" in n) {
|
|
1858
|
+
out += String(n["#text"]);
|
|
2365
1859
|
continue;
|
|
2366
1860
|
}
|
|
2367
|
-
|
|
1861
|
+
const t = tagOf2(n);
|
|
1862
|
+
const inner = () => inline(kids2(n));
|
|
1863
|
+
switch (t) {
|
|
1864
|
+
case "content": {
|
|
1865
|
+
const style = attr2(n, "styleCode").toLowerCase();
|
|
1866
|
+
const s = inner();
|
|
1867
|
+
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
1868
|
+
break;
|
|
1869
|
+
}
|
|
1870
|
+
case "sup":
|
|
1871
|
+
out += `^${inner()}^`;
|
|
1872
|
+
break;
|
|
1873
|
+
case "sub":
|
|
1874
|
+
out += `~${inner()}~`;
|
|
1875
|
+
break;
|
|
1876
|
+
case "br":
|
|
1877
|
+
out += " ";
|
|
1878
|
+
break;
|
|
1879
|
+
case "footnote":
|
|
1880
|
+
out += ` (${squash2(inner())})`;
|
|
1881
|
+
break;
|
|
1882
|
+
case "footnoteRef":
|
|
1883
|
+
case "renderMultiMedia":
|
|
1884
|
+
break;
|
|
1885
|
+
default:
|
|
1886
|
+
out += inner();
|
|
1887
|
+
}
|
|
2368
1888
|
}
|
|
2369
|
-
|
|
1889
|
+
return out;
|
|
2370
1890
|
}
|
|
2371
|
-
function
|
|
2372
|
-
const
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
if (!
|
|
2378
|
-
let data;
|
|
2379
|
-
try {
|
|
2380
|
-
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
2381
|
-
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
2382
|
-
} catch (e) {
|
|
2383
|
-
return miss(this.name, errMsg(e));
|
|
2384
|
-
}
|
|
2385
|
-
const candidates = coreCandidates(data);
|
|
2386
|
-
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
2387
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2388
|
-
return download(this.name, p, candidates, ctx);
|
|
1891
|
+
function table(n) {
|
|
1892
|
+
const row2 = (x) => kids2(x).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => squash2(inline(kids2(c))).replace(/\|/g, "\\|"));
|
|
1893
|
+
const collect = (nodes, into) => {
|
|
1894
|
+
for (const x of nodes) {
|
|
1895
|
+
if ("#text" in x) continue;
|
|
1896
|
+
if (tagOf2(x) === "tr") into.push(row2(x));
|
|
1897
|
+
else if (!["thead", "tfoot"].includes(tagOf2(x))) collect(kids2(x), into);
|
|
2389
1898
|
}
|
|
2390
1899
|
};
|
|
1900
|
+
const head = [], body = [], foot = [];
|
|
1901
|
+
for (const x of kids2(n)) {
|
|
1902
|
+
const t = tagOf2(x);
|
|
1903
|
+
if (t === "thead") collect(kids2(x), head);
|
|
1904
|
+
else if (t === "tfoot") collect(kids2(x), foot);
|
|
1905
|
+
else if (t === "tr") body.push(row2(x));
|
|
1906
|
+
else if (!("#text" in x) && t !== "caption") collect(kids2(x), body);
|
|
1907
|
+
}
|
|
1908
|
+
const rows = [...head, ...body];
|
|
1909
|
+
if (!rows.length) return "";
|
|
1910
|
+
const width = Math.max(...rows.map((r) => r.length));
|
|
1911
|
+
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
1912
|
+
const cap = child(n, "caption");
|
|
1913
|
+
const title = cap ? squash2(inline(kids2(cap))) : "";
|
|
1914
|
+
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
1915
|
+
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
2391
1916
|
}
|
|
2392
|
-
function
|
|
1917
|
+
function blocks(nodes, depth) {
|
|
2393
1918
|
const out = [];
|
|
2394
|
-
const
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
1919
|
+
for (const n of nodes) {
|
|
1920
|
+
if ("#text" in n) {
|
|
1921
|
+
const t2 = squash2(String(n["#text"]));
|
|
1922
|
+
if (t2) out.push(t2);
|
|
1923
|
+
continue;
|
|
1924
|
+
}
|
|
1925
|
+
const t = tagOf2(n);
|
|
1926
|
+
const c = kids2(n);
|
|
1927
|
+
switch (t) {
|
|
1928
|
+
case "paragraph": {
|
|
1929
|
+
const s = squash2(inline(c.filter((x) => !["table", "list"].includes(tagOf2(x)))));
|
|
1930
|
+
if (s) out.push(s);
|
|
1931
|
+
out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf2(x))), depth));
|
|
1932
|
+
break;
|
|
1933
|
+
}
|
|
1934
|
+
case "list": {
|
|
1935
|
+
const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr2(n, "listType"));
|
|
1936
|
+
let i = 0;
|
|
1937
|
+
for (const it of c.filter((x) => tagOf2(x) === "item")) {
|
|
1938
|
+
i++;
|
|
1939
|
+
const body = blocks(kids2(it).filter((x) => tagOf2(x) !== "caption"), depth).join(" ");
|
|
1940
|
+
if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
1941
|
+
}
|
|
1942
|
+
break;
|
|
1943
|
+
}
|
|
1944
|
+
case "table": {
|
|
1945
|
+
const md = table(n);
|
|
1946
|
+
if (md) out.push(md);
|
|
1947
|
+
break;
|
|
1948
|
+
}
|
|
1949
|
+
case "renderMultiMedia":
|
|
1950
|
+
case "observationMedia":
|
|
1951
|
+
case "subject":
|
|
1952
|
+
break;
|
|
1953
|
+
case "section":
|
|
1954
|
+
out.push(...section(n, depth));
|
|
1955
|
+
break;
|
|
1956
|
+
case "component":
|
|
1957
|
+
case "text":
|
|
1958
|
+
case "excerpt":
|
|
1959
|
+
case "highlight":
|
|
1960
|
+
default:
|
|
1961
|
+
out.push(...blocks(c, depth));
|
|
1962
|
+
break;
|
|
1963
|
+
}
|
|
2404
1964
|
}
|
|
2405
1965
|
return out;
|
|
2406
1966
|
}
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
var
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
1967
|
+
var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
|
|
1968
|
+
var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
|
|
1969
|
+
var plain2 = (nodes) => {
|
|
1970
|
+
let out = "";
|
|
1971
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
|
|
1972
|
+
return out;
|
|
1973
|
+
};
|
|
1974
|
+
function section(n, depth) {
|
|
1975
|
+
const code = child(n, "code");
|
|
1976
|
+
const c = code ? attr2(code, "code") : "";
|
|
1977
|
+
if (SKIP_CODES.has(c)) return [];
|
|
1978
|
+
const title = child(n, "title");
|
|
1979
|
+
let head = title ? squash2(plain2(kids2(title))) : "";
|
|
1980
|
+
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
1981
|
+
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
1982
|
+
const out = [];
|
|
1983
|
+
if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
|
|
1984
|
+
const rest = kids2(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf2(x)));
|
|
1985
|
+
out.push(...blocks(rest, head ? depth + 1 : depth));
|
|
1986
|
+
return out;
|
|
1987
|
+
}
|
|
1988
|
+
var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
|
|
1989
|
+
function parseSpl(xml) {
|
|
1990
|
+
const doc = parser3.parse(xml);
|
|
1991
|
+
const d = doc.find((x) => tagOf2(x) === "document");
|
|
1992
|
+
if (!d) return null;
|
|
1993
|
+
const k = kids2(d);
|
|
1994
|
+
const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
1995
|
+
const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
|
|
1996
|
+
const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
|
|
1997
|
+
const named = (nodes) => {
|
|
1998
|
+
for (const n of nodes) {
|
|
1999
|
+
if ("#text" in n) continue;
|
|
2000
|
+
if (tagOf2(n) === "manufacturedProduct") {
|
|
2001
|
+
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
2002
|
+
if (child(inner2, "name")) return inner2;
|
|
2003
|
+
}
|
|
2004
|
+
const h = named(kids2(n));
|
|
2005
|
+
if (h) return h;
|
|
2419
2006
|
}
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2007
|
+
return void 0;
|
|
2008
|
+
};
|
|
2009
|
+
const inner = named(k);
|
|
2010
|
+
const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
2011
|
+
const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
|
|
2012
|
+
const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
2013
|
+
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
2014
|
+
const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
|
|
2015
|
+
const org = findDeep2(k, "representedOrganization");
|
|
2016
|
+
const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
|
|
2017
|
+
const body = findDeep2(k, "structuredBody");
|
|
2018
|
+
const sections2 = body ? blocks(kids2(body), 2) : [];
|
|
2019
|
+
const docTitle = child(d, "title");
|
|
2020
|
+
const lines = [];
|
|
2021
|
+
let cur = "";
|
|
2022
|
+
for (const x of docTitle ? kids2(docTitle) : []) {
|
|
2023
|
+
if (!("#text" in x) && tagOf2(x) === "br") {
|
|
2024
|
+
lines.push(cur);
|
|
2025
|
+
cur = "";
|
|
2026
|
+
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
2426
2027
|
}
|
|
2028
|
+
lines.push(cur);
|
|
2029
|
+
const highlights = lines.map(squash2).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
2030
|
+
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
2031
|
+
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
2032
|
+
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
2033
|
+
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
2034
|
+
}
|
|
2035
|
+
function headLineSpl(s) {
|
|
2036
|
+
return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
|
|
2037
|
+
}
|
|
2038
|
+
function renderLabelMd(s) {
|
|
2039
|
+
const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
|
|
2040
|
+
if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
|
|
2041
|
+
parts.push(...s.sections.flatMap((b) => [b, ""]));
|
|
2042
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
2043
|
+
}
|
|
2044
|
+
|
|
2045
|
+
// src/ctgov.ts
|
|
2046
|
+
var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
|
|
2047
|
+
var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
|
|
2048
|
+
async function fetchStudy(nctid) {
|
|
2049
|
+
const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
|
|
2050
|
+
if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
|
|
2051
|
+
return json2;
|
|
2052
|
+
}
|
|
2053
|
+
var STATUS = {
|
|
2054
|
+
NOT_YET_RECRUITING: "Not yet recruiting",
|
|
2055
|
+
RECRUITING: "Recruiting",
|
|
2056
|
+
ENROLLING_BY_INVITATION: "Enrolling by invitation",
|
|
2057
|
+
ACTIVE_NOT_RECRUITING: "Active, not recruiting",
|
|
2058
|
+
SUSPENDED: "Suspended",
|
|
2059
|
+
TERMINATED: "Terminated",
|
|
2060
|
+
COMPLETED: "Completed",
|
|
2061
|
+
WITHDRAWN: "Withdrawn",
|
|
2062
|
+
UNKNOWN: "Unknown status",
|
|
2063
|
+
AVAILABLE: "Available",
|
|
2064
|
+
NO_LONGER_AVAILABLE: "No longer available",
|
|
2065
|
+
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
2066
|
+
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
2067
|
+
WITHHELD: "Withheld"
|
|
2427
2068
|
};
|
|
2428
|
-
|
|
2069
|
+
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
2070
|
+
var words = (v) => {
|
|
2071
|
+
const s = String(v ?? "");
|
|
2072
|
+
if (!s) return "";
|
|
2073
|
+
if (STATUS[s]) return STATUS[s];
|
|
2074
|
+
if (PHASE[s]) return PHASE[s];
|
|
2075
|
+
const w = s.toLowerCase().replace(/_/g, " ");
|
|
2076
|
+
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
2077
|
+
};
|
|
2078
|
+
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
2079
|
+
var list = (a) => Array.isArray(a) ? a : [];
|
|
2080
|
+
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
2081
|
+
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
2082
|
+
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
2083
|
+
function parseStudy(json2) {
|
|
2084
|
+
let d;
|
|
2085
|
+
try {
|
|
2086
|
+
d = JSON.parse(json2);
|
|
2087
|
+
} catch {
|
|
2088
|
+
return null;
|
|
2089
|
+
}
|
|
2090
|
+
const ps = d?.protocolSection;
|
|
2091
|
+
if (!ps) return null;
|
|
2092
|
+
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
2093
|
+
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
2094
|
+
const nctid = String(idm.nctId ?? "").toUpperCase();
|
|
2429
2095
|
const out = [];
|
|
2430
|
-
const
|
|
2431
|
-
|
|
2432
|
-
if (
|
|
2433
|
-
seen.add(url);
|
|
2434
|
-
out.push({ url, version, source: "europepmc", host, pdf });
|
|
2096
|
+
const sec = (title, lines) => {
|
|
2097
|
+
const body = lines.filter((l) => !!l && l.trim() !== "");
|
|
2098
|
+
if (body.length) out.push(`## ${title}`, ...body);
|
|
2435
2099
|
};
|
|
2436
|
-
const
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2100
|
+
const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
|
|
2101
|
+
sec("Identification", [
|
|
2102
|
+
idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
|
|
2103
|
+
idm.acronym ? `Acronym: ${idm.acronym}` : null,
|
|
2104
|
+
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
2105
|
+
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
2106
|
+
]);
|
|
2107
|
+
sec("Status", [
|
|
2108
|
+
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
2109
|
+
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
2110
|
+
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
2111
|
+
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
2112
|
+
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
2113
|
+
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
2114
|
+
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
2115
|
+
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
2116
|
+
]);
|
|
2117
|
+
const rp = sp.responsibleParty;
|
|
2118
|
+
sec("Sponsor and collaborators", [
|
|
2119
|
+
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
2120
|
+
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
2121
|
+
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
2122
|
+
]);
|
|
2123
|
+
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
2124
|
+
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
2125
|
+
const di = dm.designInfo ?? {};
|
|
2126
|
+
const mi = di.maskingInfo ?? {};
|
|
2127
|
+
sec("Design", [
|
|
2128
|
+
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
2129
|
+
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
2130
|
+
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
2131
|
+
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
2132
|
+
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
2133
|
+
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
2134
|
+
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
2135
|
+
]);
|
|
2136
|
+
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
2137
|
+
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
2138
|
+
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
2139
|
+
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
2140
|
+
sec("Outcomes", [
|
|
2141
|
+
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
2142
|
+
...list(om.primaryOutcomes).map(outcome),
|
|
2143
|
+
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
2144
|
+
...list(om.secondaryOutcomes).map(outcome),
|
|
2145
|
+
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
2146
|
+
...list(om.otherOutcomes).map(outcome)
|
|
2147
|
+
]);
|
|
2148
|
+
sec("Eligibility", [
|
|
2149
|
+
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
2150
|
+
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
2151
|
+
]);
|
|
2152
|
+
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
2153
|
+
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
2154
|
+
const refs = list(rf.references);
|
|
2155
|
+
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
2156
|
+
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
2157
|
+
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
2158
|
+
return {
|
|
2159
|
+
nctid,
|
|
2160
|
+
title: para(idm.briefTitle) || nctid,
|
|
2161
|
+
acronym: idm.acronym ?? null,
|
|
2162
|
+
sponsor: sp.leadSponsor?.name ?? null,
|
|
2163
|
+
firstPosted: st.studyFirstPostDateStruct?.date ?? null,
|
|
2164
|
+
lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
|
|
2165
|
+
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
2166
|
+
hasResults: !!d.hasResults,
|
|
2167
|
+
publications,
|
|
2168
|
+
sections: out
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
function results(rs) {
|
|
2172
|
+
const out = ["## Results"];
|
|
2173
|
+
const pf = rs.participantFlowModule;
|
|
2174
|
+
if (pf) {
|
|
2175
|
+
out.push("### Participant flow");
|
|
2176
|
+
const groups = list(pf.groups);
|
|
2177
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
2178
|
+
if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
|
|
2179
|
+
for (const p of list(pf.periods)) {
|
|
2180
|
+
const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
|
|
2181
|
+
const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
|
|
2182
|
+
for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
|
|
2183
|
+
out.push(table2(head, rows));
|
|
2184
|
+
}
|
|
2441
2185
|
}
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2186
|
+
const bc = rs.baselineCharacteristicsModule;
|
|
2187
|
+
if (bc) {
|
|
2188
|
+
out.push("### Baseline characteristics");
|
|
2189
|
+
const groups = list(bc.groups);
|
|
2190
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
2191
|
+
for (const m of list(bc.measures)) {
|
|
2192
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
2193
|
+
const rows = [];
|
|
2194
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
2195
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
2196
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
2197
|
+
})]);
|
|
2198
|
+
out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
const om = rs.outcomeMeasuresModule;
|
|
2202
|
+
if (om) {
|
|
2203
|
+
out.push("### Outcome measures");
|
|
2204
|
+
for (const m of list(om.outcomeMeasures)) {
|
|
2205
|
+
const groups = list(m.groups);
|
|
2206
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
2207
|
+
out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
|
|
2208
|
+
if (m.description) out.push(para(m.description));
|
|
2209
|
+
out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
|
|
2210
|
+
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
2211
|
+
const rows = [];
|
|
2212
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
2213
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
2214
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
2215
|
+
})]);
|
|
2216
|
+
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
2217
|
+
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
2218
|
+
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
2219
|
+
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
const ae = rs.adverseEventsModule;
|
|
2223
|
+
if (ae) {
|
|
2224
|
+
out.push("### Adverse events");
|
|
2225
|
+
const groups = list(ae.eventGroups);
|
|
2226
|
+
out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
|
|
2227
|
+
const summary = [
|
|
2228
|
+
["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
|
|
2229
|
+
["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
|
|
2230
|
+
["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
|
|
2231
|
+
].filter((r) => r.slice(1).some(Boolean));
|
|
2232
|
+
out.push(table2(["", ...groups.map((g) => g.title)], summary));
|
|
2233
|
+
const events = (title, evs) => {
|
|
2234
|
+
if (!evs.length) return;
|
|
2235
|
+
out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
|
|
2236
|
+
const s = list(e.stats).find((x) => x.groupId === g.id);
|
|
2237
|
+
return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
|
|
2238
|
+
})])));
|
|
2239
|
+
};
|
|
2240
|
+
events("Serious adverse events", list(ae.seriousEvents));
|
|
2241
|
+
events("Other adverse events", list(ae.otherEvents));
|
|
2447
2242
|
}
|
|
2448
2243
|
return out;
|
|
2449
2244
|
}
|
|
2245
|
+
function headLineStudy(s) {
|
|
2246
|
+
return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
|
|
2247
|
+
}
|
|
2248
|
+
function renderStudyMd(s) {
|
|
2249
|
+
const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
|
|
2250
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
2251
|
+
}
|
|
2450
2252
|
|
|
2451
|
-
// src/
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2253
|
+
// src/entry.ts
|
|
2254
|
+
function emptyStatus(requested, type, id, libraryDir) {
|
|
2255
|
+
return {
|
|
2256
|
+
requested,
|
|
2257
|
+
type,
|
|
2258
|
+
id,
|
|
2259
|
+
entry: null,
|
|
2260
|
+
entry_dir: null,
|
|
2261
|
+
library_dir: libraryDir,
|
|
2262
|
+
pmid: null,
|
|
2263
|
+
setid: null,
|
|
2264
|
+
nctid: null,
|
|
2265
|
+
publications: [],
|
|
2266
|
+
pmcid: null,
|
|
2267
|
+
doi: null,
|
|
2268
|
+
title: null,
|
|
2269
|
+
first_author: null,
|
|
2270
|
+
year: null,
|
|
2271
|
+
publication_types: [],
|
|
2272
|
+
retracted: false,
|
|
2273
|
+
corrections: [],
|
|
2274
|
+
coi: null,
|
|
2275
|
+
funding: [],
|
|
2276
|
+
citable: true,
|
|
2277
|
+
comment: null,
|
|
2278
|
+
has_abstract: null,
|
|
2279
|
+
level: null,
|
|
2280
|
+
version: null,
|
|
2281
|
+
retrieved_by: null,
|
|
2282
|
+
retrieved_date: null,
|
|
2283
|
+
pubmed_checked: null,
|
|
2284
|
+
attempts: [],
|
|
2285
|
+
files: [],
|
|
2286
|
+
glyphs_unresolved: null
|
|
2287
|
+
};
|
|
2288
|
+
}
|
|
2289
|
+
var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
2290
|
+
function headLine(r) {
|
|
2291
|
+
return [
|
|
2292
|
+
r.authors.join(", ") || null,
|
|
2293
|
+
r.journal,
|
|
2294
|
+
r.year,
|
|
2295
|
+
r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
|
|
2296
|
+
r.doi ? `DOI ${r.doi}` : null,
|
|
2297
|
+
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
2298
|
+
].filter(Boolean).join(" \xB7 ");
|
|
2299
|
+
}
|
|
2300
|
+
function renderAbstractMd(r) {
|
|
2301
|
+
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
2302
|
+
return `# ${r.title}
|
|
2303
|
+
|
|
2304
|
+
${headLine(r)}
|
|
2305
|
+
|
|
2306
|
+
## Abstract
|
|
2307
|
+
|
|
2308
|
+
${body}
|
|
2309
|
+
`;
|
|
2310
|
+
}
|
|
2311
|
+
function readHeadLine(p) {
|
|
2312
|
+
try {
|
|
2313
|
+
const lines = readFileSync4(p.md, "utf8").split("\n").filter((l) => l.trim());
|
|
2314
|
+
return lines[1] ?? "";
|
|
2315
|
+
} catch {
|
|
2316
|
+
return "";
|
|
2462
2317
|
}
|
|
2463
|
-
return null;
|
|
2464
2318
|
}
|
|
2465
|
-
function
|
|
2319
|
+
function metaFromRecord(r, files, converter) {
|
|
2466
2320
|
return {
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2321
|
+
pmid: r.pmid,
|
|
2322
|
+
setid: null,
|
|
2323
|
+
nctid: null,
|
|
2324
|
+
publications: [],
|
|
2325
|
+
pmcid: r.pmcid,
|
|
2326
|
+
doi: r.doi,
|
|
2327
|
+
title: r.title,
|
|
2328
|
+
first_author: r.firstAuthor,
|
|
2329
|
+
year: r.year,
|
|
2330
|
+
publication_types: r.publicationTypes,
|
|
2331
|
+
retracted: isRetracted(r),
|
|
2332
|
+
corrections: r.corrections,
|
|
2333
|
+
coi: r.coi,
|
|
2334
|
+
funding: r.funding,
|
|
2335
|
+
has_abstract: r.abstract.length > 0,
|
|
2336
|
+
level: "abstract",
|
|
2337
|
+
version: null,
|
|
2338
|
+
files,
|
|
2339
|
+
retrieved_by: "pubmed",
|
|
2340
|
+
retrieved_date: today(),
|
|
2341
|
+
converted_by: converter,
|
|
2342
|
+
pubmed_checked: today(),
|
|
2343
|
+
attempts: []
|
|
2482
2344
|
};
|
|
2483
2345
|
}
|
|
2484
|
-
var
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
try {
|
|
2495
|
-
data = JSON.parse(await fetchText(`https://api.openalex.org/works/${encodeURIComponent(key)}?mailto=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
2496
|
-
} catch (e) {
|
|
2497
|
-
return miss(this.name, errMsg(e));
|
|
2498
|
-
}
|
|
2499
|
-
const candidates = openalexCandidates(data);
|
|
2500
|
-
if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
|
|
2501
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2502
|
-
return download(this.name, p, candidates, ctx);
|
|
2346
|
+
var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
|
|
2347
|
+
var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
|
|
2348
|
+
function parseCorrection(v) {
|
|
2349
|
+
const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
|
|
2350
|
+
if (!m) return null;
|
|
2351
|
+
let source;
|
|
2352
|
+
try {
|
|
2353
|
+
source = JSON.parse(m[3]);
|
|
2354
|
+
} catch {
|
|
2355
|
+
return null;
|
|
2503
2356
|
}
|
|
2504
|
-
};
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
const
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
|
|
2511
|
-
|
|
2512
|
-
|
|
2513
|
-
|
|
2514
|
-
|
|
2357
|
+
return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
|
|
2358
|
+
}
|
|
2359
|
+
function renderMeta(m) {
|
|
2360
|
+
const lines = [
|
|
2361
|
+
["pmid", m.pmid],
|
|
2362
|
+
...m.setid ? [["setid", m.setid]] : [],
|
|
2363
|
+
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
2364
|
+
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
2365
|
+
["pmcid", m.pmcid ?? ""],
|
|
2366
|
+
["doi", m.doi ?? ""],
|
|
2367
|
+
["title", JSON.stringify(m.title)],
|
|
2368
|
+
["first_author", m.first_author ?? ""],
|
|
2369
|
+
["year", m.year ?? ""],
|
|
2370
|
+
["publication_types", m.publication_types.join("; ")],
|
|
2371
|
+
["retracted", m.retracted ? "yes" : "no"],
|
|
2372
|
+
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
2373
|
+
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
2374
|
+
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
2375
|
+
// a JSON array: a statement may hold semicolons
|
|
2376
|
+
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
2377
|
+
["level", m.level],
|
|
2378
|
+
["version", m.version ?? ""],
|
|
2379
|
+
["files", m.files.join(", ")],
|
|
2380
|
+
["retrieved_by", m.retrieved_by],
|
|
2381
|
+
["retrieved_date", m.retrieved_date],
|
|
2382
|
+
["converted_by", m.converted_by],
|
|
2383
|
+
["pubmed_checked", m.pubmed_checked],
|
|
2384
|
+
["attempts", m.attempts.join(", ")]
|
|
2385
|
+
];
|
|
2386
|
+
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
2387
|
+
}
|
|
2388
|
+
function parseFrontmatter(md) {
|
|
2389
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(md);
|
|
2390
|
+
const fm = {}, all = {};
|
|
2391
|
+
for (const line of (m?.[1] ?? "").split("\n")) {
|
|
2392
|
+
const i = line.indexOf(":");
|
|
2393
|
+
if (i <= 0) continue;
|
|
2394
|
+
const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
|
|
2395
|
+
fm[k] = v;
|
|
2396
|
+
(all[k] ??= []).push(v);
|
|
2515
2397
|
}
|
|
2516
|
-
return
|
|
2398
|
+
return { fm, all };
|
|
2517
2399
|
}
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
const
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2400
|
+
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
2401
|
+
function splitAttempts(v) {
|
|
2402
|
+
if (!v) return [];
|
|
2403
|
+
const out = [];
|
|
2404
|
+
let cur = "", depth = 0;
|
|
2405
|
+
for (const ch of v) {
|
|
2406
|
+
if (ch === "(") depth++;
|
|
2407
|
+
else if (ch === ")") depth = Math.max(0, depth - 1);
|
|
2408
|
+
if (ch === "," && depth === 0) {
|
|
2409
|
+
out.push(cur);
|
|
2410
|
+
cur = "";
|
|
2411
|
+
} else cur += ch;
|
|
2412
|
+
}
|
|
2413
|
+
out.push(cur);
|
|
2414
|
+
const merged = [];
|
|
2415
|
+
for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
|
|
2416
|
+
if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
|
|
2417
|
+
else merged.push(piece);
|
|
2418
|
+
}
|
|
2419
|
+
return merged;
|
|
2529
2420
|
}
|
|
2530
|
-
var
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
}
|
|
2540
|
-
const doc = parseJats(xml);
|
|
2541
|
-
if (!doc.hasBody) return miss(this.name, "no body");
|
|
2542
|
-
const md = renderFulltextMd(doc, p.headLine);
|
|
2543
|
-
const check = checkFulltext(md, p.title);
|
|
2544
|
-
if (!check.ok) return miss(this.name, check.reason);
|
|
2545
|
-
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
2421
|
+
var opt = (v) => v ? v : null;
|
|
2422
|
+
var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
|
|
2423
|
+
function splitPublicationTypes(v) {
|
|
2424
|
+
if (!v) return [];
|
|
2425
|
+
if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
|
|
2426
|
+
const out = [];
|
|
2427
|
+
for (const piece of list2(v)) {
|
|
2428
|
+
if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
|
|
2429
|
+
else out.push(piece);
|
|
2546
2430
|
}
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
var CROSSREF = "https://api.crossref.org/works/";
|
|
2551
|
-
var TOKENED = [
|
|
2552
|
-
{ host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
|
|
2553
|
-
{ host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
|
|
2554
|
-
];
|
|
2555
|
-
var hostOf2 = (url) => {
|
|
2431
|
+
return out;
|
|
2432
|
+
}
|
|
2433
|
+
function abstractMissing(p) {
|
|
2556
2434
|
try {
|
|
2557
|
-
return
|
|
2435
|
+
return readFileSync4(p.md, "utf8").includes("_No abstract available._");
|
|
2558
2436
|
} catch {
|
|
2559
|
-
return
|
|
2437
|
+
return false;
|
|
2560
2438
|
}
|
|
2561
|
-
}
|
|
2562
|
-
function
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2439
|
+
}
|
|
2440
|
+
function correctionsFromXml(p) {
|
|
2441
|
+
try {
|
|
2442
|
+
return parsePubmedXml(readFileSync4(p.xml, "utf8"))?.corrections ?? [];
|
|
2443
|
+
} catch {
|
|
2444
|
+
return [];
|
|
2445
|
+
}
|
|
2446
|
+
}
|
|
2447
|
+
function declarationsFromXml(p) {
|
|
2448
|
+
try {
|
|
2449
|
+
const r = parsePubmedXml(readFileSync4(p.xml, "utf8"));
|
|
2450
|
+
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
2451
|
+
} catch {
|
|
2452
|
+
return { coi: null, funding: [] };
|
|
2453
|
+
}
|
|
2454
|
+
}
|
|
2455
|
+
function readMeta(p) {
|
|
2456
|
+
const { fm, all } = parseFrontmatter(readFileSync4(p.meta, "utf8"));
|
|
2457
|
+
let title = fm.title ?? "";
|
|
2458
|
+
if (title.startsWith('"')) {
|
|
2459
|
+
try {
|
|
2460
|
+
title = JSON.parse(title);
|
|
2461
|
+
} catch {
|
|
2573
2462
|
}
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2463
|
+
}
|
|
2464
|
+
const legacy = fm.pubmed_checked === void 0;
|
|
2465
|
+
const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
|
|
2466
|
+
let coi = null, funding = [];
|
|
2467
|
+
if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
|
|
2468
|
+
else {
|
|
2469
|
+
if (fm.coi.startsWith('"')) {
|
|
2470
|
+
try {
|
|
2471
|
+
coi = JSON.parse(fm.coi);
|
|
2472
|
+
} catch {
|
|
2473
|
+
coi = fm.coi;
|
|
2582
2474
|
}
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
if (/text\/plain/.test(ct)) continue;
|
|
2586
|
-
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf2(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
2475
|
+
} else coi = fm.coi || null;
|
|
2476
|
+
funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
|
|
2587
2477
|
}
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2478
|
+
return {
|
|
2479
|
+
pmid: fm.pmid ?? "",
|
|
2480
|
+
setid: opt(fm.setid),
|
|
2481
|
+
nctid: opt(fm.nctid),
|
|
2482
|
+
publications: list2(fm.publications),
|
|
2483
|
+
pmcid: opt(fm.pmcid),
|
|
2484
|
+
doi: opt(fm.doi),
|
|
2485
|
+
title,
|
|
2486
|
+
first_author: opt(fm.first_author),
|
|
2487
|
+
year: opt(fm.year),
|
|
2488
|
+
publication_types: splitPublicationTypes(fm.publication_types),
|
|
2489
|
+
retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
|
|
2490
|
+
corrections,
|
|
2491
|
+
coi,
|
|
2492
|
+
funding,
|
|
2493
|
+
has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
|
|
2494
|
+
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
2495
|
+
version: opt(fm.version),
|
|
2496
|
+
files: list2(fm.files),
|
|
2497
|
+
retrieved_by: fm.retrieved_by ?? "",
|
|
2498
|
+
retrieved_date: fm.retrieved_date ?? "",
|
|
2499
|
+
converted_by: fm.converted_by ?? "",
|
|
2500
|
+
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
2501
|
+
attempts: splitAttempts(fm.attempts)
|
|
2502
|
+
};
|
|
2591
2503
|
}
|
|
2592
|
-
function
|
|
2504
|
+
function writeAtomic(file, data) {
|
|
2505
|
+
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2506
|
+
writeFileSync2(tmp, data);
|
|
2507
|
+
renameSync2(tmp, file);
|
|
2508
|
+
}
|
|
2509
|
+
function writeMeta(p, m) {
|
|
2510
|
+
writeAtomic(p.meta, renderMeta(m));
|
|
2511
|
+
}
|
|
2512
|
+
function copyEntry(from, to) {
|
|
2513
|
+
const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
|
|
2514
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
2515
|
+
mkdirSync3(tmp, { recursive: true });
|
|
2516
|
+
const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
|
|
2517
|
+
for (const n of readdirSync2(from.entryDir)) if (join6(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join6(from.entryDir, n), join6(tmp, rename(n)));
|
|
2518
|
+
const m = readMeta(from);
|
|
2519
|
+
const copy = { ...m, files: m.files.map(rename) };
|
|
2520
|
+
writeFileSync2(join6(tmp, basename(to.meta)), renderMeta(copy));
|
|
2521
|
+
mkdirSync3(dirname(to.entryDir), { recursive: true });
|
|
2522
|
+
if (existsSync5(to.entryDir)) renameSync2(to.entryDir, old);
|
|
2523
|
+
renameSync2(tmp, to.entryDir);
|
|
2524
|
+
rmSync(old, { recursive: true, force: true });
|
|
2525
|
+
return copy;
|
|
2526
|
+
}
|
|
2527
|
+
function citability(m) {
|
|
2528
|
+
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
2529
|
+
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
2530
|
+
if (m.retracted) {
|
|
2531
|
+
const r = m.corrections.filter(isRetractionNotice);
|
|
2532
|
+
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
2533
|
+
}
|
|
2534
|
+
const notes = [
|
|
2535
|
+
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
2536
|
+
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
2537
|
+
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
2538
|
+
];
|
|
2539
|
+
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
2540
|
+
}
|
|
2541
|
+
function statusFromMeta(p, m, base) {
|
|
2542
|
+
const full = existsSync5(p.fulltextMd) ? readFileSync4(p.fulltextMd, "utf8") : null;
|
|
2543
|
+
const cit = citability(m);
|
|
2544
|
+
if (full?.includes(`
|
|
2545
|
+
${CORRECTED_HEADING}
|
|
2546
|
+
`) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
|
|
2593
2547
|
return {
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2548
|
+
...base,
|
|
2549
|
+
entry: p.entry,
|
|
2550
|
+
entry_dir: p.entryDir,
|
|
2551
|
+
pmid: m.pmid || null,
|
|
2552
|
+
setid: m.setid,
|
|
2553
|
+
nctid: m.nctid,
|
|
2554
|
+
publications: m.publications,
|
|
2555
|
+
pmcid: m.pmcid,
|
|
2556
|
+
doi: m.doi,
|
|
2557
|
+
title: m.title || null,
|
|
2558
|
+
first_author: m.first_author,
|
|
2559
|
+
year: m.year,
|
|
2560
|
+
publication_types: m.publication_types,
|
|
2561
|
+
retracted: m.retracted,
|
|
2562
|
+
corrections: m.corrections,
|
|
2563
|
+
coi: m.coi,
|
|
2564
|
+
funding: m.funding,
|
|
2565
|
+
...cit,
|
|
2566
|
+
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
2567
|
+
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
2568
|
+
// there is abstract text).
|
|
2569
|
+
has_abstract: m.has_abstract,
|
|
2570
|
+
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
2571
|
+
version: m.version,
|
|
2572
|
+
retrieved_by: m.retrieved_by || null,
|
|
2573
|
+
retrieved_date: m.retrieved_date || null,
|
|
2574
|
+
pubmed_checked: m.pubmed_checked || null,
|
|
2575
|
+
attempts: m.attempts,
|
|
2576
|
+
files: m.files,
|
|
2577
|
+
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
2608
2578
|
};
|
|
2609
2579
|
}
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2580
|
+
function readStatus(p, base) {
|
|
2581
|
+
return statusFromMeta(p, readMeta(p), base);
|
|
2582
|
+
}
|
|
2583
|
+
function entryExists(p) {
|
|
2584
|
+
return existsSync5(p.meta);
|
|
2585
|
+
}
|
|
2586
|
+
function writeEntry(p, r, xml, converter) {
|
|
2587
|
+
mkdirSync3(p.entryDir, { recursive: true });
|
|
2588
|
+
writeAtomic(p.xml, xml);
|
|
2589
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2590
|
+
const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
|
|
2591
|
+
writeMeta(p, meta);
|
|
2592
|
+
return meta;
|
|
2593
|
+
}
|
|
2594
|
+
function refreshRecord(p, m, r, xml) {
|
|
2595
|
+
writeAtomic(p.xml, xml);
|
|
2596
|
+
syncCorrectedBlock(p);
|
|
2597
|
+
const next = {
|
|
2598
|
+
...m,
|
|
2599
|
+
pmcid: r.pmcid ?? m.pmcid,
|
|
2600
|
+
doi: r.doi ?? m.doi,
|
|
2601
|
+
publication_types: r.publicationTypes,
|
|
2602
|
+
retracted: isRetracted(r),
|
|
2603
|
+
corrections: r.corrections,
|
|
2604
|
+
coi: r.coi ?? m.coi,
|
|
2605
|
+
funding: r.funding.length ? r.funding : m.funding,
|
|
2606
|
+
// the paper's own statements stay when PubMed has none
|
|
2607
|
+
pubmed_checked: today()
|
|
2608
|
+
};
|
|
2609
|
+
writeMeta(p, next);
|
|
2610
|
+
return next;
|
|
2611
|
+
}
|
|
2612
|
+
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
2613
|
+
const t = new Date(date).getTime();
|
|
2614
|
+
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
2615
|
+
}
|
|
2616
|
+
var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2617
|
+
var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2618
|
+
function declarationsFromMd(md) {
|
|
2619
|
+
let coi = null;
|
|
2620
|
+
const funding = [];
|
|
2621
|
+
const lines = md.split("\n");
|
|
2622
|
+
let section2 = "";
|
|
2623
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2624
|
+
const l = lines[i].trim();
|
|
2625
|
+
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
2626
|
+
if (h) {
|
|
2627
|
+
section2 = h[1];
|
|
2628
|
+
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
2629
|
+
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
2630
|
+
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
2631
|
+
continue;
|
|
2623
2632
|
}
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
}
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
+
if (section2 !== "Declarations" || !l) continue;
|
|
2634
|
+
const f = FUNDING_RE.exec(l);
|
|
2635
|
+
const c = COI_RE.exec(l);
|
|
2636
|
+
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
2637
|
+
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
2638
|
+
}
|
|
2639
|
+
return { coi, funding };
|
|
2640
|
+
}
|
|
2641
|
+
var CORRECTED_HEADING = "## Corrected in PubMed abstract";
|
|
2642
|
+
function correctedSentences(r) {
|
|
2633
2643
|
const out = [];
|
|
2634
|
-
for (const
|
|
2635
|
-
for (const
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
2644
|
+
for (const a of r.abstract) {
|
|
2645
|
+
for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
|
|
2646
|
+
const i = s.indexOf("[corrected]");
|
|
2647
|
+
if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
|
|
2639
2648
|
}
|
|
2640
2649
|
}
|
|
2641
|
-
return out;
|
|
2650
|
+
return out.filter(Boolean);
|
|
2642
2651
|
}
|
|
2652
|
+
function withCorrectedBlock(md, sentences) {
|
|
2653
|
+
const base = md.split(`
|
|
2654
|
+
${CORRECTED_HEADING}
|
|
2655
|
+
`)[0].trimEnd();
|
|
2656
|
+
if (!sentences.length) return base + "\n";
|
|
2657
|
+
return `${base}
|
|
2643
2658
|
|
|
2644
|
-
|
|
2645
|
-
var BUILTIN = [
|
|
2646
|
-
{ name: "inbox", make: () => inbox },
|
|
2647
|
-
{ name: "pmc", make: () => pmc },
|
|
2648
|
-
{ name: "europepmc", make: () => europepmc },
|
|
2649
|
-
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
2650
|
-
{ name: "unpaywall", make: () => unpaywall },
|
|
2651
|
-
{ name: "openalex", make: () => openalex },
|
|
2652
|
-
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
2653
|
-
];
|
|
2659
|
+
${CORRECTED_HEADING}
|
|
2654
2660
|
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2661
|
+
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
2662
|
+
|
|
2663
|
+
${sentences.join("\n\n")}
|
|
2664
|
+
`;
|
|
2665
|
+
}
|
|
2666
|
+
function syncCorrectedBlock(p) {
|
|
2667
|
+
if (!existsSync5(p.fulltextMd) || !existsSync5(p.xml)) return;
|
|
2668
|
+
let r;
|
|
2669
|
+
try {
|
|
2670
|
+
r = parsePubmedXml(readFileSync4(p.xml, "utf8"));
|
|
2671
|
+
} catch {
|
|
2672
|
+
return;
|
|
2673
|
+
}
|
|
2674
|
+
if (!r) return;
|
|
2675
|
+
const md = readFileSync4(p.fulltextMd, "utf8");
|
|
2676
|
+
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
2677
|
+
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
2678
|
+
}
|
|
2679
|
+
function applyFulltext(p, m, hit, converter) {
|
|
2680
|
+
const raw = p.fulltextRaw(hit.format);
|
|
2681
|
+
writeAtomic(raw, hit.raw);
|
|
2682
|
+
writeAtomic(p.fulltextMd, hit.md);
|
|
2683
|
+
syncCorrectedBlock(p);
|
|
2684
|
+
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
2685
|
+
const own = declarationsFromMd(hit.md);
|
|
2686
|
+
const next = {
|
|
2687
|
+
...m,
|
|
2688
|
+
level: "full_text",
|
|
2689
|
+
version: hit.version,
|
|
2690
|
+
files,
|
|
2691
|
+
retrieved_by: hit.tier,
|
|
2692
|
+
retrieved_date: today(),
|
|
2693
|
+
converted_by: converter,
|
|
2694
|
+
coi: m.coi ?? own.coi,
|
|
2695
|
+
funding: m.funding.length ? m.funding : own.funding
|
|
2670
2696
|
};
|
|
2697
|
+
writeMeta(p, next);
|
|
2698
|
+
return next;
|
|
2671
2699
|
}
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
"
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
]
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2700
|
+
function recordAttempts(p, m, labels) {
|
|
2701
|
+
if (!labels.length) return m;
|
|
2702
|
+
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
2703
|
+
const replaced = new Set(labels.map(tierOf));
|
|
2704
|
+
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
2705
|
+
writeMeta(p, next);
|
|
2706
|
+
return next;
|
|
2707
|
+
}
|
|
2708
|
+
function writeLabelEntry(p, s, xml, converter) {
|
|
2709
|
+
mkdirSync3(p.entryDir, { recursive: true });
|
|
2710
|
+
const raw = p.fulltextRaw("xml");
|
|
2711
|
+
writeAtomic(raw, xml);
|
|
2712
|
+
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
2713
|
+
const meta = {
|
|
2714
|
+
pmid: "",
|
|
2715
|
+
setid: s.setid,
|
|
2716
|
+
nctid: null,
|
|
2717
|
+
publications: [],
|
|
2718
|
+
pmcid: null,
|
|
2719
|
+
doi: null,
|
|
2720
|
+
title: s.title,
|
|
2721
|
+
first_author: s.labeler,
|
|
2722
|
+
year: s.effective?.slice(0, 4) ?? null,
|
|
2723
|
+
publication_types: ["Drug Label"],
|
|
2724
|
+
retracted: false,
|
|
2725
|
+
corrections: [],
|
|
2726
|
+
coi: null,
|
|
2727
|
+
funding: [],
|
|
2728
|
+
has_abstract: false,
|
|
2729
|
+
level: "full_text",
|
|
2730
|
+
version: s.version ? `spl ${s.version}` : null,
|
|
2731
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2732
|
+
retrieved_by: "dailymed",
|
|
2733
|
+
retrieved_date: today(),
|
|
2734
|
+
converted_by: converter,
|
|
2735
|
+
pubmed_checked: today(),
|
|
2736
|
+
attempts: []
|
|
2706
2737
|
};
|
|
2738
|
+
writeMeta(p, meta);
|
|
2739
|
+
return meta;
|
|
2707
2740
|
}
|
|
2708
|
-
function
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2741
|
+
function writeStudyEntry(p, s, json2, converter) {
|
|
2742
|
+
mkdirSync3(p.entryDir, { recursive: true });
|
|
2743
|
+
const raw = p.fulltextRaw("json");
|
|
2744
|
+
writeAtomic(raw, json2);
|
|
2745
|
+
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
2746
|
+
const meta = {
|
|
2747
|
+
pmid: "",
|
|
2748
|
+
setid: null,
|
|
2749
|
+
nctid: s.nctid,
|
|
2750
|
+
publications: s.publications,
|
|
2751
|
+
pmcid: null,
|
|
2752
|
+
doi: null,
|
|
2753
|
+
title: s.title,
|
|
2754
|
+
first_author: s.sponsor,
|
|
2755
|
+
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
2756
|
+
publication_types: ["Trial Registration"],
|
|
2757
|
+
retracted: false,
|
|
2758
|
+
corrections: [],
|
|
2759
|
+
coi: null,
|
|
2760
|
+
funding: [],
|
|
2761
|
+
has_abstract: false,
|
|
2762
|
+
level: "full_text",
|
|
2763
|
+
version: s.lastUpdate,
|
|
2764
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2765
|
+
retrieved_by: "ctgov",
|
|
2766
|
+
retrieved_date: today(),
|
|
2767
|
+
converted_by: converter,
|
|
2768
|
+
pubmed_checked: today(),
|
|
2769
|
+
attempts: []
|
|
2770
|
+
};
|
|
2771
|
+
writeMeta(p, meta);
|
|
2772
|
+
return meta;
|
|
2717
2773
|
}
|
|
2718
|
-
function
|
|
2719
|
-
|
|
2720
|
-
const
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
const
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
|
|
2727
|
-
const tiers = [], seen = /* @__PURE__ */ new Set();
|
|
2728
|
-
for (const name of order2) {
|
|
2729
|
-
if (seen.has(name)) continue;
|
|
2730
|
-
seen.add(name);
|
|
2731
|
-
const t = available.get(name);
|
|
2732
|
-
if (t) tiers.push(t);
|
|
2733
|
-
else if (name === "proxy") problems.push('AI4L_LIBRARY_TIERS: "proxy" listed, but AI4L_LIBRARY_PROXY_URL is unset');
|
|
2734
|
-
else problems.push(`AI4L_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when AI4L_LIBRARY_PROXY_URL is set)`);
|
|
2735
|
-
}
|
|
2736
|
-
const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
|
|
2737
|
-
return { tiers, all, source: sourceOf("AI4L_LIBRARY_TIERS", env), problems };
|
|
2774
|
+
function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
|
|
2775
|
+
mkdirSync3(p.entryDir, { recursive: true });
|
|
2776
|
+
const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
|
|
2777
|
+
writeAtomic(raw, json2);
|
|
2778
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2779
|
+
const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
|
|
2780
|
+
writeMeta(p, meta);
|
|
2781
|
+
return meta;
|
|
2738
2782
|
}
|
|
2739
2783
|
|
|
2740
2784
|
// src/cascade.ts
|
|
@@ -2769,7 +2813,7 @@ async function fulltextCascade(p, attempts, onEvent, tiers = resolveTiers().tier
|
|
|
2769
2813
|
for (const tier of tiers) {
|
|
2770
2814
|
if (!tier.always && shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked)) continue;
|
|
2771
2815
|
const applies = tier.applies(p);
|
|
2772
|
-
const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }) };
|
|
2816
|
+
const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
|
|
2773
2817
|
const r = applies === true ? await tier.run(p, ctx) : { ...miss(tier.name, applies), skipped: true };
|
|
2774
2818
|
onEvent?.(r);
|
|
2775
2819
|
if (r.ok) return { hit: r, misses };
|
|
@@ -2809,8 +2853,8 @@ function parseId(raw) {
|
|
|
2809
2853
|
}
|
|
2810
2854
|
|
|
2811
2855
|
// src/info.ts
|
|
2812
|
-
import { existsSync as existsSync7, readdirSync as readdirSync3, readFileSync as readFileSync6, statSync } from "fs";
|
|
2813
|
-
import { dirname as dirname2, join as
|
|
2856
|
+
import { existsSync as existsSync7, readdirSync as readdirSync3, readFileSync as readFileSync6, statSync as statSync2 } from "fs";
|
|
2857
|
+
import { dirname as dirname2, join as join7 } from "path";
|
|
2814
2858
|
|
|
2815
2859
|
// src/verify.ts
|
|
2816
2860
|
import { existsSync as existsSync6, readFileSync as readFileSync5 } from "fs";
|
|
@@ -2960,8 +3004,8 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
2960
3004
|
function* metaFiles(dir) {
|
|
2961
3005
|
if (!existsSync7(dir)) return;
|
|
2962
3006
|
for (const name of readdirSync3(dir)) {
|
|
2963
|
-
const p =
|
|
2964
|
-
if (
|
|
3007
|
+
const p = join7(dir, name);
|
|
3008
|
+
if (statSync2(p).isDirectory()) yield* metaFiles(p);
|
|
2965
3009
|
else if (name.endsWith("-meta.md")) yield p;
|
|
2966
3010
|
}
|
|
2967
3011
|
}
|
|
@@ -3000,8 +3044,8 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3000
3044
|
};
|
|
3001
3045
|
const ids = [];
|
|
3002
3046
|
for (const type of existsSync7(libraryDir) ? readdirSync3(libraryDir) : []) {
|
|
3003
|
-
const typeDir =
|
|
3004
|
-
if (!
|
|
3047
|
+
const typeDir = join7(libraryDir, type);
|
|
3048
|
+
if (!statSync2(typeDir).isDirectory()) continue;
|
|
3005
3049
|
for (const meta of metaFiles(typeDir)) {
|
|
3006
3050
|
const md = readFileSync6(meta, "utf8");
|
|
3007
3051
|
const raw = field(md, "level") || "unknown";
|
|
@@ -3015,7 +3059,7 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3015
3059
|
const dir = dirname2(meta);
|
|
3016
3060
|
for (const f of readdirSync3(dir)) {
|
|
3017
3061
|
try {
|
|
3018
|
-
const st =
|
|
3062
|
+
const st = statSync2(join7(dir, f));
|
|
3019
3063
|
if (st.isFile()) info.size_bytes += st.size;
|
|
3020
3064
|
} catch {
|
|
3021
3065
|
}
|
|
@@ -3143,45 +3187,6 @@ function search(md, query, maxHits) {
|
|
|
3143
3187
|
}
|
|
3144
3188
|
var escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3145
3189
|
|
|
3146
|
-
// src/log.ts
|
|
3147
|
-
import { appendFileSync, mkdirSync as mkdirSync3, renameSync as renameSync2, statSync as statSync2 } from "fs";
|
|
3148
|
-
import { hostname } from "os";
|
|
3149
|
-
import { join as join7 } from "path";
|
|
3150
|
-
var HOST = hostname();
|
|
3151
|
-
var LOG_FILE = join7(AI4L_LIBRARY_DIR, "library.log");
|
|
3152
|
-
function stamp() {
|
|
3153
|
-
const d = /* @__PURE__ */ new Date();
|
|
3154
|
-
const p = (n) => String(n).padStart(2, "0");
|
|
3155
|
-
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
3156
|
-
}
|
|
3157
|
-
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
3158
|
-
function rotateIfLarge(file, max = LOG_MAX_BYTES) {
|
|
3159
|
-
let size = 0;
|
|
3160
|
-
try {
|
|
3161
|
-
size = statSync2(file).size;
|
|
3162
|
-
} catch {
|
|
3163
|
-
return false;
|
|
3164
|
-
}
|
|
3165
|
-
if (size <= max) return false;
|
|
3166
|
-
try {
|
|
3167
|
-
renameSync2(file, `${file}.1`);
|
|
3168
|
-
return true;
|
|
3169
|
-
} catch {
|
|
3170
|
-
return false;
|
|
3171
|
-
}
|
|
3172
|
-
}
|
|
3173
|
-
function log(tool, id, message) {
|
|
3174
|
-
const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
3175
|
-
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
|
|
3176
|
-
`;
|
|
3177
|
-
try {
|
|
3178
|
-
mkdirSync3(AI4L_LIBRARY_DIR, { recursive: true });
|
|
3179
|
-
rotateIfLarge(LOG_FILE);
|
|
3180
|
-
appendFileSync(LOG_FILE, line);
|
|
3181
|
-
} catch {
|
|
3182
|
-
}
|
|
3183
|
-
}
|
|
3184
|
-
|
|
3185
3190
|
// src/doi.ts
|
|
3186
3191
|
var NCBI_ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi";
|
|
3187
3192
|
async function pmidForDoi(doi) {
|
|
@@ -3371,12 +3376,17 @@ function noticeTag(m) {
|
|
|
3371
3376
|
].filter(Boolean);
|
|
3372
3377
|
return parts.length ? " " + parts.join(" ") : "";
|
|
3373
3378
|
}
|
|
3379
|
+
var modelFailure = (error, cfg) => error.startsWith("no verify model configured") || !!cfg && error.startsWith(`${cfg.model}:`);
|
|
3374
3380
|
function registerTools(server2) {
|
|
3375
3381
|
async function completeFulltext(paths, meta, requested) {
|
|
3376
3382
|
const paper = { title: meta.title, pmid: meta.pmid, pmcid: meta.pmcid, doi: meta.doi, headLine: readHeadLine(paths) };
|
|
3377
3383
|
const tried = /* @__PURE__ */ new Set();
|
|
3378
3384
|
let ran = false;
|
|
3379
3385
|
const { hit } = await fulltextCascade(paper, meta.attempts, (e) => {
|
|
3386
|
+
if (e.ok === "alert") {
|
|
3387
|
+
log("library_add", requested, `!! ${e.message}`);
|
|
3388
|
+
return;
|
|
3389
|
+
}
|
|
3380
3390
|
if (e.ok === "try") tried.add(e.tier);
|
|
3381
3391
|
if (e.ok === false && e.ephemeral) return;
|
|
3382
3392
|
ran = true;
|
|
@@ -3688,7 +3698,7 @@ function registerTools(server2) {
|
|
|
3688
3698
|
const cfg = verifyConfig();
|
|
3689
3699
|
const r = await verifyClaim(claim, id, cfg, cfg ? decider(cfg) : null);
|
|
3690
3700
|
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
3691
|
-
log("library_verify", r.id, r.error ? `${secs} verdict=${"error".padEnd(5)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(5)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
3701
|
+
log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(5)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(5)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
3692
3702
|
return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
|
|
3693
3703
|
}
|
|
3694
3704
|
);
|
|
@@ -3724,11 +3734,11 @@ function registerTools(server2) {
|
|
|
3724
3734
|
}
|
|
3725
3735
|
|
|
3726
3736
|
// src/index.ts
|
|
3727
|
-
loadDotenv();
|
|
3728
3737
|
var server = new McpServer(
|
|
3729
3738
|
{ name: pkg.name, version: pkg.version },
|
|
3730
3739
|
{ instructions: INSTRUCTIONS }
|
|
3731
3740
|
);
|
|
3732
3741
|
registerTools(server);
|
|
3742
|
+
for (const problem of resolveTiers().problems) log("startup", null, `!! ${problem}`);
|
|
3733
3743
|
var transport = new StdioServerTransport();
|
|
3734
3744
|
await server.connect(transport);
|