fh-library 0.2.17 → 0.2.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +5 -5
- package/dist/index.js +1810 -1636
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,10 +11,10 @@ function dotenvFile(env = process.env) {
|
|
|
11
11
|
return join(resolve(lib), ".env");
|
|
12
12
|
}
|
|
13
13
|
var loadedFromDotenv = /* @__PURE__ */ new Set();
|
|
14
|
-
function loadDotenv(
|
|
15
|
-
if (!existsSync(
|
|
14
|
+
function loadDotenv(file3 = dotenvFile()) {
|
|
15
|
+
if (!existsSync(file3)) return [];
|
|
16
16
|
const loaded = [];
|
|
17
|
-
for (const raw of readFileSync(
|
|
17
|
+
for (const raw of readFileSync(file3, "utf8").split(/\r?\n/)) {
|
|
18
18
|
const line = raw.trim();
|
|
19
19
|
if (!line || line.startsWith("#")) continue;
|
|
20
20
|
const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
@@ -1208,1693 +1208,1747 @@ async function unlock(url, postImpl = post) {
|
|
|
1208
1208
|
return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
|
|
1209
1209
|
}
|
|
1210
1210
|
|
|
1211
|
-
// src/
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
if (v === "submittedVersion") return "preprint";
|
|
1215
|
-
return "published";
|
|
1216
|
-
}
|
|
1217
|
-
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
1218
|
-
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
1211
|
+
// src/walls.ts
|
|
1212
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4 } from "fs";
|
|
1213
|
+
import { join as join5 } from "path";
|
|
1219
1214
|
|
|
1220
|
-
// src/
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1215
|
+
// src/entry.ts
|
|
1216
|
+
import { copyFileSync, existsSync as existsSync4, mkdirSync as mkdirSync2, readdirSync, readFileSync as readFileSync3, renameSync, rmSync, unlinkSync, writeFileSync } from "fs";
|
|
1217
|
+
import { join as join4 } from "path";
|
|
1218
|
+
import { basename, dirname } from "path";
|
|
1219
|
+
|
|
1220
|
+
// src/pubmed.ts
|
|
1221
|
+
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
1222
|
+
var CORRECTION_TYPES = {
|
|
1223
|
+
ErratumIn: "erratum_in",
|
|
1224
|
+
RetractionIn: "retraction_in",
|
|
1225
|
+
PartialRetractionIn: "partial_retraction_in",
|
|
1226
|
+
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
1227
|
+
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
1228
|
+
UpdateIn: "update_in"
|
|
1227
1229
|
};
|
|
1228
|
-
var
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
ctx.tried.add(c.url);
|
|
1241
|
-
let r = await tryOne(p, c, tier);
|
|
1242
|
-
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
1243
|
-
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
1244
|
-
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
1245
|
-
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
1246
|
-
for (const b of browserRungs()) {
|
|
1247
|
-
if (r.ok || c.headers || c.service || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
1248
|
-
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
1249
|
-
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
1250
|
-
if (!r.ok && bare(r.reason).startsWith(MISMATCH)) ctx.onAlert?.(`browser ${b.label}: ${bare(r.reason)}`);
|
|
1251
|
-
}
|
|
1252
|
-
if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
1253
|
-
r = await tryOne(p, target, tier, "unlocker");
|
|
1254
|
-
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
1255
|
-
if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
|
|
1256
|
-
}
|
|
1257
|
-
if (r.ok) return r;
|
|
1258
|
-
reasons.push(`${r.reason}: ${hostOf2(c.url)}`);
|
|
1259
|
-
}
|
|
1260
|
-
return miss(tier, reasons.join("; "));
|
|
1230
|
+
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
1231
|
+
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
1232
|
+
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
1233
|
+
function efetchUrl(pmid) {
|
|
1234
|
+
const u = new URL(EFETCH);
|
|
1235
|
+
u.searchParams.set("db", "pubmed");
|
|
1236
|
+
u.searchParams.set("id", pmid);
|
|
1237
|
+
u.searchParams.set("retmode", "xml");
|
|
1238
|
+
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
1239
|
+
u.searchParams.set("tool", "fh-library");
|
|
1240
|
+
u.searchParams.set("email", contactEmail());
|
|
1241
|
+
return u.toString();
|
|
1261
1242
|
}
|
|
1262
|
-
|
|
1263
|
-
async function tryOne(p, c, tier, via = "plain") {
|
|
1264
|
-
let got;
|
|
1243
|
+
async function fetchPubmedXml(pmid) {
|
|
1265
1244
|
try {
|
|
1266
|
-
|
|
1245
|
+
return await fetchText(efetchUrl(pmid));
|
|
1267
1246
|
} catch (e) {
|
|
1268
|
-
|
|
1269
|
-
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
1270
|
-
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
1247
|
+
throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
1271
1248
|
}
|
|
1272
|
-
const { bytes, contentType } = got;
|
|
1273
|
-
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
1274
|
-
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
1275
|
-
const r = await convertBytes(p, bytes, contentType);
|
|
1276
|
-
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
1277
|
-
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
1278
1249
|
}
|
|
1279
|
-
var
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1250
|
+
var parser2 = new XMLParser2({
|
|
1251
|
+
ignoreAttributes: false,
|
|
1252
|
+
attributeNamePrefix: "@_",
|
|
1253
|
+
parseTagValue: false,
|
|
1254
|
+
processEntities: true,
|
|
1255
|
+
stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
|
|
1256
|
+
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
1257
|
+
});
|
|
1258
|
+
function text(n) {
|
|
1259
|
+
if (n == null) return "";
|
|
1260
|
+
if (typeof n === "object") return text(n["#text"]);
|
|
1261
|
+
return String(n);
|
|
1262
|
+
}
|
|
1263
|
+
var plain = (n) => decodeEntities(text(n)).trim();
|
|
1264
|
+
function inlineText2(raw) {
|
|
1265
|
+
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
1266
|
+
}
|
|
1267
|
+
function decodeEntities(s) {
|
|
1268
|
+
return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
1269
|
+
}
|
|
1270
|
+
function parsePubmedXml(xml) {
|
|
1271
|
+
const doc = parser2.parse(xml);
|
|
1272
|
+
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
1273
|
+
if (!article) {
|
|
1274
|
+
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
1275
|
+
return book ? parseBook(book) : null;
|
|
1288
1276
|
}
|
|
1289
|
-
const
|
|
1290
|
-
|
|
1277
|
+
const cit = article.MedlineCitation ?? {};
|
|
1278
|
+
const art = cit.Article ?? {};
|
|
1279
|
+
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
1280
|
+
const idOf = (t) => {
|
|
1281
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1282
|
+
return hit ? text(hit).trim() || null : null;
|
|
1283
|
+
};
|
|
1284
|
+
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
1285
|
+
const authors = authorList.map((a) => {
|
|
1286
|
+
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
1287
|
+
const last = plain(a.LastName);
|
|
1288
|
+
const ini = plain(a.Initials);
|
|
1289
|
+
return [last, ini].filter(Boolean).join(" ");
|
|
1290
|
+
}).filter(Boolean);
|
|
1291
|
+
const first = authorList[0];
|
|
1292
|
+
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
1293
|
+
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
1294
|
+
let year = text(pub.Year).trim() || null;
|
|
1295
|
+
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
1296
|
+
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
1297
|
+
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
1298
|
+
text: inlineText2(text(a))
|
|
1299
|
+
})).filter((a) => a.text);
|
|
1300
|
+
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
1301
|
+
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
1302
|
+
if (!type) return [];
|
|
1303
|
+
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
|
|
1304
|
+
});
|
|
1305
|
+
const funding = [];
|
|
1306
|
+
for (const g of art.GrantList?.Grant ?? []) {
|
|
1307
|
+
const a = plain(g.Agency);
|
|
1308
|
+
if (a && !funding.includes(a)) funding.push(a);
|
|
1309
|
+
}
|
|
1310
|
+
return {
|
|
1311
|
+
coi: inlineText2(text(cit.CoiStatement)) || null,
|
|
1312
|
+
funding,
|
|
1313
|
+
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
1314
|
+
pmcid: idOf("pmc"),
|
|
1315
|
+
doi: idOf("doi"),
|
|
1316
|
+
title: inlineText2(text(art.ArticleTitle)),
|
|
1317
|
+
authors,
|
|
1318
|
+
firstAuthor,
|
|
1319
|
+
journal: plain(art.Journal?.Title) || null,
|
|
1320
|
+
year,
|
|
1321
|
+
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
1322
|
+
abstract,
|
|
1323
|
+
corrections
|
|
1324
|
+
};
|
|
1291
1325
|
}
|
|
1292
|
-
function
|
|
1293
|
-
const
|
|
1294
|
-
const
|
|
1295
|
-
const
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1326
|
+
function parseBook(b) {
|
|
1327
|
+
const d = b.BookDocument ?? {};
|
|
1328
|
+
const book = d.Book ?? {};
|
|
1329
|
+
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
1330
|
+
const idOf = (t) => {
|
|
1331
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1332
|
+
return hit ? text(hit).trim() || null : null;
|
|
1333
|
+
};
|
|
1334
|
+
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
1335
|
+
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
1336
|
+
const people = pick?.Author ?? [];
|
|
1337
|
+
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
1338
|
+
const first = people[0];
|
|
1339
|
+
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
|
|
1340
|
+
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
1341
|
+
return {
|
|
1342
|
+
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
1343
|
+
pmcid: null,
|
|
1344
|
+
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
1345
|
+
title: inlineText2(text(d.ArticleTitle)) || inlineText2(text(book.BookTitle)),
|
|
1346
|
+
authors,
|
|
1347
|
+
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
1348
|
+
journal: plain(book.Publisher?.PublisherName) || null,
|
|
1349
|
+
year: text(book.PubDate?.Year).trim() || null,
|
|
1350
|
+
publicationTypes: [...new Set(types)],
|
|
1351
|
+
abstract,
|
|
1352
|
+
corrections: [],
|
|
1353
|
+
coi: null,
|
|
1354
|
+
funding: []
|
|
1355
|
+
};
|
|
1299
1356
|
}
|
|
1300
|
-
async function convertBytes(p, bytes, contentType = "") {
|
|
1301
|
-
let format;
|
|
1302
|
-
let md;
|
|
1303
|
-
if (looksLikePdf(bytes)) {
|
|
1304
|
-
format = "pdf";
|
|
1305
|
-
const stats2 = {};
|
|
1306
|
-
try {
|
|
1307
|
-
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
|
|
1308
|
-
} catch (e) {
|
|
1309
|
-
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
1310
|
-
}
|
|
1311
|
-
if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
1312
|
-
} else {
|
|
1313
|
-
const text3 = bytes.toString("utf8");
|
|
1314
|
-
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
1315
|
-
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
1316
|
-
format = "xml";
|
|
1317
|
-
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
1318
|
-
if (!doc.title) doc.title = p.title;
|
|
1319
|
-
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
1320
|
-
md = renderFulltextMd(doc, p.headLine);
|
|
1321
|
-
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
1322
|
-
format = "html";
|
|
1323
|
-
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
1324
|
-
md = `# ${p.title}
|
|
1325
1357
|
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
1335
|
-
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
1336
|
-
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
1337
|
-
return { ok: true, format, md };
|
|
1358
|
+
// src/spl.ts
|
|
1359
|
+
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
1360
|
+
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
1361
|
+
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
1362
|
+
async function fetchSpl(setid) {
|
|
1363
|
+
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
1364
|
+
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
1365
|
+
return xml;
|
|
1338
1366
|
}
|
|
1339
|
-
var
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
[/ce:sup\b/g, "sup"],
|
|
1354
|
-
[/ce:inf\b/g, "sub"],
|
|
1355
|
-
[/ce:cross-refs?\b/g, "xref"],
|
|
1356
|
-
[/ce:list-item\b/g, "list-item"],
|
|
1357
|
-
[/ce:list\b/g, "list"],
|
|
1358
|
-
[/ce:label\b/g, "label"],
|
|
1359
|
-
[/ce:caption\b/g, "caption"],
|
|
1360
|
-
[/ce:table\b/g, "table-wrap"],
|
|
1361
|
-
[/ce:figure\b/g, "fig"],
|
|
1362
|
-
[/ce:bib-reference\b/g, "ref"],
|
|
1363
|
-
[/ce:bibliography\b/g, "ref-list"],
|
|
1364
|
-
[/sb:reference\b/g, "mixed-citation"],
|
|
1365
|
-
[/ce:acknowledgment\b/g, "ack"],
|
|
1366
|
-
[/ce:inter-ref\b/g, "ext-link"]
|
|
1367
|
-
];
|
|
1368
|
-
function elsevierToJats(xml) {
|
|
1369
|
-
let out = xml;
|
|
1370
|
-
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
1371
|
-
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
// src/tiers/core.ts
|
|
1375
|
-
var WINDOW_MS = 6e4;
|
|
1376
|
-
var PER_WINDOW = 10;
|
|
1377
|
-
var MAX_WAIT_MS = 65e3;
|
|
1378
|
-
var TIMEOUT_MS3 = 1e4;
|
|
1379
|
-
var calls = [];
|
|
1380
|
-
async function pace() {
|
|
1381
|
-
const now = Date.now();
|
|
1382
|
-
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
1383
|
-
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
1384
|
-
calls.push(Date.now());
|
|
1367
|
+
var parser3 = new XMLParser3({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
|
|
1368
|
+
var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
1369
|
+
var kids2 = (n) => n[tagOf2(n)] ?? [];
|
|
1370
|
+
var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
1371
|
+
var squash2 = (s) => s.replace(/\s+/g, " ").trim();
|
|
1372
|
+
var child = (n, tag) => kids2(n).find((x) => tagOf2(x) === tag);
|
|
1373
|
+
function findDeep2(nodes, tag) {
|
|
1374
|
+
for (const n of nodes) {
|
|
1375
|
+
if ("#text" in n) continue;
|
|
1376
|
+
if (tagOf2(n) === tag) return n;
|
|
1377
|
+
const h = findDeep2(kids2(n), tag);
|
|
1378
|
+
if (h) return h;
|
|
1379
|
+
}
|
|
1380
|
+
return void 0;
|
|
1385
1381
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
for (
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
if (res.ok) return await res.text();
|
|
1392
|
-
const body = await res.text().catch(() => "");
|
|
1393
|
-
if (res.status === 429 && attempt === 0) {
|
|
1394
|
-
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
1395
|
-
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
1396
|
-
await new Promise((r) => setTimeout(r, wait));
|
|
1382
|
+
function inline(nodes) {
|
|
1383
|
+
let out = "";
|
|
1384
|
+
for (const n of nodes) {
|
|
1385
|
+
if ("#text" in n) {
|
|
1386
|
+
out += String(n["#text"]);
|
|
1397
1387
|
continue;
|
|
1398
1388
|
}
|
|
1399
|
-
|
|
1389
|
+
const t = tagOf2(n);
|
|
1390
|
+
const inner = () => inline(kids2(n));
|
|
1391
|
+
switch (t) {
|
|
1392
|
+
case "content": {
|
|
1393
|
+
const style = attr2(n, "styleCode").toLowerCase();
|
|
1394
|
+
const s = inner();
|
|
1395
|
+
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
1396
|
+
break;
|
|
1397
|
+
}
|
|
1398
|
+
case "sup":
|
|
1399
|
+
out += `^${inner()}^`;
|
|
1400
|
+
break;
|
|
1401
|
+
case "sub":
|
|
1402
|
+
out += `~${inner()}~`;
|
|
1403
|
+
break;
|
|
1404
|
+
case "br":
|
|
1405
|
+
out += " ";
|
|
1406
|
+
break;
|
|
1407
|
+
case "footnote":
|
|
1408
|
+
out += ` (${squash2(inner())})`;
|
|
1409
|
+
break;
|
|
1410
|
+
case "footnoteRef":
|
|
1411
|
+
case "renderMultiMedia":
|
|
1412
|
+
break;
|
|
1413
|
+
default:
|
|
1414
|
+
out += inner();
|
|
1415
|
+
}
|
|
1400
1416
|
}
|
|
1401
|
-
|
|
1417
|
+
return out;
|
|
1402
1418
|
}
|
|
1403
|
-
function
|
|
1404
|
-
const
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
if (!
|
|
1410
|
-
let data;
|
|
1411
|
-
try {
|
|
1412
|
-
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
1413
|
-
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
1414
|
-
} catch (e) {
|
|
1415
|
-
return miss(this.name, errMsg(e));
|
|
1416
|
-
}
|
|
1417
|
-
const candidates = coreCandidates(data);
|
|
1418
|
-
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
1419
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1420
|
-
return download(this.name, p, candidates, ctx);
|
|
1419
|
+
function table(n) {
|
|
1420
|
+
const row3 = (x) => kids2(x).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => squash2(inline(kids2(c))).replace(/\|/g, "\\|"));
|
|
1421
|
+
const collect = (nodes, into) => {
|
|
1422
|
+
for (const x of nodes) {
|
|
1423
|
+
if ("#text" in x) continue;
|
|
1424
|
+
if (tagOf2(x) === "tr") into.push(row3(x));
|
|
1425
|
+
else if (!["thead", "tfoot"].includes(tagOf2(x))) collect(kids2(x), into);
|
|
1421
1426
|
}
|
|
1422
1427
|
};
|
|
1428
|
+
const head = [], body = [], foot = [];
|
|
1429
|
+
for (const x of kids2(n)) {
|
|
1430
|
+
const t = tagOf2(x);
|
|
1431
|
+
if (t === "thead") collect(kids2(x), head);
|
|
1432
|
+
else if (t === "tfoot") collect(kids2(x), foot);
|
|
1433
|
+
else if (t === "tr") body.push(row3(x));
|
|
1434
|
+
else if (!("#text" in x) && t !== "caption") collect(kids2(x), body);
|
|
1435
|
+
}
|
|
1436
|
+
const rows = [...head, ...body];
|
|
1437
|
+
if (!rows.length) return "";
|
|
1438
|
+
const width = Math.max(...rows.map((r) => r.length));
|
|
1439
|
+
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
1440
|
+
const cap = child(n, "caption");
|
|
1441
|
+
const title = cap ? squash2(inline(kids2(cap))) : "";
|
|
1442
|
+
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
1443
|
+
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
1423
1444
|
}
|
|
1424
|
-
function
|
|
1445
|
+
function blocks(nodes, depth) {
|
|
1425
1446
|
const out = [];
|
|
1426
|
-
const
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1447
|
+
for (const n of nodes) {
|
|
1448
|
+
if ("#text" in n) {
|
|
1449
|
+
const t2 = squash2(String(n["#text"]));
|
|
1450
|
+
if (t2) out.push(t2);
|
|
1451
|
+
continue;
|
|
1452
|
+
}
|
|
1453
|
+
const t = tagOf2(n);
|
|
1454
|
+
const c = kids2(n);
|
|
1455
|
+
switch (t) {
|
|
1456
|
+
case "paragraph": {
|
|
1457
|
+
const s = squash2(inline(c.filter((x) => !["table", "list"].includes(tagOf2(x)))));
|
|
1458
|
+
if (s) out.push(s);
|
|
1459
|
+
out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf2(x))), depth));
|
|
1460
|
+
break;
|
|
1461
|
+
}
|
|
1462
|
+
case "list": {
|
|
1463
|
+
const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr2(n, "listType"));
|
|
1464
|
+
let i = 0;
|
|
1465
|
+
for (const it of c.filter((x) => tagOf2(x) === "item")) {
|
|
1466
|
+
i++;
|
|
1467
|
+
const body = blocks(kids2(it).filter((x) => tagOf2(x) !== "caption"), depth).join(" ");
|
|
1468
|
+
if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
1469
|
+
}
|
|
1470
|
+
break;
|
|
1471
|
+
}
|
|
1472
|
+
case "table": {
|
|
1473
|
+
const md = table(n);
|
|
1474
|
+
if (md) out.push(md);
|
|
1475
|
+
break;
|
|
1476
|
+
}
|
|
1477
|
+
case "renderMultiMedia":
|
|
1478
|
+
case "observationMedia":
|
|
1479
|
+
case "subject":
|
|
1480
|
+
break;
|
|
1481
|
+
case "section":
|
|
1482
|
+
out.push(...section(n, depth));
|
|
1483
|
+
break;
|
|
1484
|
+
case "component":
|
|
1485
|
+
case "text":
|
|
1486
|
+
case "excerpt":
|
|
1487
|
+
case "highlight":
|
|
1488
|
+
default:
|
|
1489
|
+
out.push(...blocks(c, depth));
|
|
1490
|
+
break;
|
|
1491
|
+
}
|
|
1436
1492
|
}
|
|
1437
1493
|
return out;
|
|
1438
1494
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
var
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
let data;
|
|
1446
|
-
try {
|
|
1447
|
-
const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
|
|
1448
|
-
data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
|
|
1449
|
-
} catch (e) {
|
|
1450
|
-
return miss(this.name, errMsg(e));
|
|
1451
|
-
}
|
|
1452
|
-
const rec = data.resultList?.result?.[0];
|
|
1453
|
-
if (!rec) return miss(this.name, "not in europe pmc");
|
|
1454
|
-
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
1455
|
-
if (!candidates.length) return miss(this.name, "no free copy");
|
|
1456
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1457
|
-
return download(this.name, p, candidates, ctx);
|
|
1458
|
-
}
|
|
1495
|
+
var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
|
|
1496
|
+
var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
|
|
1497
|
+
var plain2 = (nodes) => {
|
|
1498
|
+
let out = "";
|
|
1499
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
|
|
1500
|
+
return out;
|
|
1459
1501
|
};
|
|
1460
|
-
function
|
|
1502
|
+
function section(n, depth) {
|
|
1503
|
+
const code = child(n, "code");
|
|
1504
|
+
const c = code ? attr2(code, "code") : "";
|
|
1505
|
+
if (SKIP_CODES.has(c)) return [];
|
|
1506
|
+
const title = child(n, "title");
|
|
1507
|
+
let head = title ? squash2(plain2(kids2(title))) : "";
|
|
1508
|
+
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
1509
|
+
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
1461
1510
|
const out = [];
|
|
1462
|
-
|
|
1463
|
-
const
|
|
1464
|
-
|
|
1465
|
-
seen.add(url);
|
|
1466
|
-
out.push({ url, version, source: "europepmc", host, pdf });
|
|
1467
|
-
};
|
|
1468
|
-
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
1469
|
-
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
1470
|
-
if (id && r.inEPMC === "Y") {
|
|
1471
|
-
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
1472
|
-
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
1473
|
-
}
|
|
1474
|
-
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
1475
|
-
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
1476
|
-
if (u.documentStyle === "abs") continue;
|
|
1477
|
-
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
1478
|
-
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
1479
|
-
}
|
|
1511
|
+
if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
|
|
1512
|
+
const rest = kids2(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf2(x)));
|
|
1513
|
+
out.push(...blocks(rest, head ? depth + 1 : depth));
|
|
1480
1514
|
return out;
|
|
1481
1515
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
}
|
|
1501
|
-
};
|
|
1502
|
-
function openalexCandidates(d) {
|
|
1503
|
-
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
1504
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1505
|
-
const out = [];
|
|
1506
|
-
for (const l of locs) {
|
|
1507
|
-
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
1508
|
-
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
1509
|
-
seen.add(url);
|
|
1510
|
-
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
1516
|
+
var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
|
|
1517
|
+
function parseSpl(xml) {
|
|
1518
|
+
const doc = parser3.parse(xml);
|
|
1519
|
+
const d = doc.find((x) => tagOf2(x) === "document");
|
|
1520
|
+
if (!d) return null;
|
|
1521
|
+
const k = kids2(d);
|
|
1522
|
+
const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
1523
|
+
const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
|
|
1524
|
+
const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
|
|
1525
|
+
const named = (nodes) => {
|
|
1526
|
+
for (const n of nodes) {
|
|
1527
|
+
if ("#text" in n) continue;
|
|
1528
|
+
if (tagOf2(n) === "manufacturedProduct") {
|
|
1529
|
+
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
1530
|
+
if (child(inner2, "name")) return inner2;
|
|
1531
|
+
}
|
|
1532
|
+
const h = named(kids2(n));
|
|
1533
|
+
if (h) return h;
|
|
1511
1534
|
}
|
|
1535
|
+
return void 0;
|
|
1536
|
+
};
|
|
1537
|
+
const inner = named(k);
|
|
1538
|
+
const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
1539
|
+
const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
|
|
1540
|
+
const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
1541
|
+
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
1542
|
+
const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
|
|
1543
|
+
const org = findDeep2(k, "representedOrganization");
|
|
1544
|
+
const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
|
|
1545
|
+
const body = findDeep2(k, "structuredBody");
|
|
1546
|
+
const sections2 = body ? blocks(kids2(body), 2) : [];
|
|
1547
|
+
const docTitle = child(d, "title");
|
|
1548
|
+
const lines = [];
|
|
1549
|
+
let cur = "";
|
|
1550
|
+
for (const x of docTitle ? kids2(docTitle) : []) {
|
|
1551
|
+
if (!("#text" in x) && tagOf2(x) === "br") {
|
|
1552
|
+
lines.push(cur);
|
|
1553
|
+
cur = "";
|
|
1554
|
+
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
1512
1555
|
}
|
|
1513
|
-
|
|
1556
|
+
lines.push(cur);
|
|
1557
|
+
const highlights = lines.map(squash2).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
1558
|
+
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
1559
|
+
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
1560
|
+
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
1561
|
+
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
1562
|
+
}
|
|
1563
|
+
function headLineSpl(s) {
|
|
1564
|
+
return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
|
|
1565
|
+
}
|
|
1566
|
+
function renderLabelMd(s) {
|
|
1567
|
+
const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
|
|
1568
|
+
if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
|
|
1569
|
+
parts.push(...s.sections.flatMap((b) => [b, ""]));
|
|
1570
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1514
1571
|
}
|
|
1515
1572
|
|
|
1516
|
-
// src/
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
u.searchParams.set("tool", "fh-library");
|
|
1524
|
-
u.searchParams.set("email", contactEmail());
|
|
1525
|
-
return u.toString();
|
|
1573
|
+
// src/ctgov.ts
|
|
1574
|
+
var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
|
|
1575
|
+
var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
|
|
1576
|
+
async function fetchStudy(nctid) {
|
|
1577
|
+
const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
|
|
1578
|
+
if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
|
|
1579
|
+
return json2;
|
|
1526
1580
|
}
|
|
1527
|
-
var
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
1543
|
-
}
|
|
1581
|
+
var STATUS = {
|
|
1582
|
+
NOT_YET_RECRUITING: "Not yet recruiting",
|
|
1583
|
+
RECRUITING: "Recruiting",
|
|
1584
|
+
ENROLLING_BY_INVITATION: "Enrolling by invitation",
|
|
1585
|
+
ACTIVE_NOT_RECRUITING: "Active, not recruiting",
|
|
1586
|
+
SUSPENDED: "Suspended",
|
|
1587
|
+
TERMINATED: "Terminated",
|
|
1588
|
+
COMPLETED: "Completed",
|
|
1589
|
+
WITHDRAWN: "Withdrawn",
|
|
1590
|
+
UNKNOWN: "Unknown status",
|
|
1591
|
+
AVAILABLE: "Available",
|
|
1592
|
+
NO_LONGER_AVAILABLE: "No longer available",
|
|
1593
|
+
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
1594
|
+
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
1595
|
+
WITHHELD: "Withheld"
|
|
1544
1596
|
};
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1597
|
+
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
1598
|
+
var words = (v) => {
|
|
1599
|
+
const s = String(v ?? "");
|
|
1600
|
+
if (!s) return "";
|
|
1601
|
+
if (STATUS[s]) return STATUS[s];
|
|
1602
|
+
if (PHASE[s]) return PHASE[s];
|
|
1603
|
+
const w = s.toLowerCase().replace(/_/g, " ");
|
|
1604
|
+
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
1605
|
+
};
|
|
1606
|
+
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
1607
|
+
var list = (a) => Array.isArray(a) ? a : [];
|
|
1608
|
+
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
1609
|
+
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
1610
|
+
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
1611
|
+
function parseStudy(json2) {
|
|
1612
|
+
let d;
|
|
1553
1613
|
try {
|
|
1554
|
-
|
|
1614
|
+
d = JSON.parse(json2);
|
|
1555
1615
|
} catch {
|
|
1556
|
-
return
|
|
1557
|
-
}
|
|
1558
|
-
};
|
|
1559
|
-
function tdmCandidates(work, tokens2 = {}, env = process.env) {
|
|
1560
|
-
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
1561
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1562
|
-
const candidates = [];
|
|
1563
|
-
const missingTokens = /* @__PURE__ */ new Set();
|
|
1564
|
-
for (const l of links) {
|
|
1565
|
-
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
1566
|
-
if (seen.has(l.URL)) {
|
|
1567
|
-
const prev = candidates.find((c) => c.url === l.URL);
|
|
1568
|
-
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
1569
|
-
continue;
|
|
1570
|
-
}
|
|
1571
|
-
seen.add(l.URL);
|
|
1572
|
-
const rule = TOKENED.find((r) => r.host.test(hostOf3(l.URL)));
|
|
1573
|
-
let headers;
|
|
1574
|
-
if (rule) {
|
|
1575
|
-
const token = tokens2[rule.key]?.trim() || env[rule.env]?.trim();
|
|
1576
|
-
if (!token) {
|
|
1577
|
-
missingTokens.add(rule.key);
|
|
1578
|
-
continue;
|
|
1579
|
-
}
|
|
1580
|
-
headers = { [rule.header]: token };
|
|
1581
|
-
}
|
|
1582
|
-
if (/text\/plain/.test(ct)) continue;
|
|
1583
|
-
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf3(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
1584
|
-
}
|
|
1585
|
-
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
1586
|
-
candidates.sort((a, b) => rank(a) - rank(b));
|
|
1587
|
-
return { candidates, missingTokens: [...missingTokens] };
|
|
1588
|
-
}
|
|
1589
|
-
function tdmTier(tokens2 = {}) {
|
|
1590
|
-
return {
|
|
1591
|
-
name: "tdm",
|
|
1592
|
-
applies: (p) => p.doi ? true : "no doi",
|
|
1593
|
-
async run(p, ctx) {
|
|
1594
|
-
let work;
|
|
1595
|
-
try {
|
|
1596
|
-
work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
|
|
1597
|
-
} catch (e) {
|
|
1598
|
-
return miss(this.name, errMsg(e));
|
|
1599
|
-
}
|
|
1600
|
-
const { candidates, missingTokens } = tdmCandidates(work, tokens2);
|
|
1601
|
-
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
1602
|
-
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
1603
|
-
return download(this.name, p, candidates, ctx);
|
|
1604
|
-
}
|
|
1605
|
-
};
|
|
1606
|
-
}
|
|
1607
|
-
var tdm = tdmTier();
|
|
1608
|
-
|
|
1609
|
-
// src/tiers/unpaywall.ts
|
|
1610
|
-
var unpaywall = {
|
|
1611
|
-
name: "unpaywall",
|
|
1612
|
-
applies: (p) => p.doi ? true : "no doi",
|
|
1613
|
-
async run(p, ctx) {
|
|
1614
|
-
const mail = contactEmail();
|
|
1615
|
-
let data;
|
|
1616
|
-
try {
|
|
1617
|
-
data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
1618
|
-
} catch (e) {
|
|
1619
|
-
return miss(this.name, errMsg(e));
|
|
1620
|
-
}
|
|
1621
|
-
const candidates = unpaywallCandidates(data);
|
|
1622
|
-
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
1623
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1624
|
-
return download(this.name, p, candidates, ctx);
|
|
1616
|
+
return null;
|
|
1625
1617
|
}
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
const
|
|
1629
|
-
const
|
|
1618
|
+
const ps = d?.protocolSection;
|
|
1619
|
+
if (!ps) return null;
|
|
1620
|
+
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
1621
|
+
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
1622
|
+
const nctid = String(idm.nctId ?? "").toUpperCase();
|
|
1630
1623
|
const out = [];
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
seen.add(url);
|
|
1635
|
-
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
1636
|
-
}
|
|
1637
|
-
}
|
|
1638
|
-
return out;
|
|
1639
|
-
}
|
|
1640
|
-
|
|
1641
|
-
// src/tiers/index.ts
|
|
1642
|
-
var BUILTIN = [
|
|
1643
|
-
{ name: "pmc", make: () => pmc },
|
|
1644
|
-
{ name: "europepmc", make: () => europepmc },
|
|
1645
|
-
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
1646
|
-
{ name: "unpaywall", make: () => unpaywall },
|
|
1647
|
-
{ name: "openalex", make: () => openalex },
|
|
1648
|
-
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
1649
|
-
];
|
|
1650
|
-
|
|
1651
|
-
// src/tiers/url.ts
|
|
1652
|
-
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
1653
|
-
function urlTier(name, template, headers) {
|
|
1654
|
-
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
1655
|
-
return {
|
|
1656
|
-
name,
|
|
1657
|
-
applies(p) {
|
|
1658
|
-
const missing = used.filter((k) => !p[k]);
|
|
1659
|
-
return missing.length ? `no ${missing.join("/")}` : true;
|
|
1660
|
-
},
|
|
1661
|
-
async run(p, ctx) {
|
|
1662
|
-
if (!used.length) return miss(name, "template has no placeholder");
|
|
1663
|
-
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
1664
|
-
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
1665
|
-
}
|
|
1624
|
+
const sec = (title, lines) => {
|
|
1625
|
+
const body = lines.filter((l) => !!l && l.trim() !== "");
|
|
1626
|
+
if (body.length) out.push(`## ${title}`, ...body);
|
|
1666
1627
|
};
|
|
1667
|
-
}
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
"
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
"
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
"
|
|
1691
|
-
"
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
"
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1628
|
+
const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
|
|
1629
|
+
sec("Identification", [
|
|
1630
|
+
idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
|
|
1631
|
+
idm.acronym ? `Acronym: ${idm.acronym}` : null,
|
|
1632
|
+
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
1633
|
+
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
1634
|
+
]);
|
|
1635
|
+
sec("Status", [
|
|
1636
|
+
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
1637
|
+
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
1638
|
+
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
1639
|
+
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
1640
|
+
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
1641
|
+
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
1642
|
+
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
1643
|
+
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
1644
|
+
]);
|
|
1645
|
+
const rp = sp.responsibleParty;
|
|
1646
|
+
sec("Sponsor and collaborators", [
|
|
1647
|
+
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
1648
|
+
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
1649
|
+
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
1650
|
+
]);
|
|
1651
|
+
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
1652
|
+
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
1653
|
+
const di = dm.designInfo ?? {};
|
|
1654
|
+
const mi = di.maskingInfo ?? {};
|
|
1655
|
+
sec("Design", [
|
|
1656
|
+
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
1657
|
+
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
1658
|
+
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
1659
|
+
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
1660
|
+
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
1661
|
+
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
1662
|
+
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
1663
|
+
]);
|
|
1664
|
+
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
1665
|
+
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
1666
|
+
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
1667
|
+
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
1668
|
+
sec("Outcomes", [
|
|
1669
|
+
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
1670
|
+
...list(om.primaryOutcomes).map(outcome),
|
|
1671
|
+
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
1672
|
+
...list(om.secondaryOutcomes).map(outcome),
|
|
1673
|
+
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
1674
|
+
...list(om.otherOutcomes).map(outcome)
|
|
1675
|
+
]);
|
|
1676
|
+
sec("Eligibility", [
|
|
1677
|
+
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
1678
|
+
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
1679
|
+
]);
|
|
1680
|
+
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
1681
|
+
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
1682
|
+
const refs = list(rf.references);
|
|
1683
|
+
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
1684
|
+
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
1685
|
+
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
1699
1686
|
return {
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1687
|
+
nctid,
|
|
1688
|
+
title: para(idm.briefTitle) || nctid,
|
|
1689
|
+
acronym: idm.acronym ?? null,
|
|
1690
|
+
sponsor: sp.leadSponsor?.name ?? null,
|
|
1691
|
+
firstPosted: st.studyFirstPostDateStruct?.date ?? null,
|
|
1692
|
+
lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
|
|
1693
|
+
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
1694
|
+
hasResults: !!d.hasResults,
|
|
1695
|
+
publications,
|
|
1696
|
+
sections: out
|
|
1704
1697
|
};
|
|
1705
1698
|
}
|
|
1706
|
-
function
|
|
1707
|
-
const out =
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
const
|
|
1712
|
-
|
|
1699
|
+
function results(rs) {
|
|
1700
|
+
const out = ["## Results"];
|
|
1701
|
+
const pf = rs.participantFlowModule;
|
|
1702
|
+
if (pf) {
|
|
1703
|
+
out.push("### Participant flow");
|
|
1704
|
+
const groups = list(pf.groups);
|
|
1705
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
1706
|
+
if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
|
|
1707
|
+
for (const p of list(pf.periods)) {
|
|
1708
|
+
const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
|
|
1709
|
+
const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
|
|
1710
|
+
for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
|
|
1711
|
+
out.push(table2(head, rows));
|
|
1712
|
+
}
|
|
1713
1713
|
}
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
seen.add(name);
|
|
1729
|
-
const t = available.get(name);
|
|
1730
|
-
if (t) tiers.push(t);
|
|
1731
|
-
else if (name === "proxy") problems.push('FH_LIBRARY_TIERS: "proxy" listed, but FH_LIBRARY_PROXY_URL is unset');
|
|
1732
|
-
else problems.push(`FH_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when FH_LIBRARY_PROXY_URL is set)`);
|
|
1714
|
+
const bc = rs.baselineCharacteristicsModule;
|
|
1715
|
+
if (bc) {
|
|
1716
|
+
out.push("### Baseline characteristics");
|
|
1717
|
+
const groups = list(bc.groups);
|
|
1718
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
1719
|
+
for (const m of list(bc.measures)) {
|
|
1720
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
1721
|
+
const rows = [];
|
|
1722
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
1723
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
1724
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
1725
|
+
})]);
|
|
1726
|
+
out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
|
|
1727
|
+
}
|
|
1733
1728
|
}
|
|
1734
|
-
const
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
} catch {
|
|
1755
|
-
return false;
|
|
1729
|
+
const om = rs.outcomeMeasuresModule;
|
|
1730
|
+
if (om) {
|
|
1731
|
+
out.push("### Outcome measures");
|
|
1732
|
+
for (const m of list(om.outcomeMeasures)) {
|
|
1733
|
+
const groups = list(m.groups);
|
|
1734
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
1735
|
+
out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
|
|
1736
|
+
if (m.description) out.push(para(m.description));
|
|
1737
|
+
out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
|
|
1738
|
+
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
1739
|
+
const rows = [];
|
|
1740
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
1741
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
1742
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
1743
|
+
})]);
|
|
1744
|
+
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
1745
|
+
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
1746
|
+
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
1747
|
+
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
1748
|
+
}
|
|
1756
1749
|
}
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1750
|
+
const ae = rs.adverseEventsModule;
|
|
1751
|
+
if (ae) {
|
|
1752
|
+
out.push("### Adverse events");
|
|
1753
|
+
const groups = list(ae.eventGroups);
|
|
1754
|
+
out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
|
|
1755
|
+
const summary = [
|
|
1756
|
+
["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
|
|
1757
|
+
["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
|
|
1758
|
+
["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
|
|
1759
|
+
].filter((r) => r.slice(1).some(Boolean));
|
|
1760
|
+
out.push(table2(["", ...groups.map((g) => g.title)], summary));
|
|
1761
|
+
const events = (title, evs) => {
|
|
1762
|
+
if (!evs.length) return;
|
|
1763
|
+
out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
|
|
1764
|
+
const s = list(e.stats).find((x) => x.groupId === g.id);
|
|
1765
|
+
return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
|
|
1766
|
+
})])));
|
|
1767
|
+
};
|
|
1768
|
+
events("Serious adverse events", list(ae.seriousEvents));
|
|
1769
|
+
events("Other adverse events", list(ae.otherEvents));
|
|
1763
1770
|
}
|
|
1771
|
+
return out;
|
|
1764
1772
|
}
|
|
1765
|
-
function
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
rotateIfLarge(LOG_FILE);
|
|
1772
|
-
appendFileSync(LOG_FILE, line);
|
|
1773
|
-
} catch {
|
|
1774
|
-
}
|
|
1773
|
+
function headLineStudy(s) {
|
|
1774
|
+
return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
|
|
1775
|
+
}
|
|
1776
|
+
function renderStudyMd(s) {
|
|
1777
|
+
const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
|
|
1778
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1775
1779
|
}
|
|
1776
|
-
|
|
1777
|
-
// src/stats.ts
|
|
1778
|
-
import { mkdirSync as mkdirSync4, readdirSync as readdirSync2, readFileSync as readFileSync4, statSync as statSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
1779
|
-
import { join as join6 } from "path";
|
|
1780
1780
|
|
|
1781
1781
|
// src/entry.ts
|
|
1782
|
-
|
|
1783
|
-
import { join as join5 } from "path";
|
|
1784
|
-
import { basename, dirname } from "path";
|
|
1785
|
-
|
|
1786
|
-
// src/pubmed.ts
|
|
1787
|
-
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
1788
|
-
var CORRECTION_TYPES = {
|
|
1789
|
-
ErratumIn: "erratum_in",
|
|
1790
|
-
RetractionIn: "retraction_in",
|
|
1791
|
-
PartialRetractionIn: "partial_retraction_in",
|
|
1792
|
-
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
1793
|
-
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
1794
|
-
UpdateIn: "update_in"
|
|
1795
|
-
};
|
|
1796
|
-
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
1797
|
-
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
1798
|
-
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
1799
|
-
function efetchUrl(pmid) {
|
|
1800
|
-
const u = new URL(EFETCH);
|
|
1801
|
-
u.searchParams.set("db", "pubmed");
|
|
1802
|
-
u.searchParams.set("id", pmid);
|
|
1803
|
-
u.searchParams.set("retmode", "xml");
|
|
1804
|
-
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
1805
|
-
u.searchParams.set("tool", "fh-library");
|
|
1806
|
-
u.searchParams.set("email", contactEmail());
|
|
1807
|
-
return u.toString();
|
|
1808
|
-
}
|
|
1809
|
-
async function fetchPubmedXml(pmid) {
|
|
1810
|
-
try {
|
|
1811
|
-
return await fetchText(efetchUrl(pmid));
|
|
1812
|
-
} catch (e) {
|
|
1813
|
-
throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
1814
|
-
}
|
|
1815
|
-
}
|
|
1816
|
-
var parser2 = new XMLParser2({
|
|
1817
|
-
ignoreAttributes: false,
|
|
1818
|
-
attributeNamePrefix: "@_",
|
|
1819
|
-
parseTagValue: false,
|
|
1820
|
-
processEntities: true,
|
|
1821
|
-
stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
|
|
1822
|
-
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
1823
|
-
});
|
|
1824
|
-
function text(n) {
|
|
1825
|
-
if (n == null) return "";
|
|
1826
|
-
if (typeof n === "object") return text(n["#text"]);
|
|
1827
|
-
return String(n);
|
|
1828
|
-
}
|
|
1829
|
-
var plain = (n) => decodeEntities(text(n)).trim();
|
|
1830
|
-
function inlineText2(raw) {
|
|
1831
|
-
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
1832
|
-
}
|
|
1833
|
-
function decodeEntities(s) {
|
|
1834
|
-
return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
1835
|
-
}
|
|
1836
|
-
function parsePubmedXml(xml) {
|
|
1837
|
-
const doc = parser2.parse(xml);
|
|
1838
|
-
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
1839
|
-
if (!article) {
|
|
1840
|
-
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
1841
|
-
return book ? parseBook(book) : null;
|
|
1842
|
-
}
|
|
1843
|
-
const cit = article.MedlineCitation ?? {};
|
|
1844
|
-
const art = cit.Article ?? {};
|
|
1845
|
-
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
1846
|
-
const idOf = (t) => {
|
|
1847
|
-
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1848
|
-
return hit ? text(hit).trim() || null : null;
|
|
1849
|
-
};
|
|
1850
|
-
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
1851
|
-
const authors = authorList.map((a) => {
|
|
1852
|
-
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
1853
|
-
const last = plain(a.LastName);
|
|
1854
|
-
const ini = plain(a.Initials);
|
|
1855
|
-
return [last, ini].filter(Boolean).join(" ");
|
|
1856
|
-
}).filter(Boolean);
|
|
1857
|
-
const first = authorList[0];
|
|
1858
|
-
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
1859
|
-
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
1860
|
-
let year = text(pub.Year).trim() || null;
|
|
1861
|
-
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
1862
|
-
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
1863
|
-
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
1864
|
-
text: inlineText2(text(a))
|
|
1865
|
-
})).filter((a) => a.text);
|
|
1866
|
-
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
1867
|
-
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
1868
|
-
if (!type) return [];
|
|
1869
|
-
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
|
|
1870
|
-
});
|
|
1871
|
-
const funding = [];
|
|
1872
|
-
for (const g of art.GrantList?.Grant ?? []) {
|
|
1873
|
-
const a = plain(g.Agency);
|
|
1874
|
-
if (a && !funding.includes(a)) funding.push(a);
|
|
1875
|
-
}
|
|
1876
|
-
return {
|
|
1877
|
-
coi: inlineText2(text(cit.CoiStatement)) || null,
|
|
1878
|
-
funding,
|
|
1879
|
-
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
1880
|
-
pmcid: idOf("pmc"),
|
|
1881
|
-
doi: idOf("doi"),
|
|
1882
|
-
title: inlineText2(text(art.ArticleTitle)),
|
|
1883
|
-
authors,
|
|
1884
|
-
firstAuthor,
|
|
1885
|
-
journal: plain(art.Journal?.Title) || null,
|
|
1886
|
-
year,
|
|
1887
|
-
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
1888
|
-
abstract,
|
|
1889
|
-
corrections
|
|
1890
|
-
};
|
|
1891
|
-
}
|
|
1892
|
-
function parseBook(b) {
|
|
1893
|
-
const d = b.BookDocument ?? {};
|
|
1894
|
-
const book = d.Book ?? {};
|
|
1895
|
-
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
1896
|
-
const idOf = (t) => {
|
|
1897
|
-
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1898
|
-
return hit ? text(hit).trim() || null : null;
|
|
1899
|
-
};
|
|
1900
|
-
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
1901
|
-
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
1902
|
-
const people = pick?.Author ?? [];
|
|
1903
|
-
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
1904
|
-
const first = people[0];
|
|
1905
|
-
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
|
|
1906
|
-
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
1782
|
+
function emptyStatus(requested, type, id, libraryDir) {
|
|
1907
1783
|
return {
|
|
1908
|
-
|
|
1784
|
+
requested,
|
|
1785
|
+
type,
|
|
1786
|
+
id,
|
|
1787
|
+
entry: null,
|
|
1788
|
+
entry_dir: null,
|
|
1789
|
+
library_dir: libraryDir,
|
|
1790
|
+
pmid: null,
|
|
1791
|
+
setid: null,
|
|
1792
|
+
nctid: null,
|
|
1793
|
+
publications: [],
|
|
1909
1794
|
pmcid: null,
|
|
1910
|
-
doi:
|
|
1911
|
-
title:
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
publicationTypes: [...new Set(types)],
|
|
1917
|
-
abstract,
|
|
1795
|
+
doi: null,
|
|
1796
|
+
title: null,
|
|
1797
|
+
first_author: null,
|
|
1798
|
+
year: null,
|
|
1799
|
+
publication_types: [],
|
|
1800
|
+
retracted: false,
|
|
1918
1801
|
corrections: [],
|
|
1919
1802
|
coi: null,
|
|
1920
|
-
funding: []
|
|
1803
|
+
funding: [],
|
|
1804
|
+
citable: true,
|
|
1805
|
+
comment: null,
|
|
1806
|
+
has_abstract: null,
|
|
1807
|
+
level: null,
|
|
1808
|
+
version: null,
|
|
1809
|
+
retrieved_by: null,
|
|
1810
|
+
retrieved_date: null,
|
|
1811
|
+
pubmed_checked: null,
|
|
1812
|
+
attempts: [],
|
|
1813
|
+
files: [],
|
|
1814
|
+
glyphs_unresolved: null
|
|
1921
1815
|
};
|
|
1922
1816
|
}
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1817
|
+
var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1818
|
+
function headLine(r) {
|
|
1819
|
+
return [
|
|
1820
|
+
r.authors.join(", ") || null,
|
|
1821
|
+
r.journal,
|
|
1822
|
+
r.year,
|
|
1823
|
+
r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
|
|
1824
|
+
r.doi ? `DOI ${r.doi}` : null,
|
|
1825
|
+
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
1826
|
+
].filter(Boolean).join(" \xB7 ");
|
|
1932
1827
|
}
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
const h = findDeep2(kids2(n), tag);
|
|
1944
|
-
if (h) return h;
|
|
1945
|
-
}
|
|
1946
|
-
return void 0;
|
|
1828
|
+
function renderAbstractMd(r) {
|
|
1829
|
+
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
1830
|
+
return `# ${r.title}
|
|
1831
|
+
|
|
1832
|
+
${headLine(r)}
|
|
1833
|
+
|
|
1834
|
+
## Abstract
|
|
1835
|
+
|
|
1836
|
+
${body}
|
|
1837
|
+
`;
|
|
1947
1838
|
}
|
|
1948
|
-
function
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
}
|
|
1955
|
-
const t = tagOf2(n);
|
|
1956
|
-
const inner = () => inline(kids2(n));
|
|
1957
|
-
switch (t) {
|
|
1958
|
-
case "content": {
|
|
1959
|
-
const style = attr2(n, "styleCode").toLowerCase();
|
|
1960
|
-
const s = inner();
|
|
1961
|
-
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
1962
|
-
break;
|
|
1963
|
-
}
|
|
1964
|
-
case "sup":
|
|
1965
|
-
out += `^${inner()}^`;
|
|
1966
|
-
break;
|
|
1967
|
-
case "sub":
|
|
1968
|
-
out += `~${inner()}~`;
|
|
1969
|
-
break;
|
|
1970
|
-
case "br":
|
|
1971
|
-
out += " ";
|
|
1972
|
-
break;
|
|
1973
|
-
case "footnote":
|
|
1974
|
-
out += ` (${squash2(inner())})`;
|
|
1975
|
-
break;
|
|
1976
|
-
case "footnoteRef":
|
|
1977
|
-
case "renderMultiMedia":
|
|
1978
|
-
break;
|
|
1979
|
-
default:
|
|
1980
|
-
out += inner();
|
|
1981
|
-
}
|
|
1839
|
+
function readHeadLine(p) {
|
|
1840
|
+
try {
|
|
1841
|
+
const lines = readFileSync3(p.md, "utf8").split("\n").filter((l) => l.trim());
|
|
1842
|
+
return lines[1] ?? "";
|
|
1843
|
+
} catch {
|
|
1844
|
+
return "";
|
|
1982
1845
|
}
|
|
1983
|
-
return out;
|
|
1984
1846
|
}
|
|
1985
|
-
function
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1847
|
+
function metaFromRecord(r, files, converter) {
|
|
1848
|
+
return {
|
|
1849
|
+
pmid: r.pmid,
|
|
1850
|
+
setid: null,
|
|
1851
|
+
nctid: null,
|
|
1852
|
+
publications: [],
|
|
1853
|
+
pmcid: r.pmcid,
|
|
1854
|
+
doi: r.doi,
|
|
1855
|
+
title: r.title,
|
|
1856
|
+
first_author: r.firstAuthor,
|
|
1857
|
+
year: r.year,
|
|
1858
|
+
publication_types: r.publicationTypes,
|
|
1859
|
+
retracted: isRetracted(r),
|
|
1860
|
+
corrections: r.corrections,
|
|
1861
|
+
coi: r.coi,
|
|
1862
|
+
funding: r.funding,
|
|
1863
|
+
has_abstract: r.abstract.length > 0,
|
|
1864
|
+
level: "abstract",
|
|
1865
|
+
version: null,
|
|
1866
|
+
files,
|
|
1867
|
+
retrieved_by: "pubmed",
|
|
1868
|
+
retrieved_date: today(),
|
|
1869
|
+
converted_by: converter,
|
|
1870
|
+
pubmed_checked: today(),
|
|
1871
|
+
attempts: []
|
|
1993
1872
|
};
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
1873
|
+
}
|
|
1874
|
+
var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
|
|
1875
|
+
var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
|
|
1876
|
+
function parseCorrection(v) {
|
|
1877
|
+
const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
|
|
1878
|
+
if (!m) return null;
|
|
1879
|
+
let source;
|
|
1880
|
+
try {
|
|
1881
|
+
source = JSON.parse(m[3]);
|
|
1882
|
+
} catch {
|
|
1883
|
+
return null;
|
|
2001
1884
|
}
|
|
2002
|
-
|
|
2003
|
-
if (!rows.length) return "";
|
|
2004
|
-
const width = Math.max(...rows.map((r) => r.length));
|
|
2005
|
-
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
2006
|
-
const cap = child(n, "caption");
|
|
2007
|
-
const title = cap ? squash2(inline(kids2(cap))) : "";
|
|
2008
|
-
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
2009
|
-
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
1885
|
+
return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
|
|
2010
1886
|
}
|
|
2011
|
-
function
|
|
2012
|
-
const
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
break;
|
|
2050
|
-
case "component":
|
|
2051
|
-
case "text":
|
|
2052
|
-
case "excerpt":
|
|
2053
|
-
case "highlight":
|
|
2054
|
-
default:
|
|
2055
|
-
out.push(...blocks(c, depth));
|
|
2056
|
-
break;
|
|
2057
|
-
}
|
|
1887
|
+
function renderMeta(m) {
|
|
1888
|
+
const lines = [
|
|
1889
|
+
["pmid", m.pmid],
|
|
1890
|
+
...m.setid ? [["setid", m.setid]] : [],
|
|
1891
|
+
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
1892
|
+
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
1893
|
+
["pmcid", m.pmcid ?? ""],
|
|
1894
|
+
["doi", m.doi ?? ""],
|
|
1895
|
+
["title", JSON.stringify(m.title)],
|
|
1896
|
+
["first_author", m.first_author ?? ""],
|
|
1897
|
+
["year", m.year ?? ""],
|
|
1898
|
+
["publication_types", m.publication_types.join("; ")],
|
|
1899
|
+
["retracted", m.retracted ? "yes" : "no"],
|
|
1900
|
+
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
1901
|
+
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
1902
|
+
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
1903
|
+
// a JSON array: a statement may hold semicolons
|
|
1904
|
+
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
1905
|
+
["level", m.level],
|
|
1906
|
+
["version", m.version ?? ""],
|
|
1907
|
+
["files", m.files.join(", ")],
|
|
1908
|
+
["retrieved_by", m.retrieved_by],
|
|
1909
|
+
["retrieved_date", m.retrieved_date],
|
|
1910
|
+
["converted_by", m.converted_by],
|
|
1911
|
+
["pubmed_checked", m.pubmed_checked],
|
|
1912
|
+
["attempts", m.attempts.join(", ")]
|
|
1913
|
+
];
|
|
1914
|
+
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
1915
|
+
}
|
|
1916
|
+
function parseFrontmatter(md) {
|
|
1917
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(md);
|
|
1918
|
+
const fm = {}, all = {};
|
|
1919
|
+
for (const line of (m?.[1] ?? "").split("\n")) {
|
|
1920
|
+
const i = line.indexOf(":");
|
|
1921
|
+
if (i <= 0) continue;
|
|
1922
|
+
const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
|
|
1923
|
+
fm[k] = v;
|
|
1924
|
+
(all[k] ??= []).push(v);
|
|
2058
1925
|
}
|
|
2059
|
-
return
|
|
1926
|
+
return { fm, all };
|
|
2060
1927
|
}
|
|
2061
|
-
var
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
let out = "";
|
|
2065
|
-
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
|
|
2066
|
-
return out;
|
|
2067
|
-
};
|
|
2068
|
-
function section(n, depth) {
|
|
2069
|
-
const code = child(n, "code");
|
|
2070
|
-
const c = code ? attr2(code, "code") : "";
|
|
2071
|
-
if (SKIP_CODES.has(c)) return [];
|
|
2072
|
-
const title = child(n, "title");
|
|
2073
|
-
let head = title ? squash2(plain2(kids2(title))) : "";
|
|
2074
|
-
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
2075
|
-
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
1928
|
+
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
1929
|
+
function splitAttempts(v) {
|
|
1930
|
+
if (!v) return [];
|
|
2076
1931
|
const out = [];
|
|
2077
|
-
|
|
2078
|
-
const
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
function parseSpl(xml) {
|
|
2084
|
-
const doc = parser3.parse(xml);
|
|
2085
|
-
const d = doc.find((x) => tagOf2(x) === "document");
|
|
2086
|
-
if (!d) return null;
|
|
2087
|
-
const k = kids2(d);
|
|
2088
|
-
const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
2089
|
-
const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
|
|
2090
|
-
const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
|
|
2091
|
-
const named = (nodes) => {
|
|
2092
|
-
for (const n of nodes) {
|
|
2093
|
-
if ("#text" in n) continue;
|
|
2094
|
-
if (tagOf2(n) === "manufacturedProduct") {
|
|
2095
|
-
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
2096
|
-
if (child(inner2, "name")) return inner2;
|
|
2097
|
-
}
|
|
2098
|
-
const h = named(kids2(n));
|
|
2099
|
-
if (h) return h;
|
|
2100
|
-
}
|
|
2101
|
-
return void 0;
|
|
2102
|
-
};
|
|
2103
|
-
const inner = named(k);
|
|
2104
|
-
const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
2105
|
-
const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
|
|
2106
|
-
const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
2107
|
-
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
2108
|
-
const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
|
|
2109
|
-
const org = findDeep2(k, "representedOrganization");
|
|
2110
|
-
const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
|
|
2111
|
-
const body = findDeep2(k, "structuredBody");
|
|
2112
|
-
const sections2 = body ? blocks(kids2(body), 2) : [];
|
|
2113
|
-
const docTitle = child(d, "title");
|
|
2114
|
-
const lines = [];
|
|
2115
|
-
let cur = "";
|
|
2116
|
-
for (const x of docTitle ? kids2(docTitle) : []) {
|
|
2117
|
-
if (!("#text" in x) && tagOf2(x) === "br") {
|
|
2118
|
-
lines.push(cur);
|
|
1932
|
+
let cur = "", depth = 0;
|
|
1933
|
+
for (const ch of v) {
|
|
1934
|
+
if (ch === "(") depth++;
|
|
1935
|
+
else if (ch === ")") depth = Math.max(0, depth - 1);
|
|
1936
|
+
if (ch === "," && depth === 0) {
|
|
1937
|
+
out.push(cur);
|
|
2119
1938
|
cur = "";
|
|
2120
|
-
} else cur +=
|
|
1939
|
+
} else cur += ch;
|
|
2121
1940
|
}
|
|
2122
|
-
|
|
2123
|
-
const
|
|
2124
|
-
const
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
1941
|
+
out.push(cur);
|
|
1942
|
+
const merged = [];
|
|
1943
|
+
for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
|
|
1944
|
+
if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
|
|
1945
|
+
else merged.push(piece);
|
|
1946
|
+
}
|
|
1947
|
+
return merged;
|
|
2128
1948
|
}
|
|
2129
|
-
|
|
2130
|
-
|
|
1949
|
+
var opt = (v) => v ? v : null;
|
|
1950
|
+
var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
|
|
1951
|
+
function splitPublicationTypes(v) {
|
|
1952
|
+
if (!v) return [];
|
|
1953
|
+
if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
|
|
1954
|
+
const out = [];
|
|
1955
|
+
for (const piece of list2(v)) {
|
|
1956
|
+
if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
|
|
1957
|
+
else out.push(piece);
|
|
1958
|
+
}
|
|
1959
|
+
return out;
|
|
2131
1960
|
}
|
|
2132
|
-
function
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
1961
|
+
function abstractMissing(p) {
|
|
1962
|
+
try {
|
|
1963
|
+
return readFileSync3(p.md, "utf8").includes("_No abstract available._");
|
|
1964
|
+
} catch {
|
|
1965
|
+
return false;
|
|
1966
|
+
}
|
|
2137
1967
|
}
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
|
|
2145
|
-
return json2;
|
|
1968
|
+
function correctionsFromXml(p) {
|
|
1969
|
+
try {
|
|
1970
|
+
return parsePubmedXml(readFileSync3(p.xml, "utf8"))?.corrections ?? [];
|
|
1971
|
+
} catch {
|
|
1972
|
+
return [];
|
|
1973
|
+
}
|
|
2146
1974
|
}
|
|
2147
|
-
|
|
2148
|
-
NOT_YET_RECRUITING: "Not yet recruiting",
|
|
2149
|
-
RECRUITING: "Recruiting",
|
|
2150
|
-
ENROLLING_BY_INVITATION: "Enrolling by invitation",
|
|
2151
|
-
ACTIVE_NOT_RECRUITING: "Active, not recruiting",
|
|
2152
|
-
SUSPENDED: "Suspended",
|
|
2153
|
-
TERMINATED: "Terminated",
|
|
2154
|
-
COMPLETED: "Completed",
|
|
2155
|
-
WITHDRAWN: "Withdrawn",
|
|
2156
|
-
UNKNOWN: "Unknown status",
|
|
2157
|
-
AVAILABLE: "Available",
|
|
2158
|
-
NO_LONGER_AVAILABLE: "No longer available",
|
|
2159
|
-
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
2160
|
-
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
2161
|
-
WITHHELD: "Withheld"
|
|
2162
|
-
};
|
|
2163
|
-
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
2164
|
-
var words = (v) => {
|
|
2165
|
-
const s = String(v ?? "");
|
|
2166
|
-
if (!s) return "";
|
|
2167
|
-
if (STATUS[s]) return STATUS[s];
|
|
2168
|
-
if (PHASE[s]) return PHASE[s];
|
|
2169
|
-
const w = s.toLowerCase().replace(/_/g, " ");
|
|
2170
|
-
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
2171
|
-
};
|
|
2172
|
-
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
2173
|
-
var list = (a) => Array.isArray(a) ? a : [];
|
|
2174
|
-
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
2175
|
-
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
2176
|
-
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
2177
|
-
function parseStudy(json2) {
|
|
2178
|
-
let d;
|
|
1975
|
+
function declarationsFromXml(p) {
|
|
2179
1976
|
try {
|
|
2180
|
-
|
|
1977
|
+
const r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
1978
|
+
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
2181
1979
|
} catch {
|
|
2182
|
-
return null;
|
|
1980
|
+
return { coi: null, funding: [] };
|
|
2183
1981
|
}
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
const
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
1982
|
+
}
|
|
1983
|
+
function readMeta(p) {
|
|
1984
|
+
const { fm, all } = parseFrontmatter(readFileSync3(p.meta, "utf8"));
|
|
1985
|
+
let title = fm.title ?? "";
|
|
1986
|
+
if (title.startsWith('"')) {
|
|
1987
|
+
try {
|
|
1988
|
+
title = JSON.parse(title);
|
|
1989
|
+
} catch {
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
const legacy = fm.pubmed_checked === void 0;
|
|
1993
|
+
const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
|
|
1994
|
+
let coi = null, funding = [];
|
|
1995
|
+
if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
|
|
1996
|
+
else {
|
|
1997
|
+
if (fm.coi.startsWith('"')) {
|
|
1998
|
+
try {
|
|
1999
|
+
coi = JSON.parse(fm.coi);
|
|
2000
|
+
} catch {
|
|
2001
|
+
coi = fm.coi;
|
|
2002
|
+
}
|
|
2003
|
+
} else coi = fm.coi || null;
|
|
2004
|
+
funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
|
|
2005
|
+
}
|
|
2006
|
+
return {
|
|
2007
|
+
pmid: fm.pmid ?? "",
|
|
2008
|
+
setid: opt(fm.setid),
|
|
2009
|
+
nctid: opt(fm.nctid),
|
|
2010
|
+
publications: list2(fm.publications),
|
|
2011
|
+
pmcid: opt(fm.pmcid),
|
|
2012
|
+
doi: opt(fm.doi),
|
|
2013
|
+
title,
|
|
2014
|
+
first_author: opt(fm.first_author),
|
|
2015
|
+
year: opt(fm.year),
|
|
2016
|
+
publication_types: splitPublicationTypes(fm.publication_types),
|
|
2017
|
+
retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
|
|
2018
|
+
corrections,
|
|
2019
|
+
coi,
|
|
2020
|
+
funding,
|
|
2021
|
+
has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
|
|
2022
|
+
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
2023
|
+
version: opt(fm.version),
|
|
2024
|
+
files: list2(fm.files),
|
|
2025
|
+
retrieved_by: fm.retrieved_by ?? "",
|
|
2026
|
+
retrieved_date: fm.retrieved_date ?? "",
|
|
2027
|
+
converted_by: fm.converted_by ?? "",
|
|
2028
|
+
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
2029
|
+
attempts: splitAttempts(fm.attempts)
|
|
2193
2030
|
};
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
const
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
const
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
2245
|
-
]);
|
|
2246
|
-
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
2247
|
-
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
2248
|
-
const refs = list(rf.references);
|
|
2249
|
-
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
2250
|
-
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
2251
|
-
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
2031
|
+
}
|
|
2032
|
+
function writeAtomic(file3, data) {
|
|
2033
|
+
const tmp = `${file3}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2034
|
+
writeFileSync(tmp, data);
|
|
2035
|
+
renameSync(tmp, file3);
|
|
2036
|
+
}
|
|
2037
|
+
function writeMeta(p, m) {
|
|
2038
|
+
writeAtomic(p.meta, renderMeta(m));
|
|
2039
|
+
try {
|
|
2040
|
+
unlinkSync(join4(p.entryDir, `${p.entry}-missing.md`));
|
|
2041
|
+
} catch {
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
function copyEntry(from, to) {
|
|
2045
|
+
const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
|
|
2046
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
2047
|
+
mkdirSync2(tmp, { recursive: true });
|
|
2048
|
+
const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
|
|
2049
|
+
for (const n of readdirSync(from.entryDir)) if (join4(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join4(from.entryDir, n), join4(tmp, rename(n)));
|
|
2050
|
+
const m = readMeta(from);
|
|
2051
|
+
const copy = { ...m, files: m.files.map(rename) };
|
|
2052
|
+
writeFileSync(join4(tmp, basename(to.meta)), renderMeta(copy));
|
|
2053
|
+
mkdirSync2(dirname(to.entryDir), { recursive: true });
|
|
2054
|
+
if (existsSync4(to.entryDir)) renameSync(to.entryDir, old);
|
|
2055
|
+
renameSync(tmp, to.entryDir);
|
|
2056
|
+
rmSync(old, { recursive: true, force: true });
|
|
2057
|
+
return copy;
|
|
2058
|
+
}
|
|
2059
|
+
var NOTHING_TO_VERIFY = "no abstract and no full text: nothing to verify";
|
|
2060
|
+
function citability(m) {
|
|
2061
|
+
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
2062
|
+
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
2063
|
+
if (m.retracted) {
|
|
2064
|
+
const r = m.corrections.filter(isRetractionNotice);
|
|
2065
|
+
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
2066
|
+
}
|
|
2067
|
+
if (m.level !== "full_text" && !m.has_abstract) return { citable: false, comment: NOTHING_TO_VERIFY };
|
|
2068
|
+
const notes = [
|
|
2069
|
+
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
2070
|
+
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
2071
|
+
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
2072
|
+
];
|
|
2073
|
+
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
2074
|
+
}
|
|
2075
|
+
function statusFromMeta(p, m, base) {
|
|
2076
|
+
const full = existsSync4(p.fulltextMd) ? readFileSync3(p.fulltextMd, "utf8") : null;
|
|
2077
|
+
const cit = citability(m);
|
|
2078
|
+
if (full?.includes(`
|
|
2079
|
+
${CORRECTED_HEADING}
|
|
2080
|
+
`) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
|
|
2252
2081
|
return {
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2082
|
+
...base,
|
|
2083
|
+
entry: p.entry,
|
|
2084
|
+
entry_dir: p.entryDir,
|
|
2085
|
+
pmid: m.pmid || null,
|
|
2086
|
+
setid: m.setid,
|
|
2087
|
+
nctid: m.nctid,
|
|
2088
|
+
publications: m.publications,
|
|
2089
|
+
pmcid: m.pmcid,
|
|
2090
|
+
doi: m.doi,
|
|
2091
|
+
title: m.title || null,
|
|
2092
|
+
first_author: m.first_author,
|
|
2093
|
+
year: m.year,
|
|
2094
|
+
publication_types: m.publication_types,
|
|
2095
|
+
retracted: m.retracted,
|
|
2096
|
+
corrections: m.corrections,
|
|
2097
|
+
coi: m.coi,
|
|
2098
|
+
funding: m.funding,
|
|
2099
|
+
...cit,
|
|
2100
|
+
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
2101
|
+
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
2102
|
+
// there is abstract text).
|
|
2103
|
+
has_abstract: m.has_abstract,
|
|
2104
|
+
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
2105
|
+
version: m.version,
|
|
2106
|
+
retrieved_by: m.retrieved_by || null,
|
|
2107
|
+
retrieved_date: m.retrieved_date || null,
|
|
2108
|
+
pubmed_checked: m.pubmed_checked || null,
|
|
2109
|
+
attempts: m.attempts,
|
|
2110
|
+
files: m.files,
|
|
2111
|
+
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
2263
2112
|
};
|
|
2264
2113
|
}
|
|
2265
|
-
function
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2114
|
+
function readStatus(p, base) {
|
|
2115
|
+
return statusFromMeta(p, readMeta(p), base);
|
|
2116
|
+
}
|
|
2117
|
+
function entryExists(p) {
|
|
2118
|
+
return existsSync4(p.meta);
|
|
2119
|
+
}
|
|
2120
|
+
function writeEntry(p, r, xml, converter) {
|
|
2121
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2122
|
+
writeAtomic(p.xml, xml);
|
|
2123
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2124
|
+
const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
|
|
2125
|
+
writeMeta(p, meta);
|
|
2126
|
+
return meta;
|
|
2127
|
+
}
|
|
2128
|
+
function refreshRecord(p, m, r, xml) {
|
|
2129
|
+
writeAtomic(p.xml, xml);
|
|
2130
|
+
syncCorrectedBlock(p);
|
|
2131
|
+
const next = {
|
|
2132
|
+
...m,
|
|
2133
|
+
pmcid: r.pmcid ?? m.pmcid,
|
|
2134
|
+
doi: r.doi ?? m.doi,
|
|
2135
|
+
publication_types: r.publicationTypes,
|
|
2136
|
+
retracted: isRetracted(r),
|
|
2137
|
+
corrections: r.corrections,
|
|
2138
|
+
coi: r.coi ?? m.coi,
|
|
2139
|
+
funding: r.funding.length ? r.funding : m.funding,
|
|
2140
|
+
// the paper's own statements stay when PubMed has none
|
|
2141
|
+
pubmed_checked: today()
|
|
2142
|
+
};
|
|
2143
|
+
writeMeta(p, next);
|
|
2144
|
+
return next;
|
|
2145
|
+
}
|
|
2146
|
+
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
2147
|
+
const t = new Date(date).getTime();
|
|
2148
|
+
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
2149
|
+
}
|
|
2150
|
+
var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2151
|
+
var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2152
|
+
function declarationsFromMd(md) {
|
|
2153
|
+
let coi = null;
|
|
2154
|
+
const funding = [];
|
|
2155
|
+
const lines = md.split("\n");
|
|
2156
|
+
let section2 = "";
|
|
2157
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2158
|
+
const l = lines[i].trim();
|
|
2159
|
+
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
2160
|
+
if (h) {
|
|
2161
|
+
section2 = h[1];
|
|
2162
|
+
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
2163
|
+
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
2164
|
+
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
2165
|
+
continue;
|
|
2293
2166
|
}
|
|
2167
|
+
if (section2 !== "Declarations" || !l) continue;
|
|
2168
|
+
const f = FUNDING_RE.exec(l);
|
|
2169
|
+
const c = COI_RE.exec(l);
|
|
2170
|
+
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
2171
|
+
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
2294
2172
|
}
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
out.push(
|
|
2304
|
-
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
2305
|
-
const rows = [];
|
|
2306
|
-
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
2307
|
-
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
2308
|
-
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
2309
|
-
})]);
|
|
2310
|
-
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
2311
|
-
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
2312
|
-
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
2313
|
-
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
2173
|
+
return { coi, funding };
|
|
2174
|
+
}
|
|
2175
|
+
var CORRECTED_HEADING = "## Corrected in PubMed abstract";
|
|
2176
|
+
function correctedSentences(r) {
|
|
2177
|
+
const out = [];
|
|
2178
|
+
for (const a of r.abstract) {
|
|
2179
|
+
for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
|
|
2180
|
+
const i = s.indexOf("[corrected]");
|
|
2181
|
+
if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
|
|
2314
2182
|
}
|
|
2315
2183
|
}
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2184
|
+
return out.filter(Boolean);
|
|
2185
|
+
}
|
|
2186
|
+
function withCorrectedBlock(md, sentences) {
|
|
2187
|
+
const base = md.split(`
|
|
2188
|
+
${CORRECTED_HEADING}
|
|
2189
|
+
`)[0].trimEnd();
|
|
2190
|
+
if (!sentences.length) return base + "\n";
|
|
2191
|
+
return `${base}
|
|
2192
|
+
|
|
2193
|
+
${CORRECTED_HEADING}
|
|
2194
|
+
|
|
2195
|
+
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
2196
|
+
|
|
2197
|
+
${sentences.join("\n\n")}
|
|
2198
|
+
`;
|
|
2199
|
+
}
|
|
2200
|
+
function syncCorrectedBlock(p) {
|
|
2201
|
+
if (!existsSync4(p.fulltextMd) || !existsSync4(p.xml)) return;
|
|
2202
|
+
let r;
|
|
2203
|
+
try {
|
|
2204
|
+
r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
2205
|
+
} catch {
|
|
2206
|
+
return;
|
|
2336
2207
|
}
|
|
2337
|
-
return
|
|
2208
|
+
if (!r) return;
|
|
2209
|
+
const md = readFileSync3(p.fulltextMd, "utf8");
|
|
2210
|
+
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
2211
|
+
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
2338
2212
|
}
|
|
2339
|
-
function
|
|
2340
|
-
|
|
2213
|
+
function applyFulltext(p, m, hit, converter) {
|
|
2214
|
+
const raw = p.fulltextRaw(hit.format);
|
|
2215
|
+
writeAtomic(raw, hit.raw);
|
|
2216
|
+
writeAtomic(p.fulltextMd, hit.md);
|
|
2217
|
+
syncCorrectedBlock(p);
|
|
2218
|
+
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
2219
|
+
const own = declarationsFromMd(hit.md);
|
|
2220
|
+
const next = {
|
|
2221
|
+
...m,
|
|
2222
|
+
level: "full_text",
|
|
2223
|
+
version: hit.version,
|
|
2224
|
+
files,
|
|
2225
|
+
retrieved_by: hit.tier,
|
|
2226
|
+
retrieved_date: today(),
|
|
2227
|
+
converted_by: converter,
|
|
2228
|
+
coi: m.coi ?? own.coi,
|
|
2229
|
+
funding: m.funding.length ? m.funding : own.funding
|
|
2230
|
+
};
|
|
2231
|
+
writeMeta(p, next);
|
|
2232
|
+
return next;
|
|
2341
2233
|
}
|
|
2342
|
-
function
|
|
2343
|
-
|
|
2344
|
-
|
|
2234
|
+
function recordAttempts(p, m, labels) {
|
|
2235
|
+
if (!labels.length) return m;
|
|
2236
|
+
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
2237
|
+
const replaced = new Set(labels.map(tierOf));
|
|
2238
|
+
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
2239
|
+
writeMeta(p, next);
|
|
2240
|
+
return next;
|
|
2345
2241
|
}
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2242
|
+
function writeLabelEntry(p, s, xml, converter) {
|
|
2243
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2244
|
+
const raw = p.fulltextRaw("xml");
|
|
2245
|
+
writeAtomic(raw, xml);
|
|
2246
|
+
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
2247
|
+
const meta = {
|
|
2248
|
+
pmid: "",
|
|
2249
|
+
setid: s.setid,
|
|
2250
|
+
nctid: null,
|
|
2251
|
+
publications: [],
|
|
2252
|
+
pmcid: null,
|
|
2253
|
+
doi: null,
|
|
2254
|
+
title: s.title,
|
|
2255
|
+
first_author: s.labeler,
|
|
2256
|
+
year: s.effective?.slice(0, 4) ?? null,
|
|
2257
|
+
publication_types: ["Drug Label"],
|
|
2258
|
+
retracted: false,
|
|
2259
|
+
corrections: [],
|
|
2260
|
+
coi: null,
|
|
2261
|
+
funding: [],
|
|
2262
|
+
has_abstract: false,
|
|
2263
|
+
level: "full_text",
|
|
2264
|
+
version: s.version ? `spl ${s.version}` : null,
|
|
2265
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2266
|
+
retrieved_by: "dailymed",
|
|
2267
|
+
retrieved_date: today(),
|
|
2268
|
+
converted_by: converter,
|
|
2269
|
+
pubmed_checked: today(),
|
|
2270
|
+
attempts: []
|
|
2271
|
+
};
|
|
2272
|
+
writeMeta(p, meta);
|
|
2273
|
+
return meta;
|
|
2274
|
+
}
|
|
2275
|
+
function writeStudyEntry(p, s, json2, converter) {
|
|
2276
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2277
|
+
const raw = p.fulltextRaw("json");
|
|
2278
|
+
writeAtomic(raw, json2);
|
|
2279
|
+
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
2280
|
+
const meta = {
|
|
2281
|
+
pmid: "",
|
|
2357
2282
|
setid: null,
|
|
2358
|
-
nctid:
|
|
2359
|
-
publications:
|
|
2283
|
+
nctid: s.nctid,
|
|
2284
|
+
publications: s.publications,
|
|
2360
2285
|
pmcid: null,
|
|
2361
2286
|
doi: null,
|
|
2362
|
-
title:
|
|
2363
|
-
first_author:
|
|
2364
|
-
year: null,
|
|
2365
|
-
publication_types: [],
|
|
2287
|
+
title: s.title,
|
|
2288
|
+
first_author: s.sponsor,
|
|
2289
|
+
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
2290
|
+
publication_types: ["Trial Registration"],
|
|
2366
2291
|
retracted: false,
|
|
2367
2292
|
corrections: [],
|
|
2368
2293
|
coi: null,
|
|
2369
2294
|
funding: [],
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
pubmed_checked:
|
|
2378
|
-
attempts: []
|
|
2379
|
-
files: [],
|
|
2380
|
-
glyphs_unresolved: null
|
|
2295
|
+
has_abstract: false,
|
|
2296
|
+
level: "full_text",
|
|
2297
|
+
version: s.lastUpdate,
|
|
2298
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2299
|
+
retrieved_by: "ctgov",
|
|
2300
|
+
retrieved_date: today(),
|
|
2301
|
+
converted_by: converter,
|
|
2302
|
+
pubmed_checked: today(),
|
|
2303
|
+
attempts: []
|
|
2381
2304
|
};
|
|
2305
|
+
writeMeta(p, meta);
|
|
2306
|
+
return meta;
|
|
2382
2307
|
}
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
2392
|
-
].filter(Boolean).join(" \xB7 ");
|
|
2308
|
+
function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
|
|
2309
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2310
|
+
const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
|
|
2311
|
+
writeAtomic(raw, json2);
|
|
2312
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2313
|
+
const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
|
|
2314
|
+
writeMeta(p, meta);
|
|
2315
|
+
return meta;
|
|
2393
2316
|
}
|
|
2394
|
-
function renderAbstractMd(r) {
|
|
2395
|
-
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
2396
|
-
return `# ${r.title}
|
|
2397
2317
|
|
|
2398
|
-
|
|
2318
|
+
// src/walls.ts
|
|
2319
|
+
var WALL2 = /^(challenge not cleared|empty response)/i;
|
|
2320
|
+
var WALL_STREAK = 2;
|
|
2321
|
+
var host = (url) => {
|
|
2322
|
+
try {
|
|
2323
|
+
return new URL(url).hostname.replace(/^www\./, "").toLowerCase();
|
|
2324
|
+
} catch {
|
|
2325
|
+
return "";
|
|
2326
|
+
}
|
|
2327
|
+
};
|
|
2328
|
+
var file = (dir, h) => join5(dir, "walls", `${h.replace(/[^a-z0-9.-]/g, "_")}.json`);
|
|
2329
|
+
function read(dir, h) {
|
|
2330
|
+
try {
|
|
2331
|
+
return JSON.parse(readFileSync4(file(dir, h), "utf8"));
|
|
2332
|
+
} catch {
|
|
2333
|
+
return {};
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
function walled(url, rung2, dir = FH_LIBRARY_DIR) {
|
|
2337
|
+
const h = host(url);
|
|
2338
|
+
if (!h) return null;
|
|
2339
|
+
const r = read(dir, h)[rung2];
|
|
2340
|
+
return r && r.streak >= WALL_STREAK && daysSince(r.date) < settings().retry_after_days ? r : null;
|
|
2341
|
+
}
|
|
2342
|
+
function recordRung(url, rung2, ok, reason = "", dir = FH_LIBRARY_DIR) {
|
|
2343
|
+
const h = host(url);
|
|
2344
|
+
if (!h) return;
|
|
2345
|
+
const why = reason.replace(/^(browser [^:]+|brightdata): /, "");
|
|
2346
|
+
if (!ok && !WALL2.test(why)) return;
|
|
2347
|
+
const all = read(dir, h);
|
|
2348
|
+
if (ok) {
|
|
2349
|
+
if (!all[rung2]) return;
|
|
2350
|
+
delete all[rung2];
|
|
2351
|
+
} else all[rung2] = { streak: (all[rung2]?.streak ?? 0) + 1, date: today(), reason: why };
|
|
2352
|
+
try {
|
|
2353
|
+
mkdirSync3(join5(dir, "walls"), { recursive: true });
|
|
2354
|
+
writeAtomic(file(dir, h), JSON.stringify(all) + "\n");
|
|
2355
|
+
} catch {
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
var wallNote = (r) => `${r.reason} ${r.streak}\xD7 on this host, last ${r.date}`;
|
|
2399
2359
|
|
|
2400
|
-
|
|
2360
|
+
// src/tiers/tier.ts
|
|
2361
|
+
var looked = (t, p, ctx) => ctx.lookups?.get(t.name) ?? t.lookup(p);
|
|
2362
|
+
function oaVersion(v) {
|
|
2363
|
+
if (v === "acceptedVersion") return "accepted_manuscript";
|
|
2364
|
+
if (v === "submittedVersion") return "preprint";
|
|
2365
|
+
return "published";
|
|
2366
|
+
}
|
|
2367
|
+
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
2368
|
+
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
2401
2369
|
|
|
2402
|
-
|
|
2403
|
-
|
|
2370
|
+
// src/download.ts
|
|
2371
|
+
var hostOf2 = (url) => {
|
|
2372
|
+
try {
|
|
2373
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
2374
|
+
} catch {
|
|
2375
|
+
return url;
|
|
2376
|
+
}
|
|
2377
|
+
};
|
|
2378
|
+
var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
|
|
2379
|
+
function order(candidates) {
|
|
2380
|
+
const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
|
|
2381
|
+
return [...candidates].sort((a, b) => rank(a) - rank(b));
|
|
2382
|
+
}
|
|
2383
|
+
async function download(tier, p, candidates, ctx) {
|
|
2384
|
+
const usable = candidates.filter((c) => !NCBI.test(c.url));
|
|
2385
|
+
if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
|
|
2386
|
+
const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
|
|
2387
|
+
if (!ordered.length) return miss(tier, "only copies already tried");
|
|
2388
|
+
const reasons = [];
|
|
2389
|
+
for (const c of ordered) {
|
|
2390
|
+
ctx.tried.add(c.url);
|
|
2391
|
+
let r = await tryOne(p, c, tier);
|
|
2392
|
+
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
2393
|
+
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
2394
|
+
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
2395
|
+
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
2396
|
+
for (const b of browserRungs()) {
|
|
2397
|
+
if (r.ok || c.headers || c.service || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
2398
|
+
const w = walled(target.url, `browser ${b.label}`);
|
|
2399
|
+
if (w) {
|
|
2400
|
+
ctx.onTry?.(`${target.url} skipped browser ${b.label}: ${wallNote(w)}`);
|
|
2401
|
+
continue;
|
|
2402
|
+
}
|
|
2403
|
+
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
2404
|
+
recordRung(target.url, `browser ${b.label}`, r.ok, r.ok ? "" : r.reason);
|
|
2405
|
+
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
2406
|
+
if (!r.ok && bare(r.reason).startsWith(MISMATCH)) ctx.onAlert?.(`browser ${b.label}: ${bare(r.reason)}`);
|
|
2407
|
+
}
|
|
2408
|
+
const uw = walled(target.url, "brightdata");
|
|
2409
|
+
if (uw && !r.ok && !c.headers && !c.service && unlockerConfigured()) ctx.onTry?.(`${target.url} skipped brightdata: ${wallNote(uw)}`);
|
|
2410
|
+
else if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
2411
|
+
r = await tryOne(p, target, tier, "unlocker");
|
|
2412
|
+
recordRung(target.url, "brightdata", r.ok, r.ok ? "" : r.reason);
|
|
2413
|
+
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
2414
|
+
if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
|
|
2415
|
+
}
|
|
2416
|
+
if (r.ok) return r;
|
|
2417
|
+
reasons.push(`${r.reason}: ${hostOf2(c.url)}`);
|
|
2418
|
+
}
|
|
2419
|
+
return miss(tier, reasons.join("; "));
|
|
2420
|
+
}
|
|
2421
|
+
var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
|
|
2422
|
+
async function tryOne(p, c, tier, via = "plain") {
|
|
2423
|
+
let got;
|
|
2424
|
+
try {
|
|
2425
|
+
got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
|
|
2426
|
+
} catch (e) {
|
|
2427
|
+
const body = e?.body;
|
|
2428
|
+
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
2429
|
+
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
2430
|
+
}
|
|
2431
|
+
const { bytes, contentType } = got;
|
|
2432
|
+
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
2433
|
+
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
2434
|
+
const r = await convertBytes(p, bytes, contentType);
|
|
2435
|
+
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
2436
|
+
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
2437
|
+
}
|
|
2438
|
+
var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
|
|
2439
|
+
function serviceMessage(bytes, contentType = "") {
|
|
2440
|
+
const text3 = bytes.toString("utf8").trim();
|
|
2441
|
+
if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
|
|
2442
|
+
try {
|
|
2443
|
+
const j = JSON.parse(text3);
|
|
2444
|
+
const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
|
|
2445
|
+
if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
|
|
2446
|
+
} catch {
|
|
2447
|
+
}
|
|
2448
|
+
const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
2449
|
+
return (visible || text3).slice(0, 160);
|
|
2450
|
+
}
|
|
2451
|
+
function describe(reason, format, md) {
|
|
2452
|
+
const words2 = md.split(/\s+/).filter(Boolean).length;
|
|
2453
|
+
const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
|
|
2454
|
+
const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
|
|
2455
|
+
if (reason === "no methods/results") return `no article body (${seen})`;
|
|
2456
|
+
if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
|
|
2457
|
+
return reason;
|
|
2458
|
+
}
|
|
2459
|
+
async function convertBytes(p, bytes, contentType = "") {
|
|
2460
|
+
let format;
|
|
2461
|
+
let md;
|
|
2462
|
+
if (looksLikePdf(bytes)) {
|
|
2463
|
+
format = "pdf";
|
|
2464
|
+
const stats2 = {};
|
|
2465
|
+
try {
|
|
2466
|
+
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
|
|
2467
|
+
} catch (e) {
|
|
2468
|
+
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
2469
|
+
}
|
|
2470
|
+
if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
2471
|
+
} else {
|
|
2472
|
+
const text3 = bytes.toString("utf8");
|
|
2473
|
+
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
2474
|
+
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
2475
|
+
format = "xml";
|
|
2476
|
+
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
2477
|
+
if (!doc.title) doc.title = p.title;
|
|
2478
|
+
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
2479
|
+
md = renderFulltextMd(doc, p.headLine);
|
|
2480
|
+
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
2481
|
+
format = "html";
|
|
2482
|
+
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
2483
|
+
md = `# ${p.title}
|
|
2484
|
+
|
|
2485
|
+
${p.headLine}
|
|
2486
|
+
|
|
2487
|
+
${htmlToMarkdown(body)}`;
|
|
2488
|
+
} else {
|
|
2489
|
+
return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
|
|
2493
|
+
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
2494
|
+
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
2495
|
+
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
2496
|
+
return { ok: true, format, md };
|
|
2497
|
+
}
|
|
2498
|
+
var ELSEVIER_TAGS = [
|
|
2499
|
+
[/ja:article\b/g, "article"],
|
|
2500
|
+
[/ja:body\b/g, "body"],
|
|
2501
|
+
[/ja:head\b/g, "front"],
|
|
2502
|
+
[/ce:sections\b/g, "sec-group"],
|
|
2503
|
+
[/ce:section-title\b/g, "title"],
|
|
2504
|
+
[/ce:section\b/g, "sec"],
|
|
2505
|
+
[/ce:simple-para\b/g, "p"],
|
|
2506
|
+
[/ce:para\b/g, "p"],
|
|
2507
|
+
[/ce:abstract-sec\b/g, "sec"],
|
|
2508
|
+
[/ce:abstract\b/g, "abstract"],
|
|
2509
|
+
[/ce:title\b/g, "article-title"],
|
|
2510
|
+
[/ce:italic\b/g, "italic"],
|
|
2511
|
+
[/ce:bold\b/g, "bold"],
|
|
2512
|
+
[/ce:sup\b/g, "sup"],
|
|
2513
|
+
[/ce:inf\b/g, "sub"],
|
|
2514
|
+
[/ce:cross-refs?\b/g, "xref"],
|
|
2515
|
+
[/ce:list-item\b/g, "list-item"],
|
|
2516
|
+
[/ce:list\b/g, "list"],
|
|
2517
|
+
[/ce:label\b/g, "label"],
|
|
2518
|
+
[/ce:caption\b/g, "caption"],
|
|
2519
|
+
[/ce:table\b/g, "table-wrap"],
|
|
2520
|
+
[/ce:figure\b/g, "fig"],
|
|
2521
|
+
[/ce:bib-reference\b/g, "ref"],
|
|
2522
|
+
[/ce:bibliography\b/g, "ref-list"],
|
|
2523
|
+
[/sb:reference\b/g, "mixed-citation"],
|
|
2524
|
+
[/ce:acknowledgment\b/g, "ack"],
|
|
2525
|
+
[/ce:inter-ref\b/g, "ext-link"]
|
|
2526
|
+
];
|
|
2527
|
+
function elsevierToJats(xml) {
|
|
2528
|
+
let out = xml;
|
|
2529
|
+
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
2530
|
+
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
2531
|
+
}
|
|
2532
|
+
|
|
2533
|
+
// src/tiers/core.ts
|
|
2534
|
+
var WINDOW_MS = 6e4;
|
|
2535
|
+
var PER_WINDOW = 10;
|
|
2536
|
+
var MAX_WAIT_MS = 65e3;
|
|
2537
|
+
var TIMEOUT_MS3 = 1e4;
|
|
2538
|
+
var calls = [];
|
|
2539
|
+
async function pace() {
|
|
2540
|
+
const now = Date.now();
|
|
2541
|
+
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
2542
|
+
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
2543
|
+
calls.push(Date.now());
|
|
2404
2544
|
}
|
|
2405
|
-
function
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
return
|
|
2545
|
+
async function coreFetch(url, key) {
|
|
2546
|
+
const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
|
|
2547
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
2548
|
+
await pace();
|
|
2549
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
|
|
2550
|
+
if (res.ok) return await res.text();
|
|
2551
|
+
const body = await res.text().catch(() => "");
|
|
2552
|
+
if (res.status === 429 && attempt === 0) {
|
|
2553
|
+
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
2554
|
+
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
2555
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
2556
|
+
continue;
|
|
2557
|
+
}
|
|
2558
|
+
throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
|
|
2411
2559
|
}
|
|
2560
|
+
throw new Error("HTTP 429");
|
|
2412
2561
|
}
|
|
2413
|
-
function
|
|
2562
|
+
function coreTier(apiKey) {
|
|
2563
|
+
const key = apiKey?.trim();
|
|
2414
2564
|
return {
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
version: null,
|
|
2432
|
-
files,
|
|
2433
|
-
retrieved_by: "pubmed",
|
|
2434
|
-
retrieved_date: today(),
|
|
2435
|
-
converted_by: converter,
|
|
2436
|
-
pubmed_checked: today(),
|
|
2437
|
-
attempts: []
|
|
2565
|
+
name: "core",
|
|
2566
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2567
|
+
async run(p, ctx) {
|
|
2568
|
+
if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
|
|
2569
|
+
let data;
|
|
2570
|
+
try {
|
|
2571
|
+
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
2572
|
+
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
2573
|
+
} catch (e) {
|
|
2574
|
+
return miss(this.name, errMsg(e));
|
|
2575
|
+
}
|
|
2576
|
+
const candidates = coreCandidates(data);
|
|
2577
|
+
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
2578
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2579
|
+
return download(this.name, p, candidates, ctx);
|
|
2580
|
+
}
|
|
2438
2581
|
};
|
|
2439
2582
|
}
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
const
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2583
|
+
function coreCandidates(d) {
|
|
2584
|
+
const out = [];
|
|
2585
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2586
|
+
const add = (url, pdf, host2) => {
|
|
2587
|
+
if (!url || seen.has(url)) return;
|
|
2588
|
+
seen.add(url);
|
|
2589
|
+
out.push({ url, version: "published", source: "core", host: host2, pdf });
|
|
2590
|
+
};
|
|
2591
|
+
for (const w of d.results ?? []) {
|
|
2592
|
+
add(w.downloadUrl, true, "core");
|
|
2593
|
+
for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
|
|
2594
|
+
for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
|
|
2450
2595
|
}
|
|
2451
|
-
return
|
|
2452
|
-
}
|
|
2453
|
-
function renderMeta(m) {
|
|
2454
|
-
const lines = [
|
|
2455
|
-
["pmid", m.pmid],
|
|
2456
|
-
...m.setid ? [["setid", m.setid]] : [],
|
|
2457
|
-
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
2458
|
-
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
2459
|
-
["pmcid", m.pmcid ?? ""],
|
|
2460
|
-
["doi", m.doi ?? ""],
|
|
2461
|
-
["title", JSON.stringify(m.title)],
|
|
2462
|
-
["first_author", m.first_author ?? ""],
|
|
2463
|
-
["year", m.year ?? ""],
|
|
2464
|
-
["publication_types", m.publication_types.join("; ")],
|
|
2465
|
-
["retracted", m.retracted ? "yes" : "no"],
|
|
2466
|
-
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
2467
|
-
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
2468
|
-
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
2469
|
-
// a JSON array: a statement may hold semicolons
|
|
2470
|
-
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
2471
|
-
["level", m.level],
|
|
2472
|
-
["version", m.version ?? ""],
|
|
2473
|
-
["files", m.files.join(", ")],
|
|
2474
|
-
["retrieved_by", m.retrieved_by],
|
|
2475
|
-
["retrieved_date", m.retrieved_date],
|
|
2476
|
-
["converted_by", m.converted_by],
|
|
2477
|
-
["pubmed_checked", m.pubmed_checked],
|
|
2478
|
-
["attempts", m.attempts.join(", ")]
|
|
2479
|
-
];
|
|
2480
|
-
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
2596
|
+
return out;
|
|
2481
2597
|
}
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2598
|
+
|
|
2599
|
+
// src/tiers/europepmc.ts
|
|
2600
|
+
var europepmc = {
|
|
2601
|
+
name: "europepmc",
|
|
2602
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
2603
|
+
lookup: (p) => fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`)}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }),
|
|
2604
|
+
async run(p, ctx) {
|
|
2605
|
+
let data;
|
|
2606
|
+
try {
|
|
2607
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2608
|
+
} catch (e) {
|
|
2609
|
+
return miss(this.name, errMsg(e));
|
|
2610
|
+
}
|
|
2611
|
+
const rec = data.resultList?.result?.[0];
|
|
2612
|
+
if (!rec) return miss(this.name, "not in europe pmc");
|
|
2613
|
+
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
2614
|
+
if (!candidates.length) return miss(this.name, "no free copy");
|
|
2615
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2616
|
+
return download(this.name, p, candidates, ctx);
|
|
2491
2617
|
}
|
|
2492
|
-
|
|
2493
|
-
|
|
2494
|
-
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
2495
|
-
function splitAttempts(v) {
|
|
2496
|
-
if (!v) return [];
|
|
2618
|
+
};
|
|
2619
|
+
function europepmcCandidates(r, pmcid) {
|
|
2497
2620
|
const out = [];
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
if (
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2621
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2622
|
+
const add = (url, pdf, host2, version) => {
|
|
2623
|
+
if (!url || seen.has(url)) return;
|
|
2624
|
+
seen.add(url);
|
|
2625
|
+
out.push({ url, version, source: "europepmc", host: host2, pdf });
|
|
2626
|
+
};
|
|
2627
|
+
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
2628
|
+
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
2629
|
+
if (id && r.inEPMC === "Y") {
|
|
2630
|
+
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
2631
|
+
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
2506
2632
|
}
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
if (
|
|
2511
|
-
|
|
2633
|
+
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
2634
|
+
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
2635
|
+
if (u.documentStyle === "abs") continue;
|
|
2636
|
+
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
2637
|
+
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
2512
2638
|
}
|
|
2513
|
-
return
|
|
2639
|
+
return out;
|
|
2514
2640
|
}
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2641
|
+
|
|
2642
|
+
// src/tiers/openalex.ts
|
|
2643
|
+
var openalex = {
|
|
2644
|
+
name: "openalex",
|
|
2645
|
+
applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
|
|
2646
|
+
lookup: (p) => fetchText(`https://api.openalex.org/works/${encodeURIComponent(p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`)}?mailto=${encodeURIComponent(contactEmail())}`, { "User-Agent": LIBRARY_UA }),
|
|
2647
|
+
async run(p, ctx) {
|
|
2648
|
+
let data;
|
|
2649
|
+
try {
|
|
2650
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2651
|
+
} catch (e) {
|
|
2652
|
+
return miss(this.name, errMsg(e));
|
|
2653
|
+
}
|
|
2654
|
+
const candidates = openalexCandidates(data);
|
|
2655
|
+
if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
|
|
2656
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2657
|
+
return download(this.name, p, candidates, ctx);
|
|
2658
|
+
}
|
|
2659
|
+
};
|
|
2660
|
+
function openalexCandidates(d) {
|
|
2661
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
2662
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2520
2663
|
const out = [];
|
|
2521
|
-
for (const
|
|
2522
|
-
|
|
2523
|
-
|
|
2664
|
+
for (const l of locs) {
|
|
2665
|
+
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
2666
|
+
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
2667
|
+
seen.add(url);
|
|
2668
|
+
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
2669
|
+
}
|
|
2524
2670
|
}
|
|
2525
2671
|
return out;
|
|
2526
2672
|
}
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2673
|
+
|
|
2674
|
+
// src/tiers/pmc.ts
|
|
2675
|
+
function pmcEfetchUrl(pmcid) {
|
|
2676
|
+
const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
|
|
2677
|
+
u.searchParams.set("db", "pmc");
|
|
2678
|
+
u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
|
|
2679
|
+
u.searchParams.set("retmode", "xml");
|
|
2680
|
+
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
2681
|
+
u.searchParams.set("tool", "fh-library");
|
|
2682
|
+
u.searchParams.set("email", contactEmail());
|
|
2683
|
+
return u.toString();
|
|
2533
2684
|
}
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2685
|
+
var pmc = {
|
|
2686
|
+
name: "pmc",
|
|
2687
|
+
applies: (p) => p.pmcid ? true : "no pmcid",
|
|
2688
|
+
async run(p) {
|
|
2689
|
+
let xml;
|
|
2690
|
+
try {
|
|
2691
|
+
xml = await fetchText(pmcEfetchUrl(p.pmcid));
|
|
2692
|
+
} catch (e) {
|
|
2693
|
+
return miss(this.name, errMsg(e));
|
|
2694
|
+
}
|
|
2695
|
+
const doc = parseJats(xml);
|
|
2696
|
+
if (!doc.hasBody) return miss(this.name, "no body");
|
|
2697
|
+
const md = renderFulltextMd(doc, p.headLine);
|
|
2698
|
+
const check = checkFulltext(md, p.title);
|
|
2699
|
+
if (!check.ok) return miss(this.name, check.reason);
|
|
2700
|
+
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
2539
2701
|
}
|
|
2540
|
-
}
|
|
2541
|
-
|
|
2702
|
+
};
|
|
2703
|
+
|
|
2704
|
+
// src/tiers/tdm.ts
|
|
2705
|
+
var CROSSREF = "https://api.crossref.org/works/";
|
|
2706
|
+
var TOKENED = [
|
|
2707
|
+
{ host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
|
|
2708
|
+
{ host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
|
|
2709
|
+
];
|
|
2710
|
+
var hostOf3 = (url) => {
|
|
2542
2711
|
try {
|
|
2543
|
-
|
|
2544
|
-
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
2712
|
+
return new URL(url).hostname;
|
|
2545
2713
|
} catch {
|
|
2546
|
-
return
|
|
2714
|
+
return "";
|
|
2547
2715
|
}
|
|
2548
|
-
}
|
|
2549
|
-
function
|
|
2550
|
-
const
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2716
|
+
};
|
|
2717
|
+
function tdmCandidates(work, tokens2 = {}, env = process.env) {
|
|
2718
|
+
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
2719
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2720
|
+
const candidates = [];
|
|
2721
|
+
const missingTokens = /* @__PURE__ */ new Set();
|
|
2722
|
+
for (const l of links) {
|
|
2723
|
+
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
2724
|
+
if (seen.has(l.URL)) {
|
|
2725
|
+
const prev = candidates.find((c) => c.url === l.URL);
|
|
2726
|
+
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
2727
|
+
continue;
|
|
2556
2728
|
}
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
coi = JSON.parse(fm.coi);
|
|
2566
|
-
} catch {
|
|
2567
|
-
coi = fm.coi;
|
|
2729
|
+
seen.add(l.URL);
|
|
2730
|
+
const rule = TOKENED.find((r) => r.host.test(hostOf3(l.URL)));
|
|
2731
|
+
let headers;
|
|
2732
|
+
if (rule) {
|
|
2733
|
+
const token = tokens2[rule.key]?.trim() || env[rule.env]?.trim();
|
|
2734
|
+
if (!token) {
|
|
2735
|
+
missingTokens.add(rule.key);
|
|
2736
|
+
continue;
|
|
2568
2737
|
}
|
|
2569
|
-
|
|
2570
|
-
|
|
2738
|
+
headers = { [rule.header]: token };
|
|
2739
|
+
}
|
|
2740
|
+
if (/text\/plain/.test(ct)) continue;
|
|
2741
|
+
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf3(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
2571
2742
|
}
|
|
2743
|
+
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
2744
|
+
candidates.sort((a, b) => rank(a) - rank(b));
|
|
2745
|
+
return { candidates, missingTokens: [...missingTokens] };
|
|
2746
|
+
}
|
|
2747
|
+
function tdmTier(tokens2 = {}) {
|
|
2572
2748
|
return {
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
2589
|
-
version: opt(fm.version),
|
|
2590
|
-
files: list2(fm.files),
|
|
2591
|
-
retrieved_by: fm.retrieved_by ?? "",
|
|
2592
|
-
retrieved_date: fm.retrieved_date ?? "",
|
|
2593
|
-
converted_by: fm.converted_by ?? "",
|
|
2594
|
-
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
2595
|
-
attempts: splitAttempts(fm.attempts)
|
|
2749
|
+
name: "tdm",
|
|
2750
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2751
|
+
lookup: (p) => fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }),
|
|
2752
|
+
async run(p, ctx) {
|
|
2753
|
+
let work;
|
|
2754
|
+
try {
|
|
2755
|
+
work = JSON.parse(await looked(this, p, ctx));
|
|
2756
|
+
} catch (e) {
|
|
2757
|
+
return miss(this.name, errMsg(e));
|
|
2758
|
+
}
|
|
2759
|
+
const { candidates, missingTokens } = tdmCandidates(work, tokens2);
|
|
2760
|
+
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
2761
|
+
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
2762
|
+
return download(this.name, p, candidates, ctx);
|
|
2763
|
+
}
|
|
2596
2764
|
};
|
|
2597
2765
|
}
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2766
|
+
var tdm = tdmTier();
|
|
2767
|
+
|
|
2768
|
+
// src/tiers/unpaywall.ts
|
|
2769
|
+
var unpaywall = {
|
|
2770
|
+
name: "unpaywall",
|
|
2771
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2772
|
+
lookup: (p) => fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(contactEmail())}`, { "User-Agent": LIBRARY_UA }),
|
|
2773
|
+
async run(p, ctx) {
|
|
2774
|
+
let data;
|
|
2775
|
+
try {
|
|
2776
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2777
|
+
} catch (e) {
|
|
2778
|
+
return miss(this.name, errMsg(e));
|
|
2779
|
+
}
|
|
2780
|
+
const candidates = unpaywallCandidates(data);
|
|
2781
|
+
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
2782
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2783
|
+
return download(this.name, p, candidates, ctx);
|
|
2608
2784
|
}
|
|
2609
|
-
}
|
|
2610
|
-
function
|
|
2611
|
-
const
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
if (existsSync4(to.entryDir)) renameSync2(to.entryDir, old);
|
|
2621
|
-
renameSync2(tmp, to.entryDir);
|
|
2622
|
-
rmSync(old, { recursive: true, force: true });
|
|
2623
|
-
return copy;
|
|
2624
|
-
}
|
|
2625
|
-
var NOTHING_TO_VERIFY = "no abstract and no full text: nothing to verify";
|
|
2626
|
-
function citability(m) {
|
|
2627
|
-
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
2628
|
-
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
2629
|
-
if (m.retracted) {
|
|
2630
|
-
const r = m.corrections.filter(isRetractionNotice);
|
|
2631
|
-
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
2785
|
+
};
|
|
2786
|
+
function unpaywallCandidates(d) {
|
|
2787
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
|
|
2788
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2789
|
+
const out = [];
|
|
2790
|
+
for (const l of locs) {
|
|
2791
|
+
for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
|
|
2792
|
+
if (!url || seen.has(url)) continue;
|
|
2793
|
+
seen.add(url);
|
|
2794
|
+
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
2795
|
+
}
|
|
2632
2796
|
}
|
|
2633
|
-
|
|
2634
|
-
const notes = [
|
|
2635
|
-
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
2636
|
-
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
2637
|
-
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
2638
|
-
];
|
|
2639
|
-
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
2797
|
+
return out;
|
|
2640
2798
|
}
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2799
|
+
|
|
2800
|
+
// src/tiers/index.ts
|
|
2801
|
+
var BUILTIN = [
|
|
2802
|
+
{ name: "pmc", make: () => pmc },
|
|
2803
|
+
{ name: "europepmc", make: () => europepmc },
|
|
2804
|
+
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
2805
|
+
{ name: "unpaywall", make: () => unpaywall },
|
|
2806
|
+
{ name: "openalex", make: () => openalex },
|
|
2807
|
+
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
2808
|
+
];
|
|
2809
|
+
|
|
2810
|
+
// src/tiers/url.ts
|
|
2811
|
+
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
2812
|
+
function urlTier(name, template, headers) {
|
|
2813
|
+
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
2647
2814
|
return {
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
first_author: m.first_author,
|
|
2659
|
-
year: m.year,
|
|
2660
|
-
publication_types: m.publication_types,
|
|
2661
|
-
retracted: m.retracted,
|
|
2662
|
-
corrections: m.corrections,
|
|
2663
|
-
coi: m.coi,
|
|
2664
|
-
funding: m.funding,
|
|
2665
|
-
...cit,
|
|
2666
|
-
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
2667
|
-
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
2668
|
-
// there is abstract text).
|
|
2669
|
-
has_abstract: m.has_abstract,
|
|
2670
|
-
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
2671
|
-
version: m.version,
|
|
2672
|
-
retrieved_by: m.retrieved_by || null,
|
|
2673
|
-
retrieved_date: m.retrieved_date || null,
|
|
2674
|
-
pubmed_checked: m.pubmed_checked || null,
|
|
2675
|
-
attempts: m.attempts,
|
|
2676
|
-
files: m.files,
|
|
2677
|
-
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
2815
|
+
name,
|
|
2816
|
+
applies(p) {
|
|
2817
|
+
const missing = used.filter((k) => !p[k]);
|
|
2818
|
+
return missing.length ? `no ${missing.join("/")}` : true;
|
|
2819
|
+
},
|
|
2820
|
+
async run(p, ctx) {
|
|
2821
|
+
if (!used.length) return miss(name, "template has no placeholder");
|
|
2822
|
+
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
2823
|
+
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
2824
|
+
}
|
|
2678
2825
|
};
|
|
2679
2826
|
}
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2827
|
+
|
|
2828
|
+
// src/config.ts
|
|
2829
|
+
var VARIABLES = [
|
|
2830
|
+
"FH_LIBRARY_DIR",
|
|
2831
|
+
"FH_LIBRARY_ENV",
|
|
2832
|
+
"FH_LIBRARY_TIERS",
|
|
2833
|
+
"FH_LIBRARY_CONTACT_EMAIL",
|
|
2834
|
+
"FH_LIBRARY_ELSEVIER_API_KEY",
|
|
2835
|
+
"FH_LIBRARY_WILEY_TDM_TOKEN",
|
|
2836
|
+
"FH_LIBRARY_CORE_API_KEY",
|
|
2837
|
+
"FH_LIBRARY_PROXY_URL",
|
|
2838
|
+
"FH_LIBRARY_PROXY_HEADERS",
|
|
2839
|
+
"FH_LIBRARY_PLAYWRIGHT_1",
|
|
2840
|
+
"FH_LIBRARY_PLAYWRIGHT_2",
|
|
2841
|
+
"FH_LIBRARY_CHROME_1",
|
|
2842
|
+
"FH_LIBRARY_PLAYWRIGHT_HEADLESS",
|
|
2843
|
+
"FH_LIBRARY_VERIFY_URL",
|
|
2844
|
+
"FH_LIBRARY_VERIFY_API_KEY",
|
|
2845
|
+
"FH_LIBRARY_VERIFY_MODEL",
|
|
2846
|
+
"FH_LIBRARY_VERIFY_THRESHOLD",
|
|
2847
|
+
"FH_LIBRARY_VERIFY_TIMEOUT_SECS",
|
|
2848
|
+
"FH_LIBRARY_RETRY_AFTER_DAYS",
|
|
2849
|
+
"FH_LIBRARY_RETRY_BLOCKED",
|
|
2850
|
+
"FH_LIBRARY_REFRESH_AFTER_DAYS",
|
|
2851
|
+
"FH_LIBRARY_SEARCH_CACHE_DAYS",
|
|
2852
|
+
"FH_LIBRARY_NCBI_API_KEY",
|
|
2853
|
+
"FH_LIBRARY_BRIGHTDATA_API_TOKEN",
|
|
2854
|
+
"FH_LIBRARY_BRIGHTDATA_ZONE"
|
|
2855
|
+
];
|
|
2856
|
+
var sources = (env = process.env) => Object.fromEntries(VARIABLES.map((v) => [v, sourceOf(v, env)]));
|
|
2857
|
+
function settings(env = process.env) {
|
|
2858
|
+
return {
|
|
2859
|
+
retry_after_days: num("FH_LIBRARY_RETRY_AFTER_DAYS", 1, env),
|
|
2860
|
+
retry_blocked: bool("FH_LIBRARY_RETRY_BLOCKED", false, env),
|
|
2861
|
+
refresh_after_days: num("FH_LIBRARY_REFRESH_AFTER_DAYS", 30, env),
|
|
2862
|
+
search_cache_days: num("FH_LIBRARY_SEARCH_CACHE_DAYS", 7, env)
|
|
2708
2863
|
};
|
|
2709
|
-
writeMeta(p, next);
|
|
2710
|
-
return next;
|
|
2711
|
-
}
|
|
2712
|
-
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
2713
|
-
const t = new Date(date).getTime();
|
|
2714
|
-
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
2715
2864
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
for (let i = 0; i < lines.length; i++) {
|
|
2724
|
-
const l = lines[i].trim();
|
|
2725
|
-
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
2726
|
-
if (h) {
|
|
2727
|
-
section2 = h[1];
|
|
2728
|
-
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
2729
|
-
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
2730
|
-
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
2731
|
-
continue;
|
|
2732
|
-
}
|
|
2733
|
-
if (section2 !== "Declarations" || !l) continue;
|
|
2734
|
-
const f = FUNDING_RE.exec(l);
|
|
2735
|
-
const c = COI_RE.exec(l);
|
|
2736
|
-
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
2737
|
-
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
2865
|
+
function proxyHeaders(spec) {
|
|
2866
|
+
const out = {};
|
|
2867
|
+
for (const part of spec.split(";")) {
|
|
2868
|
+
const i = part.indexOf(":");
|
|
2869
|
+
if (i < 0) continue;
|
|
2870
|
+
const name = part.slice(0, i).trim(), value = part.slice(i + 1).trim();
|
|
2871
|
+
if (name && value) out[name] = value;
|
|
2738
2872
|
}
|
|
2739
|
-
return
|
|
2873
|
+
return out;
|
|
2740
2874
|
}
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
const
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2875
|
+
function resolveTiers(env = process.env) {
|
|
2876
|
+
const problems = [];
|
|
2877
|
+
const available = /* @__PURE__ */ new Map();
|
|
2878
|
+
const tokens2 = { elsevier: str("FH_LIBRARY_ELSEVIER_API_KEY", env) || void 0, wiley: str("FH_LIBRARY_WILEY_TDM_TOKEN", env) || void 0 };
|
|
2879
|
+
for (const t of BUILTIN) available.set(t.name, t.make({ tokens: tokens2, api_key: str("FH_LIBRARY_CORE_API_KEY", env) || void 0 }));
|
|
2880
|
+
const proxyUrl = str("FH_LIBRARY_PROXY_URL", env);
|
|
2881
|
+
if (proxyUrl) available.set("proxy", urlTier("proxy", proxyUrl, proxyHeaders(str("FH_LIBRARY_PROXY_HEADERS", env))));
|
|
2882
|
+
const listed = str("FH_LIBRARY_TIERS", env);
|
|
2883
|
+
const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
|
|
2884
|
+
const tiers = [], seen = /* @__PURE__ */ new Set();
|
|
2885
|
+
for (const name of order2) {
|
|
2886
|
+
if (seen.has(name)) continue;
|
|
2887
|
+
seen.add(name);
|
|
2888
|
+
const t = available.get(name);
|
|
2889
|
+
if (t) tiers.push(t);
|
|
2890
|
+
else if (name === "proxy") problems.push('FH_LIBRARY_TIERS: "proxy" listed, but FH_LIBRARY_PROXY_URL is unset');
|
|
2891
|
+
else problems.push(`FH_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when FH_LIBRARY_PROXY_URL is set)`);
|
|
2749
2892
|
}
|
|
2750
|
-
|
|
2893
|
+
const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
|
|
2894
|
+
return { tiers, all, source: sourceOf("FH_LIBRARY_TIERS", env), problems };
|
|
2751
2895
|
}
|
|
2752
|
-
function withCorrectedBlock(md, sentences) {
|
|
2753
|
-
const base = md.split(`
|
|
2754
|
-
${CORRECTED_HEADING}
|
|
2755
|
-
`)[0].trimEnd();
|
|
2756
|
-
if (!sentences.length) return base + "\n";
|
|
2757
|
-
return `${base}
|
|
2758
|
-
|
|
2759
|
-
${CORRECTED_HEADING}
|
|
2760
|
-
|
|
2761
|
-
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
2762
2896
|
|
|
2763
|
-
|
|
2764
|
-
|
|
2897
|
+
// src/log.ts
|
|
2898
|
+
import { appendFileSync, mkdirSync as mkdirSync4, renameSync as renameSync2, statSync } from "fs";
|
|
2899
|
+
import { hostname } from "os";
|
|
2900
|
+
import { join as join6 } from "path";
|
|
2901
|
+
var HOST = hostname();
|
|
2902
|
+
var LOG_FILE = join6(FH_LIBRARY_DIR, "library.log");
|
|
2903
|
+
function stamp() {
|
|
2904
|
+
const d = /* @__PURE__ */ new Date();
|
|
2905
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
2906
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
2765
2907
|
}
|
|
2766
|
-
|
|
2767
|
-
|
|
2768
|
-
let
|
|
2908
|
+
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
2909
|
+
function rotateIfLarge(file3, max = LOG_MAX_BYTES) {
|
|
2910
|
+
let size = 0;
|
|
2769
2911
|
try {
|
|
2770
|
-
|
|
2912
|
+
size = statSync(file3).size;
|
|
2771
2913
|
} catch {
|
|
2772
|
-
return;
|
|
2914
|
+
return false;
|
|
2915
|
+
}
|
|
2916
|
+
if (size <= max) return false;
|
|
2917
|
+
try {
|
|
2918
|
+
renameSync2(file3, `${file3}.1`);
|
|
2919
|
+
return true;
|
|
2920
|
+
} catch {
|
|
2921
|
+
return false;
|
|
2773
2922
|
}
|
|
2774
|
-
if (!r) return;
|
|
2775
|
-
const md = readFileSync3(p.fulltextMd, "utf8");
|
|
2776
|
-
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
2777
|
-
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
2778
|
-
}
|
|
2779
|
-
function applyFulltext(p, m, hit, converter) {
|
|
2780
|
-
const raw = p.fulltextRaw(hit.format);
|
|
2781
|
-
writeAtomic(raw, hit.raw);
|
|
2782
|
-
writeAtomic(p.fulltextMd, hit.md);
|
|
2783
|
-
syncCorrectedBlock(p);
|
|
2784
|
-
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
2785
|
-
const own = declarationsFromMd(hit.md);
|
|
2786
|
-
const next = {
|
|
2787
|
-
...m,
|
|
2788
|
-
level: "full_text",
|
|
2789
|
-
version: hit.version,
|
|
2790
|
-
files,
|
|
2791
|
-
retrieved_by: hit.tier,
|
|
2792
|
-
retrieved_date: today(),
|
|
2793
|
-
converted_by: converter,
|
|
2794
|
-
coi: m.coi ?? own.coi,
|
|
2795
|
-
funding: m.funding.length ? m.funding : own.funding
|
|
2796
|
-
};
|
|
2797
|
-
writeMeta(p, next);
|
|
2798
|
-
return next;
|
|
2799
|
-
}
|
|
2800
|
-
function recordAttempts(p, m, labels) {
|
|
2801
|
-
if (!labels.length) return m;
|
|
2802
|
-
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
2803
|
-
const replaced = new Set(labels.map(tierOf));
|
|
2804
|
-
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
2805
|
-
writeMeta(p, next);
|
|
2806
|
-
return next;
|
|
2807
|
-
}
|
|
2808
|
-
function writeLabelEntry(p, s, xml, converter) {
|
|
2809
|
-
mkdirSync3(p.entryDir, { recursive: true });
|
|
2810
|
-
const raw = p.fulltextRaw("xml");
|
|
2811
|
-
writeAtomic(raw, xml);
|
|
2812
|
-
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
2813
|
-
const meta = {
|
|
2814
|
-
pmid: "",
|
|
2815
|
-
setid: s.setid,
|
|
2816
|
-
nctid: null,
|
|
2817
|
-
publications: [],
|
|
2818
|
-
pmcid: null,
|
|
2819
|
-
doi: null,
|
|
2820
|
-
title: s.title,
|
|
2821
|
-
first_author: s.labeler,
|
|
2822
|
-
year: s.effective?.slice(0, 4) ?? null,
|
|
2823
|
-
publication_types: ["Drug Label"],
|
|
2824
|
-
retracted: false,
|
|
2825
|
-
corrections: [],
|
|
2826
|
-
coi: null,
|
|
2827
|
-
funding: [],
|
|
2828
|
-
has_abstract: false,
|
|
2829
|
-
level: "full_text",
|
|
2830
|
-
version: s.version ? `spl ${s.version}` : null,
|
|
2831
|
-
files: [basename(raw), basename(p.fulltextMd)],
|
|
2832
|
-
retrieved_by: "dailymed",
|
|
2833
|
-
retrieved_date: today(),
|
|
2834
|
-
converted_by: converter,
|
|
2835
|
-
pubmed_checked: today(),
|
|
2836
|
-
attempts: []
|
|
2837
|
-
};
|
|
2838
|
-
writeMeta(p, meta);
|
|
2839
|
-
return meta;
|
|
2840
|
-
}
|
|
2841
|
-
function writeStudyEntry(p, s, json2, converter) {
|
|
2842
|
-
mkdirSync3(p.entryDir, { recursive: true });
|
|
2843
|
-
const raw = p.fulltextRaw("json");
|
|
2844
|
-
writeAtomic(raw, json2);
|
|
2845
|
-
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
2846
|
-
const meta = {
|
|
2847
|
-
pmid: "",
|
|
2848
|
-
setid: null,
|
|
2849
|
-
nctid: s.nctid,
|
|
2850
|
-
publications: s.publications,
|
|
2851
|
-
pmcid: null,
|
|
2852
|
-
doi: null,
|
|
2853
|
-
title: s.title,
|
|
2854
|
-
first_author: s.sponsor,
|
|
2855
|
-
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
2856
|
-
publication_types: ["Trial Registration"],
|
|
2857
|
-
retracted: false,
|
|
2858
|
-
corrections: [],
|
|
2859
|
-
coi: null,
|
|
2860
|
-
funding: [],
|
|
2861
|
-
has_abstract: false,
|
|
2862
|
-
level: "full_text",
|
|
2863
|
-
version: s.lastUpdate,
|
|
2864
|
-
files: [basename(raw), basename(p.fulltextMd)],
|
|
2865
|
-
retrieved_by: "ctgov",
|
|
2866
|
-
retrieved_date: today(),
|
|
2867
|
-
converted_by: converter,
|
|
2868
|
-
pubmed_checked: today(),
|
|
2869
|
-
attempts: []
|
|
2870
|
-
};
|
|
2871
|
-
writeMeta(p, meta);
|
|
2872
|
-
return meta;
|
|
2873
2923
|
}
|
|
2874
|
-
function
|
|
2875
|
-
|
|
2876
|
-
const
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2924
|
+
function log(tool, id, message) {
|
|
2925
|
+
const clean2 = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
2926
|
+
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean2}
|
|
2927
|
+
`;
|
|
2928
|
+
try {
|
|
2929
|
+
mkdirSync4(FH_LIBRARY_DIR, { recursive: true });
|
|
2930
|
+
rotateIfLarge(LOG_FILE);
|
|
2931
|
+
appendFileSync(LOG_FILE, line);
|
|
2932
|
+
} catch {
|
|
2933
|
+
}
|
|
2882
2934
|
}
|
|
2883
2935
|
|
|
2884
2936
|
// src/stats.ts
|
|
2937
|
+
import { mkdirSync as mkdirSync5, readdirSync as readdirSync2, readFileSync as readFileSync5, statSync as statSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
2938
|
+
import { join as join7 } from "path";
|
|
2885
2939
|
var HEARTBEAT_MS = 5 * 6e4;
|
|
2886
2940
|
var STALE_MS = 20 * 6e4;
|
|
2887
2941
|
var stats = null;
|
|
2888
|
-
var
|
|
2942
|
+
var file2 = null;
|
|
2889
2943
|
function startStats(pkg2, libraryDir = FH_LIBRARY_DIR) {
|
|
2890
2944
|
stats = { host: HOST, pid: process.pid, package: pkg2.name, version: pkg2.version, started: (/* @__PURE__ */ new Date()).toISOString(), verify: { hits: 0, misses: 0 }, fetch: { hits: 0, misses: 0 }, search: { hits: 0, misses: 0 }, pubmed: { hits: 0, misses: 0 } };
|
|
2891
|
-
|
|
2892
|
-
sweep(
|
|
2945
|
+
file2 = join7(libraryDir, ".servers", `${HOST}-${process.pid}.json`);
|
|
2946
|
+
sweep(join7(libraryDir, ".servers"));
|
|
2893
2947
|
publish();
|
|
2894
2948
|
setInterval(publish, HEARTBEAT_MS).unref();
|
|
2895
2949
|
const stop = () => {
|
|
2896
2950
|
try {
|
|
2897
|
-
if (
|
|
2951
|
+
if (file2) unlinkSync2(file2);
|
|
2898
2952
|
} catch {
|
|
2899
2953
|
}
|
|
2900
2954
|
};
|
|
@@ -2912,7 +2966,7 @@ function sweep(dir, staleMs = STALE_MS) {
|
|
|
2912
2966
|
}
|
|
2913
2967
|
for (const name of names) {
|
|
2914
2968
|
if (!name.endsWith(".json")) continue;
|
|
2915
|
-
const p =
|
|
2969
|
+
const p = join7(dir, name);
|
|
2916
2970
|
try {
|
|
2917
2971
|
if (Date.now() - statSync2(p).mtimeMs > staleMs) {
|
|
2918
2972
|
unlinkSync2(p);
|
|
@@ -2930,16 +2984,16 @@ function bump(kind, hit, n = 1) {
|
|
|
2930
2984
|
}
|
|
2931
2985
|
var currentStats = () => stats;
|
|
2932
2986
|
function publish() {
|
|
2933
|
-
if (!stats || !
|
|
2987
|
+
if (!stats || !file2) return;
|
|
2934
2988
|
try {
|
|
2935
|
-
|
|
2936
|
-
writeAtomic(
|
|
2989
|
+
mkdirSync5(join7(file2, ".."), { recursive: true });
|
|
2990
|
+
writeAtomic(file2, JSON.stringify(stats, null, 2) + "\n");
|
|
2937
2991
|
} catch {
|
|
2938
2992
|
}
|
|
2939
2993
|
}
|
|
2940
2994
|
|
|
2941
2995
|
// src/tools.ts
|
|
2942
|
-
import { existsSync as existsSync12, readFileSync as
|
|
2996
|
+
import { existsSync as existsSync12, readFileSync as readFileSync13 } from "fs";
|
|
2943
2997
|
import { z } from "zod";
|
|
2944
2998
|
|
|
2945
2999
|
// src/cascade.ts
|
|
@@ -2971,10 +3025,18 @@ function shouldSkip(attempts, tier, retryAfterDays, now = /* @__PURE__ */ new Da
|
|
|
2971
3025
|
async function fulltextCascade(p, attempts, onEvent, tiers = resolveTiers().tiers, retryAfterDays = settings().retry_after_days, retryBlocked = settings().retry_blocked) {
|
|
2972
3026
|
const misses = [];
|
|
2973
3027
|
const tried = /* @__PURE__ */ new Set();
|
|
3028
|
+
const lookups = /* @__PURE__ */ new Map();
|
|
3029
|
+
for (const tier of tiers) {
|
|
3030
|
+
if (!tier.lookup || shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked) || tier.applies(p) !== true) continue;
|
|
3031
|
+
const q = tier.lookup(p);
|
|
3032
|
+
q.catch(() => {
|
|
3033
|
+
});
|
|
3034
|
+
lookups.set(tier.name, q);
|
|
3035
|
+
}
|
|
2974
3036
|
for (const tier of tiers) {
|
|
2975
3037
|
if (shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked)) continue;
|
|
2976
3038
|
const applies = tier.applies(p);
|
|
2977
|
-
const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
|
|
3039
|
+
const ctx = { tried, lookups, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
|
|
2978
3040
|
const r = applies === true ? await tier.run(p, ctx) : { ...miss(tier.name, applies), skipped: true };
|
|
2979
3041
|
onEvent?.(r);
|
|
2980
3042
|
if (r.ok) return { hit: r, misses };
|
|
@@ -3014,11 +3076,11 @@ function parseId(raw) {
|
|
|
3014
3076
|
}
|
|
3015
3077
|
|
|
3016
3078
|
// src/info.ts
|
|
3017
|
-
import { existsSync as existsSync6, readdirSync as readdirSync3, readFileSync as
|
|
3018
|
-
import { dirname as dirname2, join as
|
|
3079
|
+
import { existsSync as existsSync6, readdirSync as readdirSync3, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
|
|
3080
|
+
import { dirname as dirname2, join as join8 } from "path";
|
|
3019
3081
|
|
|
3020
3082
|
// src/verify.ts
|
|
3021
|
-
import { existsSync as existsSync5, readFileSync as
|
|
3083
|
+
import { existsSync as existsSync5, readFileSync as readFileSync6 } from "fs";
|
|
3022
3084
|
var DEFAULT_MODEL = "pplx-decider-flash-v1.2";
|
|
3023
3085
|
var QUESTION = "Does the paper state this, in substance? Ignore section labels and definitions of terms in parentheses. Every fact and number of the statement must be stated in the paper or follow from its numbers; a rounded number counts. If any part of the statement is absent from the paper or contradicted by it, answer false.";
|
|
3024
3086
|
function verifyConfig(env = process.env) {
|
|
@@ -3051,7 +3113,26 @@ function isLocal(baseUrl) {
|
|
|
3051
3113
|
}
|
|
3052
3114
|
}
|
|
3053
3115
|
function cleanMarkdown(md) {
|
|
3054
|
-
return md.replace(/!\[[^\]]*\]\([^)]*\)/g, "").replace(/\]\([^)]*\)/g, "]").replace(/^\s*#+\s*(see more about|more for you|back to top)\b.*$/gim, "").replace(/\n{3,}/g, "\n\n");
|
|
3116
|
+
return paperYearOnly(dropReferences(md)).replace(/!\[[^\]]*\]\([^)]*\)/g, "").replace(/\]\([^)]*\)/g, "]").replace(/^\s*#+\s*(see more about|more for you|back to top)\b.*$/gim, "").replace(/\n{3,}/g, "\n\n");
|
|
3117
|
+
}
|
|
3118
|
+
var REF_HEAD = /^(#{1,6})\s*\**\s*(references|bibliography|literature cited|reference list|works cited|cited literature)\b[^\n]*$|^\**(references|bibliography|literature cited)\**:?[ \t]*$/gim;
|
|
3119
|
+
function dropReferences(md) {
|
|
3120
|
+
let out = md;
|
|
3121
|
+
for (; ; ) {
|
|
3122
|
+
REF_HEAD.lastIndex = 0;
|
|
3123
|
+
const m = REF_HEAD.exec(out);
|
|
3124
|
+
if (!m) return out;
|
|
3125
|
+
const level = m[1]?.length ?? 6;
|
|
3126
|
+
const rest = out.slice(m.index + m[0].length);
|
|
3127
|
+
const next = new RegExp(`^#{1,${level}}\\s`, "m").exec(rest);
|
|
3128
|
+
out = out.slice(0, m.index) + (next ? rest.slice(next.index) : "");
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
function paperYearOnly(md) {
|
|
3132
|
+
return md.replace(/^(#[^\n]*\n\n)([^\n]*· (?:PMID \d+|DOI 10\.)[^\n]*)$/m, (_, title, line) => {
|
|
3133
|
+
const year = / · (\d{4})(?: · |$)/.exec(line)?.[1];
|
|
3134
|
+
return year ? `${title}${year}` : title.trimEnd();
|
|
3135
|
+
});
|
|
3055
3136
|
}
|
|
3056
3137
|
function windows(md, max) {
|
|
3057
3138
|
const out = [];
|
|
@@ -3121,11 +3202,11 @@ async function readPaper(raw, libraryDir = FH_LIBRARY_DIR) {
|
|
|
3121
3202
|
const paths = entryPaths(libraryDir, p.type, p.id);
|
|
3122
3203
|
if (!entryExists(paths)) return { id: p.requested, error: "not in library" };
|
|
3123
3204
|
const full = existsSync5(paths.fulltextMd);
|
|
3124
|
-
if (full) return { id: p.requested, basis: "full_text", text:
|
|
3205
|
+
if (full) return { id: p.requested, basis: "full_text", text: readFileSync6(paths.fulltextMd, "utf8") };
|
|
3125
3206
|
const meta = readMeta(paths);
|
|
3126
|
-
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text:
|
|
3207
|
+
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync6(paths.md, "utf8") };
|
|
3127
3208
|
}
|
|
3128
|
-
async function verifyClaim(claim, id, cfg, decide,
|
|
3209
|
+
async function verifyClaim(claim, id, cfg, decide, read2 = readPaper) {
|
|
3129
3210
|
const out = { claim, id, verdict: "error", reason: null, score: null, basis: null, windows: 0, glyphs_unresolved: 0, superseded: null, model: cfg?.model ?? null };
|
|
3130
3211
|
const words2 = claim.trim().split(/\s+/).filter(Boolean).length;
|
|
3131
3212
|
if (!words2) return { ...out, error: "claim empty" };
|
|
@@ -3134,7 +3215,7 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
3134
3215
|
if (frag) return { ...out, error: frag };
|
|
3135
3216
|
const groups = citationGroups(claim);
|
|
3136
3217
|
if (groups.length) return { ...out, error: groups.length > 1 ? `compound claim: ${groups.length} citation groups ${groups.join(" ")} \u2014 one finding per call, without its sources` : `claim names a source ${groups[0]} \u2014 send the finding without its sources; the paper is the id` };
|
|
3137
|
-
const paper = await
|
|
3218
|
+
const paper = await read2(id);
|
|
3138
3219
|
out.id = paper.id;
|
|
3139
3220
|
if ("error" in paper) return { ...out, error: paper.error };
|
|
3140
3221
|
if (!cfg || !decide) return { ...out, error: "no verify model configured (FH_LIBRARY_VERIFY_URL)" };
|
|
@@ -3166,7 +3247,7 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
3166
3247
|
function* metaFiles(dir) {
|
|
3167
3248
|
if (!existsSync6(dir)) return;
|
|
3168
3249
|
for (const name of readdirSync3(dir)) {
|
|
3169
|
-
const p =
|
|
3250
|
+
const p = join8(dir, name);
|
|
3170
3251
|
if (statSync3(p).isDirectory()) yield* metaFiles(p);
|
|
3171
3252
|
else if (name.endsWith("-meta.md")) yield p;
|
|
3172
3253
|
}
|
|
@@ -3184,10 +3265,10 @@ function field(md, key) {
|
|
|
3184
3265
|
const m = new RegExp(`^${key}:\\s*(.*)$`, "m").exec(md);
|
|
3185
3266
|
return m ? m[1].trim() : "";
|
|
3186
3267
|
}
|
|
3187
|
-
function libraryInfo(libraryDir,
|
|
3268
|
+
function libraryInfo(libraryDir, host2, withIds, tiers = resolveTiers()) {
|
|
3188
3269
|
const info = {
|
|
3189
3270
|
library_dir: libraryDir,
|
|
3190
|
-
host,
|
|
3271
|
+
host: host2,
|
|
3191
3272
|
tiers: tiers.all,
|
|
3192
3273
|
config: { ...settings(), problems: tiers.problems, sources: sources() },
|
|
3193
3274
|
download: downloadInfo(),
|
|
@@ -3206,10 +3287,10 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3206
3287
|
};
|
|
3207
3288
|
const ids = [];
|
|
3208
3289
|
for (const type of existsSync6(libraryDir) ? readdirSync3(libraryDir) : []) {
|
|
3209
|
-
const typeDir =
|
|
3290
|
+
const typeDir = join8(libraryDir, type);
|
|
3210
3291
|
if (!statSync3(typeDir).isDirectory()) continue;
|
|
3211
3292
|
for (const meta of metaFiles(typeDir)) {
|
|
3212
|
-
const md =
|
|
3293
|
+
const md = readFileSync7(meta, "utf8");
|
|
3213
3294
|
const raw = field(md, "level") || "unknown";
|
|
3214
3295
|
const level = raw === "abstract" && field(md, "has_abstract") === "no" ? "title" : raw;
|
|
3215
3296
|
info.entries++;
|
|
@@ -3221,7 +3302,7 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3221
3302
|
const dir = dirname2(meta);
|
|
3222
3303
|
for (const f of readdirSync3(dir)) {
|
|
3223
3304
|
try {
|
|
3224
|
-
const st = statSync3(
|
|
3305
|
+
const st = statSync3(join8(dir, f));
|
|
3225
3306
|
if (st.isFile()) info.size_bytes += st.size;
|
|
3226
3307
|
} catch {
|
|
3227
3308
|
}
|
|
@@ -3250,7 +3331,7 @@ function noticeTypes(md, metaPath) {
|
|
|
3250
3331
|
return out;
|
|
3251
3332
|
}
|
|
3252
3333
|
try {
|
|
3253
|
-
const xml =
|
|
3334
|
+
const xml = readFileSync7(metaPath.replace(/-meta\.md$/, "-abstract.xml"), "utf8");
|
|
3254
3335
|
for (const m of xml.matchAll(/<CommentsCorrections RefType="([A-Za-z]+)"/g)) if (XML_TYPES[m[1]]) out.add(XML_TYPES[m[1]]);
|
|
3255
3336
|
} catch {
|
|
3256
3337
|
}
|
|
@@ -3376,15 +3457,18 @@ function search(md, query, maxHits, opts = {}) {
|
|
|
3376
3457
|
const df = /* @__PURE__ */ new Map();
|
|
3377
3458
|
for (const x of scored) for (const s of x.matched) df.set(s, (df.get(s) ?? 0) + 1);
|
|
3378
3459
|
const rarity = (x) => x.matched.reduce((n, s) => n + 1 / df.get(s), 0);
|
|
3379
|
-
const
|
|
3380
|
-
const
|
|
3460
|
+
const pool = partial ? scored : full;
|
|
3461
|
+
const pick = pool.sort((a, b) => b.matched.length - a.matched.length || (partial ? rarity(b) - rarity(a) : 0) || b.hits - a.hits || a.p.text.length - b.p.text.length).slice(0, maxHits);
|
|
3462
|
+
const passages = pick.map(({ p, matched }) => {
|
|
3381
3463
|
const w = clip(p, matched, window);
|
|
3382
|
-
const out = { section: p.section, text: w.text
|
|
3464
|
+
const out = { section: p.section, text: w.text };
|
|
3383
3465
|
if (partial) out.matched = terms.filter((t) => matched.includes(stem(t)));
|
|
3384
3466
|
if (w.clipped) out.clipped = true;
|
|
3385
3467
|
return out;
|
|
3386
3468
|
});
|
|
3387
|
-
|
|
3469
|
+
const r = { passages, partial, missing_terms };
|
|
3470
|
+
if (pool.length > passages.length) r.more = pool.length - passages.length;
|
|
3471
|
+
return r;
|
|
3388
3472
|
}
|
|
3389
3473
|
function paragraphs(md, scope) {
|
|
3390
3474
|
const lines = md.split("\n");
|
|
@@ -3629,8 +3713,8 @@ function parseCsl(json2) {
|
|
|
3629
3713
|
|
|
3630
3714
|
// src/urlfetch.ts
|
|
3631
3715
|
import { createHash as createHash2 } from "crypto";
|
|
3632
|
-
import { existsSync as existsSync7, mkdirSync as
|
|
3633
|
-
import { join as
|
|
3716
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync6, readFileSync as readFileSync8 } from "fs";
|
|
3717
|
+
import { join as join9 } from "path";
|
|
3634
3718
|
var TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src)$/i;
|
|
3635
3719
|
function normalizeUrl(raw) {
|
|
3636
3720
|
let u;
|
|
@@ -3652,8 +3736,8 @@ var urlHash = (normalized2) => createHash2("sha1").update(normalized2).digest("h
|
|
|
3652
3736
|
function urlPaths(libraryDir, normalized2) {
|
|
3653
3737
|
const hash = urlHash(normalized2);
|
|
3654
3738
|
const entry = `url-${hash}`;
|
|
3655
|
-
const entryDir =
|
|
3656
|
-
return { hash, entry, entryDir, meta:
|
|
3739
|
+
const entryDir = join9(libraryDir, "url", hash.slice(0, 2), hash.slice(0, 4), hash);
|
|
3740
|
+
return { hash, entry, entryDir, meta: join9(entryDir, `${entry}-meta.md`), md: join9(entryDir, `${entry}-page.md`), raw: (ext) => join9(entryDir, `${entry}-page.${ext}`) };
|
|
3657
3741
|
}
|
|
3658
3742
|
function renderUrlMeta(m) {
|
|
3659
3743
|
const lines = [
|
|
@@ -3673,7 +3757,7 @@ function renderUrlMeta(m) {
|
|
|
3673
3757
|
}
|
|
3674
3758
|
function readUrlMeta(p) {
|
|
3675
3759
|
if (!existsSync7(p.meta)) return null;
|
|
3676
|
-
const md =
|
|
3760
|
+
const md = readFileSync8(p.meta, "utf8");
|
|
3677
3761
|
const f = (k) => {
|
|
3678
3762
|
const m = new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md);
|
|
3679
3763
|
return m ? m[1].trim() : "";
|
|
@@ -3699,7 +3783,7 @@ function readUrlMeta(p) {
|
|
|
3699
3783
|
};
|
|
3700
3784
|
}
|
|
3701
3785
|
function writeUrlMeta(p, m) {
|
|
3702
|
-
|
|
3786
|
+
mkdirSync6(p.entryDir, { recursive: true });
|
|
3703
3787
|
writeAtomic(p.meta, renderUrlMeta(m));
|
|
3704
3788
|
}
|
|
3705
3789
|
var SHELL_WORDS = 100;
|
|
@@ -3746,13 +3830,22 @@ var climb = async (url, onTry, onAlert) => {
|
|
|
3746
3830
|
let shell = !r.ok ? r.shell : void 0;
|
|
3747
3831
|
for (const b of browserRungs()) {
|
|
3748
3832
|
if (!escalates(r)) break;
|
|
3833
|
+
const w = walled(target, `browser ${b.label}`);
|
|
3834
|
+
if (w) {
|
|
3835
|
+
onTry?.(`skipped browser ${b.label}: ${wallNote(w)}`);
|
|
3836
|
+
continue;
|
|
3837
|
+
}
|
|
3749
3838
|
r = await rung(target, { browser: b.endpoint, label: b.label });
|
|
3839
|
+
recordRung(target, `browser ${b.label}`, r.ok, r.ok ? "" : r.reason);
|
|
3750
3840
|
if (!r.ok && r.shell && !shell) shell = r.shell;
|
|
3751
3841
|
onTry?.(r.ok ? `ok browser ${b.label}` : `failed browser ${b.label}: ${r.reason}`);
|
|
3752
3842
|
if (!r.ok && r.reason.startsWith(MISMATCH)) onAlert?.(`browser ${b.label}: ${r.reason}`);
|
|
3753
3843
|
}
|
|
3754
|
-
|
|
3844
|
+
const uw = !r.ok && unlockerConfigured() ? walled(target, "brightdata") : null;
|
|
3845
|
+
if (uw) onTry?.(`skipped brightdata: ${wallNote(uw)}`);
|
|
3846
|
+
else if (!r.ok && unlockerConfigured() && (r.via !== "fetch" || /^(HTTP 403|wall page|script shell)/.test(r.reason))) {
|
|
3755
3847
|
r = await rung(target, "brightdata");
|
|
3848
|
+
recordRung(target, "brightdata", r.ok, r.ok ? "" : r.reason);
|
|
3756
3849
|
onTry?.(r.ok ? "ok brightdata" : `failed brightdata: ${r.reason}`);
|
|
3757
3850
|
if (!r.ok && /^unlocker HTTP/.test(r.reason)) onAlert?.(`brightdata: ${r.reason}`);
|
|
3758
3851
|
}
|
|
@@ -3857,16 +3950,16 @@ function libraryIdForUrl(raw) {
|
|
|
3857
3950
|
} catch {
|
|
3858
3951
|
return null;
|
|
3859
3952
|
}
|
|
3860
|
-
const
|
|
3953
|
+
const host2 = u.hostname.toLowerCase().replace(/^www\./, ""), path = u.pathname;
|
|
3861
3954
|
let m;
|
|
3862
|
-
if (
|
|
3863
|
-
if (
|
|
3864
|
-
if (
|
|
3865
|
-
if (
|
|
3955
|
+
if (host2 === "pubmed.ncbi.nlm.nih.gov" && (m = /^\/(\d{1,8})\/?$/.exec(path))) return `pmid:${m[1]}`;
|
|
3956
|
+
if (host2 === "ncbi.nlm.nih.gov" && (m = /^\/pubmed\/(\d{1,8})\/?$/.exec(path))) return `pmid:${m[1]}`;
|
|
3957
|
+
if (host2 === "clinicaltrials.gov" && (m = /^\/(?:study|ct2\/show)\/(NCT\d{8})\/?$/i.exec(path))) return `nct:${m[1].slice(3)}`;
|
|
3958
|
+
if (host2 === "dailymed.nlm.nih.gov" && /\/(drugInfo|lookup)\.cfm$/i.test(path)) {
|
|
3866
3959
|
const id = u.searchParams.get("setid");
|
|
3867
3960
|
if (id && /^[0-9a-f-]{36}$/i.test(id)) return `dailymed:${id.toLowerCase()}`;
|
|
3868
3961
|
}
|
|
3869
|
-
if ((
|
|
3962
|
+
if ((host2 === "doi.org" || host2 === "dx.doi.org") && (m = /^\/(10\.\S+)$/.exec(decodeURIComponent(path)))) return `doi:${m[1]}`;
|
|
3870
3963
|
return null;
|
|
3871
3964
|
}
|
|
3872
3965
|
var failureText = (url, reason) => {
|
|
@@ -3881,17 +3974,17 @@ function titleLine(m, md) {
|
|
|
3881
3974
|
}
|
|
3882
3975
|
function serve(requested, p, m, args, cached) {
|
|
3883
3976
|
const maxLength = args.max_length ?? 5e3, startIndex = args.start_index ?? 0;
|
|
3884
|
-
const
|
|
3977
|
+
const read2 = (file3) => existsSync7(file3) ? readFileSync8(file3, "utf8") : "";
|
|
3885
3978
|
if (args.title_only) return { ok: true, text: `Contents of ${requested}:
|
|
3886
|
-
${titleLine(m,
|
|
3979
|
+
${titleLine(m, read2(p.md))}`, cached, meta: m };
|
|
3887
3980
|
const rawFile = m.files.find((f) => !f.endsWith("-meta.md") && !f.endsWith("-page.md"));
|
|
3888
3981
|
let body, prefix = "";
|
|
3889
|
-
if (m.format === "markdown") body =
|
|
3982
|
+
if (m.format === "markdown") body = read2(p.md);
|
|
3890
3983
|
else if (m.format === "text" || args.raw) {
|
|
3891
|
-
body = rawFile ?
|
|
3984
|
+
body = rawFile ? read2(join9(p.entryDir, rawFile)) : read2(p.md);
|
|
3892
3985
|
prefix = `Content type ${m.content_type || "(not declared)"} cannot be simplified to markdown, but here is the raw content:
|
|
3893
3986
|
`;
|
|
3894
|
-
} else body =
|
|
3987
|
+
} else body = read2(p.md);
|
|
3895
3988
|
return { ok: true, text: fetchAnswer(requested, body, prefix, maxLength, startIndex), cached, meta: m };
|
|
3896
3989
|
}
|
|
3897
3990
|
var inflight = /* @__PURE__ */ new Map();
|
|
@@ -3915,7 +4008,7 @@ async function libraryFetch(libraryDir, requested, args, converter, logs = {}, f
|
|
|
3915
4008
|
return;
|
|
3916
4009
|
}
|
|
3917
4010
|
const page = await convert(r.bytes, r.contentType);
|
|
3918
|
-
|
|
4011
|
+
mkdirSync6(p.entryDir, { recursive: true });
|
|
3919
4012
|
const files = [`${p.entry}-meta.md`];
|
|
3920
4013
|
if (page.md !== null) {
|
|
3921
4014
|
writeAtomic(p.md, page.md);
|
|
@@ -3937,8 +4030,8 @@ async function libraryFetch(libraryDir, requested, args, converter, logs = {}, f
|
|
|
3937
4030
|
|
|
3938
4031
|
// src/verifycache.ts
|
|
3939
4032
|
import { createHash as createHash3 } from "crypto";
|
|
3940
|
-
import { existsSync as existsSync8, mkdirSync as
|
|
3941
|
-
import { dirname as dirname3, join as
|
|
4033
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync7, readFileSync as readFileSync9 } from "fs";
|
|
4034
|
+
import { dirname as dirname3, join as join10 } from "path";
|
|
3942
4035
|
var sha = (s) => createHash3("sha1").update(s).digest("hex");
|
|
3943
4036
|
var normalizeClaim = (claim) => claim.replace(/\s+/g, " ").trim();
|
|
3944
4037
|
function foldClaim(claim) {
|
|
@@ -3950,36 +4043,36 @@ ${id}
|
|
|
3950
4043
|
${model}
|
|
3951
4044
|
${threshold}
|
|
3952
4045
|
${text3}`).slice(0, 16);
|
|
3953
|
-
var verdictFile = (libraryDir, key) =>
|
|
3954
|
-
function readVerdict(
|
|
3955
|
-
if (!existsSync8(
|
|
4046
|
+
var verdictFile = (libraryDir, key) => join10(libraryDir, "verify", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
|
|
4047
|
+
function readVerdict(file3) {
|
|
4048
|
+
if (!existsSync8(file3)) return null;
|
|
3956
4049
|
try {
|
|
3957
|
-
return JSON.parse(
|
|
4050
|
+
return JSON.parse(readFileSync9(file3, "utf8"));
|
|
3958
4051
|
} catch {
|
|
3959
4052
|
return null;
|
|
3960
4053
|
}
|
|
3961
4054
|
}
|
|
3962
|
-
function lookup(
|
|
3963
|
-
const c = readVerdict(
|
|
4055
|
+
function lookup(file3, maxDays) {
|
|
4056
|
+
const c = readVerdict(file3);
|
|
3964
4057
|
return c && daysSince(c.date) < maxDays ? c : null;
|
|
3965
4058
|
}
|
|
3966
|
-
function remember(
|
|
3967
|
-
|
|
3968
|
-
writeAtomic(
|
|
4059
|
+
function remember(file3, c) {
|
|
4060
|
+
mkdirSync7(dirname3(file3), { recursive: true });
|
|
4061
|
+
writeAtomic(file3, JSON.stringify(c, null, 2) + "\n");
|
|
3969
4062
|
}
|
|
3970
|
-
async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR,
|
|
3971
|
-
const paper = await
|
|
4063
|
+
async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR, read2 = readPaper) {
|
|
4064
|
+
const paper = await read2(id, libraryDir);
|
|
3972
4065
|
const once = () => paper;
|
|
3973
4066
|
if ("error" in paper || !cfg) return { ...await verifyClaim(claim, id, cfg, decide, once), cached: false };
|
|
3974
4067
|
const text3 = textHash(paper.text);
|
|
3975
|
-
const
|
|
3976
|
-
const hit = lookup(
|
|
4068
|
+
const file3 = verdictFile(libraryDir, cacheKey(claim, paper.id, cfg.model, cfg.threshold, text3));
|
|
4069
|
+
const hit = lookup(file3, settings().refresh_after_days);
|
|
3977
4070
|
if (hit) {
|
|
3978
4071
|
return { claim, id: paper.id, verdict: hit.verdict, reason: hit.reason, score: hit.score, basis: hit.basis, windows: hit.windows, glyphs_unresolved: hit.glyphs_unresolved, superseded: hit.superseded, model: hit.model, cached: true };
|
|
3979
4072
|
}
|
|
3980
4073
|
const r = await verifyClaim(claim, id, cfg, decide, once);
|
|
3981
4074
|
if (!r.error && r.score !== null) {
|
|
3982
|
-
remember(
|
|
4075
|
+
remember(file3, { key: file3.slice(-21, -5), id: paper.id, claim: normalizeClaim(claim), model: cfg.model, threshold: cfg.threshold, text: text3, date: today(), verdict: r.verdict, reason: r.reason, score: r.score, basis: r.basis, windows: r.windows, glyphs_unresolved: r.glyphs_unresolved, superseded: r.superseded });
|
|
3983
4076
|
}
|
|
3984
4077
|
return { ...r, cached: false };
|
|
3985
4078
|
}
|
|
@@ -4001,11 +4094,35 @@ function leanObject(o) {
|
|
|
4001
4094
|
return out;
|
|
4002
4095
|
}
|
|
4003
4096
|
var leanJson = (o) => compact2(lean(o));
|
|
4097
|
+
var STATUS_DROP = /* @__PURE__ */ new Set(["requested", "type", "id", "entry", "retrieved_date", "pubmed_checked", "attempts", "coi", "funding", "title", "first_author", "doi"]);
|
|
4098
|
+
function statusAnswer(s) {
|
|
4099
|
+
const o = {};
|
|
4100
|
+
for (const [k, v] of Object.entries(s)) if (!STATUS_DROP.has(k)) o[k] = v;
|
|
4101
|
+
if (Array.isArray(o.publication_types)) o.publication_types = reviewTypes(o.publication_types);
|
|
4102
|
+
return leanJson(o);
|
|
4103
|
+
}
|
|
4104
|
+
function verdictAnswer(r) {
|
|
4105
|
+
if (r.error) return compact2({ verdict: "error", error: r.error });
|
|
4106
|
+
const o = { verdict: r.verdict, score: r.score === null ? null : Math.round(r.score * 100) / 100, basis: r.basis ?? null };
|
|
4107
|
+
if (r.reason) o.reason = r.reason;
|
|
4108
|
+
if (r.superseded) o.superseded = true;
|
|
4109
|
+
if (r.glyphs_unresolved) o.glyphs_unresolved = r.glyphs_unresolved;
|
|
4110
|
+
return compact2(o);
|
|
4111
|
+
}
|
|
4112
|
+
function readAnswer(o) {
|
|
4113
|
+
const { requested: _r, entry: _e, ...rest } = o;
|
|
4114
|
+
if (rest.partial === false) delete rest.partial;
|
|
4115
|
+
return leanJson(rest);
|
|
4116
|
+
}
|
|
4117
|
+
function searchAnswer(r) {
|
|
4118
|
+
return compact2({ total: r.total, columns: r.columns, rows: r.rows });
|
|
4119
|
+
}
|
|
4120
|
+
var reviewTypes = (types) => types.filter((t) => t !== "Journal Article" && !/^Research Support\b/.test(t));
|
|
4004
4121
|
|
|
4005
4122
|
// src/trials.ts
|
|
4006
4123
|
import { createHash as createHash4 } from "crypto";
|
|
4007
|
-
import { existsSync as existsSync9, mkdirSync as
|
|
4008
|
-
import { dirname as dirname4, join as
|
|
4124
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync8, readFileSync as readFileSync10 } from "fs";
|
|
4125
|
+
import { dirname as dirname4, join as join11 } from "path";
|
|
4009
4126
|
var searchCacheDays = () => settings().search_cache_days;
|
|
4010
4127
|
var API2 = "https://clinicaltrials.gov/api/v2/studies";
|
|
4011
4128
|
var FIELDS = ["NCTId", "BriefTitle", "OverallStatus", "WhyStopped", "Phase", "StudyType", "EnrollmentCount", "EnrollmentType", "StartDate", "PrimaryCompletionDate", "CompletionDate", "LastUpdatePostDate", "PrimaryOutcomeMeasure", "PrimaryOutcomeTimeFrame", "Condition", "InterventionName", "LeadSponsorName", "HasResults"];
|
|
@@ -4020,7 +4137,7 @@ var COLUMNS = [
|
|
|
4020
4137
|
// "Terminated — slow accrual": a trial past its completion date is not "ongoing" (ER.md 28.7)
|
|
4021
4138
|
"phase",
|
|
4022
4139
|
"enrollment",
|
|
4023
|
-
// "
|
|
4140
|
+
// "48" actual, "~100" estimated — participant count (28.5)
|
|
4024
4141
|
"start",
|
|
4025
4142
|
"primary_completion",
|
|
4026
4143
|
"primary_outcome",
|
|
@@ -4043,7 +4160,7 @@ function normalizeQuery(q) {
|
|
|
4043
4160
|
return out;
|
|
4044
4161
|
}
|
|
4045
4162
|
var searchKey = (q) => createHash4("sha1").update(JSON.stringify(["clinicaltrials", q.query ?? "", q.intervention ?? "", q.condition ?? "", q.status, q.sort, q.max_results])).digest("hex").slice(0, 16);
|
|
4046
|
-
var searchFile = (libraryDir, key) =>
|
|
4163
|
+
var searchFile = (libraryDir, key) => join11(libraryDir, "search", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
|
|
4047
4164
|
function searchUrl(q) {
|
|
4048
4165
|
const p = new URLSearchParams();
|
|
4049
4166
|
if (q.query) p.set("query.term", q.query);
|
|
@@ -4079,7 +4196,7 @@ function parseRows(json2) {
|
|
|
4079
4196
|
String(id.briefTitle ?? ""),
|
|
4080
4197
|
status,
|
|
4081
4198
|
phases.length ? phases.map((x) => PHASE2[x] ?? x).join(" / ") : null,
|
|
4082
|
-
typeof en.count === "number" ? `${
|
|
4199
|
+
typeof en.count === "number" ? `${String(en.type).toUpperCase() === "ESTIMATED" ? "~" : ""}${en.count}` : null,
|
|
4083
4200
|
st.startDateStruct?.date ?? null,
|
|
4084
4201
|
st.primaryCompletionDateStruct?.date ?? null,
|
|
4085
4202
|
primary,
|
|
@@ -4094,10 +4211,10 @@ var liveFetch = (url) => fetchText(url, { "User-Agent": LIBRARY_UA, Accept: "app
|
|
|
4094
4211
|
async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFetch) {
|
|
4095
4212
|
const q = normalizeQuery(raw);
|
|
4096
4213
|
if ("error" in q) return { ok: false, error: q.error };
|
|
4097
|
-
const
|
|
4098
|
-
if (existsSync9(
|
|
4214
|
+
const file3 = searchFile(libraryDir, searchKey(q));
|
|
4215
|
+
if (existsSync9(file3)) {
|
|
4099
4216
|
try {
|
|
4100
|
-
const r = JSON.parse(
|
|
4217
|
+
const r = JSON.parse(readFileSync10(file3, "utf8"));
|
|
4101
4218
|
if (daysSince(r.searched) < searchCacheDays()) return { ok: true, result: r, cached: true };
|
|
4102
4219
|
} catch {
|
|
4103
4220
|
}
|
|
@@ -4116,8 +4233,8 @@ async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFe
|
|
|
4116
4233
|
}
|
|
4117
4234
|
const result = { source: "clinicaltrials", query: q, total: parsed.total, columns: COLUMNS, rows: parsed.rows, searched: today() };
|
|
4118
4235
|
try {
|
|
4119
|
-
|
|
4120
|
-
writeAtomic(
|
|
4236
|
+
mkdirSync8(dirname4(file3), { recursive: true });
|
|
4237
|
+
writeAtomic(file3, JSON.stringify(result, null, 2) + "\n");
|
|
4121
4238
|
} catch {
|
|
4122
4239
|
}
|
|
4123
4240
|
return { ok: true, result, cached: false };
|
|
@@ -4129,7 +4246,7 @@ function trialsLogId(q) {
|
|
|
4129
4246
|
|
|
4130
4247
|
// src/pubmedsearch.ts
|
|
4131
4248
|
import { createHash as createHash5 } from "crypto";
|
|
4132
|
-
import { existsSync as existsSync10, mkdirSync as
|
|
4249
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync9, readFileSync as readFileSync11 } from "fs";
|
|
4133
4250
|
import { dirname as dirname5 } from "path";
|
|
4134
4251
|
var MAX_RESULTS2 = 100;
|
|
4135
4252
|
var DEFAULT_RESULTS2 = 20;
|
|
@@ -4198,7 +4315,7 @@ function parseEsummary(json2, ids, libraryDir) {
|
|
|
4198
4315
|
const year = /\b(\d{4})\b/.exec(String(s.pubdate ?? ""))?.[1] ?? null;
|
|
4199
4316
|
const first = String(s.sortfirstauthor ?? (Array.isArray(s.authors) && s.authors[0]?.name) ?? "").replace(/\s+[A-Z]{1,3}$/, "").trim() || null;
|
|
4200
4317
|
const pmc2 = (Array.isArray(s.articleids) ? s.articleids : []).find((a) => a.idtype === "pmc")?.value ?? null;
|
|
4201
|
-
const types = Array.isArray(s.pubtype) ? s.pubtype.
|
|
4318
|
+
const types = Array.isArray(s.pubtype) ? reviewTypes(s.pubtype.map(String)).join("; ") : "";
|
|
4202
4319
|
const paths = entryPaths(libraryDir, "pmid", id);
|
|
4203
4320
|
const level = entryExists(paths) ? String(readStatus(paths, emptyStatus(`pmid:${id}`, "pmid", id, libraryDir)).level) : null;
|
|
4204
4321
|
rows.push([id, String(s.title ?? "").replace(/\.$/, "") || null, first, year, s.source ? String(s.source) : null, types || null, pmc2 ? String(pmc2) : null, level]);
|
|
@@ -4209,10 +4326,10 @@ var live = (url) => fetchText(url, { "User-Agent": "fh-library", Accept: "applic
|
|
|
4209
4326
|
async function searchArticles(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = live) {
|
|
4210
4327
|
const q = normalizeQuery2(raw);
|
|
4211
4328
|
if ("error" in q) return { ok: false, error: q.error };
|
|
4212
|
-
const
|
|
4213
|
-
if (existsSync10(
|
|
4329
|
+
const file3 = searchFile(libraryDir, searchKey2(q));
|
|
4330
|
+
if (existsSync10(file3)) {
|
|
4214
4331
|
try {
|
|
4215
|
-
const r = JSON.parse(
|
|
4332
|
+
const r = JSON.parse(readFileSync11(file3, "utf8"));
|
|
4216
4333
|
if (daysSince(r.searched) < searchCacheDays()) return { ok: true, result: r, cached: true };
|
|
4217
4334
|
} catch {
|
|
4218
4335
|
}
|
|
@@ -4233,8 +4350,8 @@ async function searchArticles(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = live
|
|
|
4233
4350
|
}
|
|
4234
4351
|
const result = { source: "pubmed", query: q, total, columns: COLUMNS2, rows, searched: today() };
|
|
4235
4352
|
try {
|
|
4236
|
-
|
|
4237
|
-
writeAtomic(
|
|
4353
|
+
mkdirSync9(dirname5(file3), { recursive: true });
|
|
4354
|
+
writeAtomic(file3, JSON.stringify(result, null, 2) + "\n");
|
|
4238
4355
|
} catch {
|
|
4239
4356
|
}
|
|
4240
4357
|
return { ok: true, result, cached: false };
|
|
@@ -4290,11 +4407,10 @@ function recordSummary(json2) {
|
|
|
4290
4407
|
start: dated2(st.startDateStruct),
|
|
4291
4408
|
primary_completion: dated2(st.primaryCompletionDateStruct),
|
|
4292
4409
|
completion: dated2(st.completionDateStruct),
|
|
4293
|
-
first_posted: st.studyFirstPostDateStruct?.date ?? null,
|
|
4294
4410
|
last_update: st.lastUpdatePostDateStruct?.date ?? null,
|
|
4295
4411
|
results_first_posted: st.resultsFirstPostDateStruct?.date ?? null,
|
|
4296
4412
|
sponsor: para2(sp.leadSponsor?.name),
|
|
4297
|
-
collaborators:
|
|
4413
|
+
collaborators: collaboratorsLine(list3(sp.collaborators).map((c) => para2(c.name)).filter(Boolean)),
|
|
4298
4414
|
conditions: joined2(list3(ps.conditionsModule?.conditions).map((c) => para2(c))),
|
|
4299
4415
|
interventions: joined2(list3(ai.interventions).map((i) => para2(`${i.type ? `${words(i.type)}: ` : ""}${i.name ?? ""}`))),
|
|
4300
4416
|
primary_outcomes: list3(om.primaryOutcomes).map(outcome).filter(Boolean),
|
|
@@ -4306,23 +4422,69 @@ function recordSummary(json2) {
|
|
|
4306
4422
|
summary: para2(de.briefSummary, 400)
|
|
4307
4423
|
};
|
|
4308
4424
|
}
|
|
4425
|
+
function collaboratorsLine(names) {
|
|
4426
|
+
if (!names.length) return null;
|
|
4427
|
+
return names.slice(0, 3).join("; ") + (names.length > 3 ? ` +${names.length - 3}` : "");
|
|
4428
|
+
}
|
|
4429
|
+
var ORDER = ["Outcome measures", "Adverse events", "Groups", "Participant flow", "Baseline characteristics"];
|
|
4430
|
+
function reorderResults(part) {
|
|
4431
|
+
const chunks = part.split(/^(?=### )/m).map((c) => c.trim()).filter(Boolean);
|
|
4432
|
+
const groups = /* @__PURE__ */ new Map();
|
|
4433
|
+
const sections2 = [];
|
|
4434
|
+
for (const c of chunks) {
|
|
4435
|
+
const name = /^### (.+)$/m.exec(c)?.[1].trim() ?? "";
|
|
4436
|
+
const lines = c.split("\n").slice(1);
|
|
4437
|
+
let a = lines.findIndex((l) => /^- \*\*[^*]+\*\*/.test(l));
|
|
4438
|
+
let b = a < 0 ? -1 : lines.findIndex((l, i) => i > a && (/^\|/.test(l) || /^\*\*/.test(l) || /^#### /.test(l)));
|
|
4439
|
+
if (a >= 0 && b < 0) b = lines.length;
|
|
4440
|
+
if (a < 0) {
|
|
4441
|
+
sections2.push({ name, body: lines.join("\n").trim() });
|
|
4442
|
+
continue;
|
|
4443
|
+
}
|
|
4444
|
+
const kept = [];
|
|
4445
|
+
let cur = [];
|
|
4446
|
+
const flush = () => {
|
|
4447
|
+
if (!cur.length) return;
|
|
4448
|
+
const block = cur.join("\n").trim();
|
|
4449
|
+
const title = /^- \*\*([^*]+)\*\*/.exec(block)[1];
|
|
4450
|
+
if (!groups.has(title)) groups.set(title, block);
|
|
4451
|
+
else if (groups.get(title) !== block) kept.push(block);
|
|
4452
|
+
cur = [];
|
|
4453
|
+
};
|
|
4454
|
+
for (const l of lines.slice(a, b)) {
|
|
4455
|
+
if (/^- \*\*[^*]+\*\*/.test(l)) flush();
|
|
4456
|
+
cur.push(l);
|
|
4457
|
+
}
|
|
4458
|
+
flush();
|
|
4459
|
+
const body = [...lines.slice(0, a), ...kept, ...lines.slice(b)].join("\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
4460
|
+
sections2.push({ name, body });
|
|
4461
|
+
}
|
|
4462
|
+
if (groups.size) sections2.push({ name: "Groups", body: [...groups.values()].join("\n\n") });
|
|
4463
|
+
const rank = (n) => {
|
|
4464
|
+
const i = ORDER.indexOf(n);
|
|
4465
|
+
return i < 0 ? ORDER.length : i;
|
|
4466
|
+
};
|
|
4467
|
+
return sections2.map((x, i) => ({ ...x, i })).sort((x, y) => rank(x.name) - rank(y.name) || x.i - y.i).filter((x) => x.body).map((x) => `### ${x.name}
|
|
4468
|
+
|
|
4469
|
+
${x.body}`).join("\n\n");
|
|
4470
|
+
}
|
|
4309
4471
|
function resultsText(nct, md, maxChars) {
|
|
4310
4472
|
const i = md.search(/^## Results\s*$/m);
|
|
4311
4473
|
if (i < 0) return { nct, has_results: false, text: null, truncated: false, outcomes: [], note: "no posted results in the registry record" };
|
|
4312
|
-
const part = md.slice(i).replace(/^## Results\s*\n/, "").trim();
|
|
4474
|
+
const part = reorderResults(md.slice(i).replace(/^## Results\s*\n/, "").replace(/\n## [\s\S]*$/, "").trim());
|
|
4313
4475
|
const outcomes = [...part.matchAll(/^#### (.+)$/gm)].map((m) => m[1].trim());
|
|
4314
4476
|
const truncated = part.length > maxChars;
|
|
4315
4477
|
return { nct, has_results: true, text: truncated ? part.slice(0, maxChars).trimEnd() + "\n\n[truncated]" : part, truncated, outcomes, note: truncated ? "cut at max_chars; library_read(id, section: <an outcome title>) returns one measure whole" : null };
|
|
4316
4478
|
}
|
|
4317
4479
|
|
|
4318
4480
|
// src/missing.ts
|
|
4319
|
-
import { existsSync as existsSync11, mkdirSync as
|
|
4320
|
-
import { join as
|
|
4321
|
-
var missingFile = (p) =>
|
|
4481
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync10, readFileSync as readFileSync12 } from "fs";
|
|
4482
|
+
import { join as join12 } from "path";
|
|
4483
|
+
var missingFile = (p) => join12(p.entryDir, `${p.entry}-missing.md`);
|
|
4322
4484
|
function readMissing(p) {
|
|
4323
4485
|
const f = missingFile(p);
|
|
4324
4486
|
if (!existsSync11(f)) return null;
|
|
4325
|
-
const md =
|
|
4487
|
+
const md = readFileSync12(f, "utf8");
|
|
4326
4488
|
const get = (k) => /^$/.test(k) ? "" : (new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md)?.[1] ?? "").trim();
|
|
4327
4489
|
const checked = get("checked"), reason = get("reason");
|
|
4328
4490
|
return checked ? { checked, reason: reason || "not found" } : null;
|
|
@@ -4333,7 +4495,7 @@ function freshMissing(p, retryDays) {
|
|
|
4333
4495
|
}
|
|
4334
4496
|
function rememberMissing(p, reason) {
|
|
4335
4497
|
const m = { checked: today(), reason };
|
|
4336
|
-
|
|
4498
|
+
mkdirSync10(p.entryDir, { recursive: true });
|
|
4337
4499
|
writeAtomic(missingFile(p), `checked: ${m.checked}
|
|
4338
4500
|
reason: ${m.reason}
|
|
4339
4501
|
`);
|
|
@@ -4365,7 +4527,7 @@ function splitArticles(xml) {
|
|
|
4365
4527
|
}
|
|
4366
4528
|
return out;
|
|
4367
4529
|
}
|
|
4368
|
-
var row2 = (pmid, s, level) => s ? [pmid, s.title, s.first_author, s.year, s.publication_types.
|
|
4530
|
+
var row2 = (pmid, s, level) => s ? [pmid, s.title ? s.title.replace(/\.$/, "") : s.title, s.first_author, s.year, reviewTypes(s.publication_types).join("; ") || null, s.retracted, level] : [pmid, null, null, null, null, null, level];
|
|
4369
4531
|
async function fetchArticles(ids, libraryDir = FH_LIBRARY_DIR, converter = "fh-library", fetchImpl = fetchText) {
|
|
4370
4532
|
const result = { columns: COLUMNS3, rows: [], new: 0, cached: 0, not_found: 0, invalid: 0 };
|
|
4371
4533
|
const order2 = [];
|
|
@@ -4428,7 +4590,7 @@ async function fetchArticles(ids, libraryDir = FH_LIBRARY_DIR, converter = "fh-l
|
|
|
4428
4590
|
// src/tools.ts
|
|
4429
4591
|
var pkg = (() => {
|
|
4430
4592
|
try {
|
|
4431
|
-
return JSON.parse(
|
|
4593
|
+
return JSON.parse(readFileSync13(new URL("../package.json", import.meta.url), "utf8"));
|
|
4432
4594
|
} catch {
|
|
4433
4595
|
return { name: "fh-library", version: "0.0.0" };
|
|
4434
4596
|
}
|
|
@@ -4458,7 +4620,7 @@ Tools
|
|
|
4458
4620
|
Hints
|
|
4459
4621
|
- library_read and library_verify store a missing record themselves; the library_read answer then carries added: true. No library_add necessary first.
|
|
4460
4622
|
- citable false \u2014 the paper is retracted (a retraction notice, or PubMed's publication type), or holds nothing to read: no abstract and no full text. comment says which; corrections lists the notices as {type, pmid, source}.
|
|
4461
|
-
- level \u2014 what is stored: "full_text", "abstract" (no full text could be found;
|
|
4623
|
+
- level \u2014 what is stored: "full_text", "abstract" (no full text could be found; the log names each source tried and why it failed) or "title" (not even an abstract; citable false).
|
|
4462
4624
|
- error "pmid not found", "doi not found", "ctgov: not a study record", "dailymed: not an SPL document" \u2014 no service knows the ID. Remembered for a day (FH_LIBRARY_RETRY_AFTER_DAYS): the same call answers from disk with the date it was checked; after that the service is asked again. "invalid id" \u2014 the ID itself is malformed; the same call gives the same answer.
|
|
4463
4625
|
- Calls may run in parallel: library_add, library_read and library_verify. Two calls on the same missing record share one add.
|
|
4464
4626
|
- library_fetch takes any URL that is not a paper, label or trial record; those have IDs, and library_add, library_read and library_verify work on the stored document. A second pass over the same review reads every page from disk; a failed fetch is remembered for a day.
|
|
@@ -4483,14 +4645,14 @@ Reading a document
|
|
|
4483
4645
|
- partial true \u2014 no passage holds every term; missing_terms are the words the document never uses. The same query without them finds the passage if there is one.
|
|
4484
4646
|
- Numbers are often in tables; a matching table row comes with its header row.
|
|
4485
4647
|
- section and query together search inside that section only.
|
|
4486
|
-
- section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name.
|
|
4648
|
+
- section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name, else the statements of the PubMed or Crossref record.
|
|
4487
4649
|
|
|
4488
|
-
Status fields:
|
|
4650
|
+
Status fields: pmid, pmcid, setid, nctid, publications, year, publication_types, retracted, corrections, citable, comment, has_abstract, level, version, retrieved_by, glyphs_unresolved, error. Funding and conflicts of interest: library_read section "Declarations". Answers carry only what the call did not send: no echo of the ID, claim or query; a field that is null or an empty list is left out. library_verify always has verdict, score (null: no judgement) and basis.
|
|
4489
4651
|
|
|
4490
4652
|
Where records come from: PubMed for the record and abstract; the full text from PubMed Central, Europe PMC, publishers' text-mining links, Unpaywall, OpenAlex, CORE, and a proxy URL template from the configuration. Every action is appended as one line to <library>/library.log.`;
|
|
4491
4653
|
var idSchema = z.string().describe('ID as "type:value": a paper "pmid:30955454" (bare digits count as pmid) or "doi:10.5812/asjsm.110753" (a paper PubMed does not index; a DOI PubMed indexes is taken as its PMID), a drug label "dailymed:<set ID>" (the set ID from its DailyMed page URL), or a trial record "nct:05173896" (the eight digits; nct:NCT05173896 and a bare NCT05173896 are accepted).');
|
|
4492
4654
|
var levelWord = (m) => m.level === "abstract" && !m.has_abstract ? "title" : m.level;
|
|
4493
|
-
var json = (s) => ({ content: [{ type: "text", text:
|
|
4655
|
+
var json = (s) => ({ content: [{ type: "text", text: statusAnswer(s) }] });
|
|
4494
4656
|
function noticeTag(m) {
|
|
4495
4657
|
const n = (t) => m.corrections.filter((c) => c.type === t).length;
|
|
4496
4658
|
const parts = [
|
|
@@ -4798,7 +4960,7 @@ function registerTools(server2) {
|
|
|
4798
4960
|
);
|
|
4799
4961
|
server2.tool(
|
|
4800
4962
|
"library_read",
|
|
4801
|
-
`One piece of a stored document's Markdown, never the whole. No section/query: the outline (headings with word counts; full text or abstract only). section: that heading's own text (case-insensitive substring; a repeated heading as "Participants [3/12]"), capped at max_chars, its subsections listed. query: passages holding every term, each a window of at most 1000 chars with its section, best first, at most max_hits; partial: true with missing_terms when no passage holds them all. section and query together: inside that section. A missing paper is added first (added: true).`,
|
|
4963
|
+
`One piece of a stored document's Markdown, never the whole. No section/query: the outline (headings with word counts; full text or abstract only). section: that heading's own text (case-insensitive substring; a repeated heading as "Participants [3/12]"), capped at max_chars, its subsections listed. query: passages holding every term, each a window of at most 1000 chars with its section, best first, at most max_hits (more: how many further passages match); partial: true with missing_terms when no passage holds them all. section and query together: inside that section. A missing paper is added first (added: true).`,
|
|
4802
4964
|
{
|
|
4803
4965
|
id: idSchema,
|
|
4804
4966
|
section: z.string().optional().describe("Heading from the outline (case-insensitive substring)."),
|
|
@@ -4808,7 +4970,7 @@ function registerTools(server2) {
|
|
|
4808
4970
|
},
|
|
4809
4971
|
async ({ id, section: section2, query, max_chars, max_hits }) => {
|
|
4810
4972
|
const parsed = parseId(id);
|
|
4811
|
-
const out = (o2) => ({ content: [{ type: "text", text:
|
|
4973
|
+
const out = (o2) => ({ content: [{ type: "text", text: readAnswer(o2) }] });
|
|
4812
4974
|
if (parsed.error) {
|
|
4813
4975
|
log("library_read", parsed.requested, `error ${parsed.error}`);
|
|
4814
4976
|
return out({ requested: parsed.requested, error: parsed.error });
|
|
@@ -4825,7 +4987,7 @@ function registerTools(server2) {
|
|
|
4825
4987
|
added = true;
|
|
4826
4988
|
}
|
|
4827
4989
|
const full = existsSync12(paths.fulltextMd);
|
|
4828
|
-
const md =
|
|
4990
|
+
const md = readFileSync13(full ? paths.fulltextMd : paths.md, "utf8");
|
|
4829
4991
|
const text3 = full ? "full_text" : "abstract";
|
|
4830
4992
|
const base = added ? { requested: parsed.requested, entry: paths.entry, text: text3, added: true } : { requested: parsed.requested, entry: paths.entry, text: text3 };
|
|
4831
4993
|
const headings2 = () => outline(md, text3).sections;
|
|
@@ -4846,13 +5008,25 @@ function registerTools(server2) {
|
|
|
4846
5008
|
let sec = readSection(md, section2, cap);
|
|
4847
5009
|
let via = "";
|
|
4848
5010
|
if (!sec && /abstract/i.test(section2) && full && existsSync12(paths.md)) {
|
|
4849
|
-
sec = readSection(
|
|
5011
|
+
sec = readSection(readFileSync13(paths.md, "utf8"), "abstract", cap);
|
|
4850
5012
|
via = " (from the abstract file)";
|
|
4851
5013
|
}
|
|
4852
5014
|
if (!sec && /declaration/i.test(section2)) {
|
|
4853
5015
|
sec = assembleDeclarations(md, cap);
|
|
4854
5016
|
via = " (assembled)";
|
|
4855
5017
|
}
|
|
5018
|
+
if (!sec && /declaration/i.test(section2)) {
|
|
5019
|
+
const st = readStatus(paths, emptyStatus(parsed.requested, parsed.type, parsed.id, FH_LIBRARY_DIR));
|
|
5020
|
+
const body = [st.coi ? `### Conflicts of interest
|
|
5021
|
+
|
|
5022
|
+
${st.coi}` : "", st.funding?.length ? `### Funding
|
|
5023
|
+
|
|
5024
|
+
${st.funding.map((f) => `- ${f}`).join("\n")}` : ""].filter(Boolean).join("\n\n");
|
|
5025
|
+
if (body) {
|
|
5026
|
+
sec = { heading: "Declarations (from the record)", words: body.split(/\s+/).length, text: body, returned_words: body.split(/\s+/).length, truncated: false };
|
|
5027
|
+
via = " (from the record)";
|
|
5028
|
+
}
|
|
5029
|
+
}
|
|
4856
5030
|
if (!sec) {
|
|
4857
5031
|
log("library_read", parsed.requested, `section "${section2}" not found`);
|
|
4858
5032
|
return out({ ...base, error: `no section matching "${section2}"`, outline: headings2() });
|
|
@@ -4894,7 +5068,7 @@ function registerTools(server2) {
|
|
|
4894
5068
|
if (!r.error && r.score !== null) bump("verify", cached);
|
|
4895
5069
|
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
4896
5070
|
log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${cached ? " cached" : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
4897
|
-
return { content: [{ type: "text", text:
|
|
5071
|
+
return { content: [{ type: "text", text: verdictAnswer(r) }] };
|
|
4898
5072
|
}
|
|
4899
5073
|
);
|
|
4900
5074
|
server2.tool(
|
|
@@ -4957,8 +5131,8 @@ function registerTools(server2) {
|
|
|
4957
5131
|
return { content: [{ type: "text", text: `Failed to fetch ${url}: ${added.error}` }], isError: true };
|
|
4958
5132
|
}
|
|
4959
5133
|
}
|
|
4960
|
-
const
|
|
4961
|
-
const md = existsSync12(
|
|
5134
|
+
const file3 = pid.type === "pmid" ? paths.md : existsSync12(paths.fulltextMd) ? paths.fulltextMd : paths.md;
|
|
5135
|
+
const md = existsSync12(file3) ? readFileSync13(file3, "utf8") : "";
|
|
4962
5136
|
const text3 = title_only ? `Contents of ${url}:
|
|
4963
5137
|
${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "", max_length, start_index);
|
|
4964
5138
|
bump("fetch", had);
|
|
@@ -4990,7 +5164,7 @@ ${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "
|
|
|
4990
5164
|
if (r.new) bump("pubmed", false, r.new);
|
|
4991
5165
|
const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
|
|
4992
5166
|
log("library_pubmed", `pmids:${r.rows.length}`, `articles ${secs} cached=${r.cached} new=${r.new}${r.not_found ? ` not_found=${r.not_found}` : ""}${r.invalid ? ` invalid=${r.invalid}` : ""}`);
|
|
4993
|
-
return { content: [{ type: "text", text:
|
|
5167
|
+
return { content: [{ type: "text", text: compact2({ columns: r.columns, rows: r.rows }) }] };
|
|
4994
5168
|
}
|
|
4995
5169
|
);
|
|
4996
5170
|
server2.tool(
|
|
@@ -5013,12 +5187,12 @@ ${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "
|
|
|
5013
5187
|
bump("search", r.cached);
|
|
5014
5188
|
const q = r.result.query;
|
|
5015
5189
|
log("library_search", `pm:${logId2(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""}${q.years ? ` years=${q.years}` : ""} | ${q.query}`);
|
|
5016
|
-
return { content: [{ type: "text", text:
|
|
5190
|
+
return { content: [{ type: "text", text: searchAnswer(r.result) }] };
|
|
5017
5191
|
}
|
|
5018
5192
|
);
|
|
5019
5193
|
server2.tool(
|
|
5020
5194
|
"library_clinicaltrials_search_studies",
|
|
5021
|
-
`Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment, start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${searchCacheDays()} days. One trial in detail: library_clinicaltrials_get_study_record.`,
|
|
5195
|
+
`Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment ("~100" estimated), start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${searchCacheDays()} days. One trial in detail: library_clinicaltrials_get_study_record.`,
|
|
5022
5196
|
{
|
|
5023
5197
|
query: z.string().optional().describe("Free-text search across all fields (title, conditions, interventions, summary, \u2026); AND / OR / NOT allowed."),
|
|
5024
5198
|
intervention: z.string().optional().describe('Intervention or treatment, e.g. "tadalafil", "cognitive behavioral therapy".'),
|
|
@@ -5038,7 +5212,7 @@ ${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "
|
|
|
5038
5212
|
bump("search", r.cached);
|
|
5039
5213
|
const q = r.result.query, parts = [q.intervention && `intervention=${q.intervention}`, q.condition && `condition=${q.condition}`, q.query && `query=${q.query}`].filter(Boolean).join(" ");
|
|
5040
5214
|
log("library_search", `ct:${trialsLogId(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total} status=${q.status}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""} | ${parts}`);
|
|
5041
|
-
return { content: [{ type: "text", text:
|
|
5215
|
+
return { content: [{ type: "text", text: searchAnswer(r.result) }] };
|
|
5042
5216
|
}
|
|
5043
5217
|
);
|
|
5044
5218
|
const nctEntry = async (id) => {
|
|
@@ -5065,7 +5239,7 @@ ${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "
|
|
|
5065
5239
|
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
5066
5240
|
}
|
|
5067
5241
|
const raw = e.paths.fulltextRaw("json");
|
|
5068
|
-
const r = existsSync12(raw) ? recordSummary(
|
|
5242
|
+
const r = existsSync12(raw) ? recordSummary(readFileSync13(raw, "utf8")) : null;
|
|
5069
5243
|
if (!r) {
|
|
5070
5244
|
log("library_trials", e.requested, "record error: no stored record");
|
|
5071
5245
|
return { content: [{ type: "text", text: compact2({ error: "no stored record" }) }], isError: true };
|
|
@@ -5084,7 +5258,7 @@ ${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "
|
|
|
5084
5258
|
log("library_trials", nct, `results error: ${e.error}`);
|
|
5085
5259
|
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
5086
5260
|
}
|
|
5087
|
-
const md = existsSync12(e.paths.fulltextMd) ?
|
|
5261
|
+
const md = existsSync12(e.paths.fulltextMd) ? readFileSync13(e.paths.fulltextMd, "utf8") : "";
|
|
5088
5262
|
const r = resultsText(e.requested.replace(/^nct:/, "NCT"), md, max_chars);
|
|
5089
5263
|
log("library_trials", e.requested, `results ${r.has_results ? `${r.outcomes.length} outcomes${r.truncated ? " truncated" : ""}` : "none"}`);
|
|
5090
5264
|
return { content: [{ type: "text", text: leanJson(r) }] };
|