fh-library 0.2.16 → 0.2.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -130
- package/README.md +12 -7
- package/dist/index.js +1821 -1650
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,10 +11,10 @@ function dotenvFile(env = process.env) {
|
|
|
11
11
|
return join(resolve(lib), ".env");
|
|
12
12
|
}
|
|
13
13
|
var loadedFromDotenv = /* @__PURE__ */ new Set();
|
|
14
|
-
function loadDotenv(
|
|
15
|
-
if (!existsSync(
|
|
14
|
+
function loadDotenv(file3 = dotenvFile()) {
|
|
15
|
+
if (!existsSync(file3)) return [];
|
|
16
16
|
const loaded = [];
|
|
17
|
-
for (const raw of readFileSync(
|
|
17
|
+
for (const raw of readFileSync(file3, "utf8").split(/\r?\n/)) {
|
|
18
18
|
const line = raw.trim();
|
|
19
19
|
if (!line || line.startsWith("#")) continue;
|
|
20
20
|
const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
@@ -1208,1729 +1208,1747 @@ async function unlock(url, postImpl = post) {
|
|
|
1208
1208
|
return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
|
|
1209
1209
|
}
|
|
1210
1210
|
|
|
1211
|
-
// src/
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
if (v === "submittedVersion") return "preprint";
|
|
1215
|
-
return "published";
|
|
1216
|
-
}
|
|
1217
|
-
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
1218
|
-
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
1211
|
+
// src/walls.ts
|
|
1212
|
+
import { mkdirSync as mkdirSync3, readFileSync as readFileSync4 } from "fs";
|
|
1213
|
+
import { join as join5 } from "path";
|
|
1219
1214
|
|
|
1220
|
-
// src/
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1215
|
+
// src/entry.ts
|
|
1216
|
+
import { copyFileSync, existsSync as existsSync4, mkdirSync as mkdirSync2, readdirSync, readFileSync as readFileSync3, renameSync, rmSync, unlinkSync, writeFileSync } from "fs";
|
|
1217
|
+
import { join as join4 } from "path";
|
|
1218
|
+
import { basename, dirname } from "path";
|
|
1219
|
+
|
|
1220
|
+
// src/pubmed.ts
|
|
1221
|
+
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
1222
|
+
var CORRECTION_TYPES = {
|
|
1223
|
+
ErratumIn: "erratum_in",
|
|
1224
|
+
RetractionIn: "retraction_in",
|
|
1225
|
+
PartialRetractionIn: "partial_retraction_in",
|
|
1226
|
+
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
1227
|
+
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
1228
|
+
UpdateIn: "update_in"
|
|
1227
1229
|
};
|
|
1228
|
-
var
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
ctx.tried.add(c.url);
|
|
1241
|
-
let r = await tryOne(p, c, tier);
|
|
1242
|
-
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
1243
|
-
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
1244
|
-
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
1245
|
-
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
1246
|
-
for (const b of browserRungs()) {
|
|
1247
|
-
if (r.ok || c.headers || c.service || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
1248
|
-
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
1249
|
-
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
1250
|
-
if (!r.ok && bare(r.reason).startsWith(MISMATCH)) ctx.onAlert?.(`browser ${b.label}: ${bare(r.reason)}`);
|
|
1251
|
-
}
|
|
1252
|
-
if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
1253
|
-
r = await tryOne(p, target, tier, "unlocker");
|
|
1254
|
-
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
1255
|
-
if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
|
|
1256
|
-
}
|
|
1257
|
-
if (r.ok) return r;
|
|
1258
|
-
reasons.push(`${r.reason}: ${hostOf2(c.url)}`);
|
|
1259
|
-
}
|
|
1260
|
-
return miss(tier, reasons.join("; "));
|
|
1230
|
+
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
1231
|
+
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
1232
|
+
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
1233
|
+
function efetchUrl(pmid) {
|
|
1234
|
+
const u = new URL(EFETCH);
|
|
1235
|
+
u.searchParams.set("db", "pubmed");
|
|
1236
|
+
u.searchParams.set("id", pmid);
|
|
1237
|
+
u.searchParams.set("retmode", "xml");
|
|
1238
|
+
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
1239
|
+
u.searchParams.set("tool", "fh-library");
|
|
1240
|
+
u.searchParams.set("email", contactEmail());
|
|
1241
|
+
return u.toString();
|
|
1261
1242
|
}
|
|
1262
|
-
|
|
1263
|
-
async function tryOne(p, c, tier, via = "plain") {
|
|
1264
|
-
let got;
|
|
1243
|
+
async function fetchPubmedXml(pmid) {
|
|
1265
1244
|
try {
|
|
1266
|
-
|
|
1245
|
+
return await fetchText(efetchUrl(pmid));
|
|
1267
1246
|
} catch (e) {
|
|
1268
|
-
|
|
1269
|
-
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
1270
|
-
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
1247
|
+
throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
1271
1248
|
}
|
|
1272
|
-
const { bytes, contentType } = got;
|
|
1273
|
-
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
1274
|
-
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
1275
|
-
const r = await convertBytes(p, bytes, contentType);
|
|
1276
|
-
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
1277
|
-
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
1278
1249
|
}
|
|
1279
|
-
var
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1250
|
+
var parser2 = new XMLParser2({
|
|
1251
|
+
ignoreAttributes: false,
|
|
1252
|
+
attributeNamePrefix: "@_",
|
|
1253
|
+
parseTagValue: false,
|
|
1254
|
+
processEntities: true,
|
|
1255
|
+
stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
|
|
1256
|
+
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
1257
|
+
});
|
|
1258
|
+
function text(n) {
|
|
1259
|
+
if (n == null) return "";
|
|
1260
|
+
if (typeof n === "object") return text(n["#text"]);
|
|
1261
|
+
return String(n);
|
|
1262
|
+
}
|
|
1263
|
+
var plain = (n) => decodeEntities(text(n)).trim();
|
|
1264
|
+
function inlineText2(raw) {
|
|
1265
|
+
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
1266
|
+
}
|
|
1267
|
+
function decodeEntities(s) {
|
|
1268
|
+
return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
1269
|
+
}
|
|
1270
|
+
function parsePubmedXml(xml) {
|
|
1271
|
+
const doc = parser2.parse(xml);
|
|
1272
|
+
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
1273
|
+
if (!article) {
|
|
1274
|
+
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
1275
|
+
return book ? parseBook(book) : null;
|
|
1288
1276
|
}
|
|
1289
|
-
const
|
|
1290
|
-
|
|
1277
|
+
const cit = article.MedlineCitation ?? {};
|
|
1278
|
+
const art = cit.Article ?? {};
|
|
1279
|
+
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
1280
|
+
const idOf = (t) => {
|
|
1281
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1282
|
+
return hit ? text(hit).trim() || null : null;
|
|
1283
|
+
};
|
|
1284
|
+
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
1285
|
+
const authors = authorList.map((a) => {
|
|
1286
|
+
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
1287
|
+
const last = plain(a.LastName);
|
|
1288
|
+
const ini = plain(a.Initials);
|
|
1289
|
+
return [last, ini].filter(Boolean).join(" ");
|
|
1290
|
+
}).filter(Boolean);
|
|
1291
|
+
const first = authorList[0];
|
|
1292
|
+
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
1293
|
+
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
1294
|
+
let year = text(pub.Year).trim() || null;
|
|
1295
|
+
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
1296
|
+
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
1297
|
+
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
1298
|
+
text: inlineText2(text(a))
|
|
1299
|
+
})).filter((a) => a.text);
|
|
1300
|
+
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
1301
|
+
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
1302
|
+
if (!type) return [];
|
|
1303
|
+
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
|
|
1304
|
+
});
|
|
1305
|
+
const funding = [];
|
|
1306
|
+
for (const g of art.GrantList?.Grant ?? []) {
|
|
1307
|
+
const a = plain(g.Agency);
|
|
1308
|
+
if (a && !funding.includes(a)) funding.push(a);
|
|
1309
|
+
}
|
|
1310
|
+
return {
|
|
1311
|
+
coi: inlineText2(text(cit.CoiStatement)) || null,
|
|
1312
|
+
funding,
|
|
1313
|
+
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
1314
|
+
pmcid: idOf("pmc"),
|
|
1315
|
+
doi: idOf("doi"),
|
|
1316
|
+
title: inlineText2(text(art.ArticleTitle)),
|
|
1317
|
+
authors,
|
|
1318
|
+
firstAuthor,
|
|
1319
|
+
journal: plain(art.Journal?.Title) || null,
|
|
1320
|
+
year,
|
|
1321
|
+
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
1322
|
+
abstract,
|
|
1323
|
+
corrections
|
|
1324
|
+
};
|
|
1291
1325
|
}
|
|
1292
|
-
function
|
|
1293
|
-
const
|
|
1294
|
-
const
|
|
1295
|
-
const
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1326
|
+
function parseBook(b) {
|
|
1327
|
+
const d = b.BookDocument ?? {};
|
|
1328
|
+
const book = d.Book ?? {};
|
|
1329
|
+
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
1330
|
+
const idOf = (t) => {
|
|
1331
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
1332
|
+
return hit ? text(hit).trim() || null : null;
|
|
1333
|
+
};
|
|
1334
|
+
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
1335
|
+
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
1336
|
+
const people = pick?.Author ?? [];
|
|
1337
|
+
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
1338
|
+
const first = people[0];
|
|
1339
|
+
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
|
|
1340
|
+
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
1341
|
+
return {
|
|
1342
|
+
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
1343
|
+
pmcid: null,
|
|
1344
|
+
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
1345
|
+
title: inlineText2(text(d.ArticleTitle)) || inlineText2(text(book.BookTitle)),
|
|
1346
|
+
authors,
|
|
1347
|
+
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
1348
|
+
journal: plain(book.Publisher?.PublisherName) || null,
|
|
1349
|
+
year: text(book.PubDate?.Year).trim() || null,
|
|
1350
|
+
publicationTypes: [...new Set(types)],
|
|
1351
|
+
abstract,
|
|
1352
|
+
corrections: [],
|
|
1353
|
+
coi: null,
|
|
1354
|
+
funding: []
|
|
1355
|
+
};
|
|
1299
1356
|
}
|
|
1300
|
-
async function convertBytes(p, bytes, contentType = "") {
|
|
1301
|
-
let format;
|
|
1302
|
-
let md;
|
|
1303
|
-
if (looksLikePdf(bytes)) {
|
|
1304
|
-
format = "pdf";
|
|
1305
|
-
const stats2 = {};
|
|
1306
|
-
try {
|
|
1307
|
-
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
|
|
1308
|
-
} catch (e) {
|
|
1309
|
-
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
1310
|
-
}
|
|
1311
|
-
if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
1312
|
-
} else {
|
|
1313
|
-
const text3 = bytes.toString("utf8");
|
|
1314
|
-
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
1315
|
-
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
1316
|
-
format = "xml";
|
|
1317
|
-
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
1318
|
-
if (!doc.title) doc.title = p.title;
|
|
1319
|
-
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
1320
|
-
md = renderFulltextMd(doc, p.headLine);
|
|
1321
|
-
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
1322
|
-
format = "html";
|
|
1323
|
-
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
1324
|
-
md = `# ${p.title}
|
|
1325
1357
|
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
1335
|
-
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
1336
|
-
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
1337
|
-
return { ok: true, format, md };
|
|
1358
|
+
// src/spl.ts
|
|
1359
|
+
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
1360
|
+
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
1361
|
+
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
1362
|
+
async function fetchSpl(setid) {
|
|
1363
|
+
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
1364
|
+
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
1365
|
+
return xml;
|
|
1338
1366
|
}
|
|
1339
|
-
var
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
[/ce:sup\b/g, "sup"],
|
|
1354
|
-
[/ce:inf\b/g, "sub"],
|
|
1355
|
-
[/ce:cross-refs?\b/g, "xref"],
|
|
1356
|
-
[/ce:list-item\b/g, "list-item"],
|
|
1357
|
-
[/ce:list\b/g, "list"],
|
|
1358
|
-
[/ce:label\b/g, "label"],
|
|
1359
|
-
[/ce:caption\b/g, "caption"],
|
|
1360
|
-
[/ce:table\b/g, "table-wrap"],
|
|
1361
|
-
[/ce:figure\b/g, "fig"],
|
|
1362
|
-
[/ce:bib-reference\b/g, "ref"],
|
|
1363
|
-
[/ce:bibliography\b/g, "ref-list"],
|
|
1364
|
-
[/sb:reference\b/g, "mixed-citation"],
|
|
1365
|
-
[/ce:acknowledgment\b/g, "ack"],
|
|
1366
|
-
[/ce:inter-ref\b/g, "ext-link"]
|
|
1367
|
-
];
|
|
1368
|
-
function elsevierToJats(xml) {
|
|
1369
|
-
let out = xml;
|
|
1370
|
-
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
1371
|
-
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
1372
|
-
}
|
|
1373
|
-
|
|
1374
|
-
// src/tiers/core.ts
|
|
1375
|
-
var WINDOW_MS = 6e4;
|
|
1376
|
-
var PER_WINDOW = 10;
|
|
1377
|
-
var MAX_WAIT_MS = 65e3;
|
|
1378
|
-
var TIMEOUT_MS3 = 1e4;
|
|
1379
|
-
var calls = [];
|
|
1380
|
-
async function pace() {
|
|
1381
|
-
const now = Date.now();
|
|
1382
|
-
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
1383
|
-
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
1384
|
-
calls.push(Date.now());
|
|
1367
|
+
var parser3 = new XMLParser3({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
|
|
1368
|
+
var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
1369
|
+
var kids2 = (n) => n[tagOf2(n)] ?? [];
|
|
1370
|
+
var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
1371
|
+
var squash2 = (s) => s.replace(/\s+/g, " ").trim();
|
|
1372
|
+
var child = (n, tag) => kids2(n).find((x) => tagOf2(x) === tag);
|
|
1373
|
+
function findDeep2(nodes, tag) {
|
|
1374
|
+
for (const n of nodes) {
|
|
1375
|
+
if ("#text" in n) continue;
|
|
1376
|
+
if (tagOf2(n) === tag) return n;
|
|
1377
|
+
const h = findDeep2(kids2(n), tag);
|
|
1378
|
+
if (h) return h;
|
|
1379
|
+
}
|
|
1380
|
+
return void 0;
|
|
1385
1381
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
for (
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
if (res.ok) return await res.text();
|
|
1392
|
-
const body = await res.text().catch(() => "");
|
|
1393
|
-
if (res.status === 429 && attempt === 0) {
|
|
1394
|
-
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
1395
|
-
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
1396
|
-
await new Promise((r) => setTimeout(r, wait));
|
|
1382
|
+
function inline(nodes) {
|
|
1383
|
+
let out = "";
|
|
1384
|
+
for (const n of nodes) {
|
|
1385
|
+
if ("#text" in n) {
|
|
1386
|
+
out += String(n["#text"]);
|
|
1397
1387
|
continue;
|
|
1398
1388
|
}
|
|
1399
|
-
|
|
1389
|
+
const t = tagOf2(n);
|
|
1390
|
+
const inner = () => inline(kids2(n));
|
|
1391
|
+
switch (t) {
|
|
1392
|
+
case "content": {
|
|
1393
|
+
const style = attr2(n, "styleCode").toLowerCase();
|
|
1394
|
+
const s = inner();
|
|
1395
|
+
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
1396
|
+
break;
|
|
1397
|
+
}
|
|
1398
|
+
case "sup":
|
|
1399
|
+
out += `^${inner()}^`;
|
|
1400
|
+
break;
|
|
1401
|
+
case "sub":
|
|
1402
|
+
out += `~${inner()}~`;
|
|
1403
|
+
break;
|
|
1404
|
+
case "br":
|
|
1405
|
+
out += " ";
|
|
1406
|
+
break;
|
|
1407
|
+
case "footnote":
|
|
1408
|
+
out += ` (${squash2(inner())})`;
|
|
1409
|
+
break;
|
|
1410
|
+
case "footnoteRef":
|
|
1411
|
+
case "renderMultiMedia":
|
|
1412
|
+
break;
|
|
1413
|
+
default:
|
|
1414
|
+
out += inner();
|
|
1415
|
+
}
|
|
1400
1416
|
}
|
|
1401
|
-
|
|
1417
|
+
return out;
|
|
1402
1418
|
}
|
|
1403
|
-
function
|
|
1404
|
-
const
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
if (!
|
|
1410
|
-
let data;
|
|
1411
|
-
try {
|
|
1412
|
-
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
1413
|
-
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
1414
|
-
} catch (e) {
|
|
1415
|
-
return miss(this.name, errMsg(e));
|
|
1416
|
-
}
|
|
1417
|
-
const candidates = coreCandidates(data);
|
|
1418
|
-
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
1419
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1420
|
-
return download(this.name, p, candidates, ctx);
|
|
1419
|
+
function table(n) {
|
|
1420
|
+
const row3 = (x) => kids2(x).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => squash2(inline(kids2(c))).replace(/\|/g, "\\|"));
|
|
1421
|
+
const collect = (nodes, into) => {
|
|
1422
|
+
for (const x of nodes) {
|
|
1423
|
+
if ("#text" in x) continue;
|
|
1424
|
+
if (tagOf2(x) === "tr") into.push(row3(x));
|
|
1425
|
+
else if (!["thead", "tfoot"].includes(tagOf2(x))) collect(kids2(x), into);
|
|
1421
1426
|
}
|
|
1422
1427
|
};
|
|
1428
|
+
const head = [], body = [], foot = [];
|
|
1429
|
+
for (const x of kids2(n)) {
|
|
1430
|
+
const t = tagOf2(x);
|
|
1431
|
+
if (t === "thead") collect(kids2(x), head);
|
|
1432
|
+
else if (t === "tfoot") collect(kids2(x), foot);
|
|
1433
|
+
else if (t === "tr") body.push(row3(x));
|
|
1434
|
+
else if (!("#text" in x) && t !== "caption") collect(kids2(x), body);
|
|
1435
|
+
}
|
|
1436
|
+
const rows = [...head, ...body];
|
|
1437
|
+
if (!rows.length) return "";
|
|
1438
|
+
const width = Math.max(...rows.map((r) => r.length));
|
|
1439
|
+
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
1440
|
+
const cap = child(n, "caption");
|
|
1441
|
+
const title = cap ? squash2(inline(kids2(cap))) : "";
|
|
1442
|
+
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
1443
|
+
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
1423
1444
|
}
|
|
1424
|
-
function
|
|
1445
|
+
function blocks(nodes, depth) {
|
|
1425
1446
|
const out = [];
|
|
1426
|
-
const
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1447
|
+
for (const n of nodes) {
|
|
1448
|
+
if ("#text" in n) {
|
|
1449
|
+
const t2 = squash2(String(n["#text"]));
|
|
1450
|
+
if (t2) out.push(t2);
|
|
1451
|
+
continue;
|
|
1452
|
+
}
|
|
1453
|
+
const t = tagOf2(n);
|
|
1454
|
+
const c = kids2(n);
|
|
1455
|
+
switch (t) {
|
|
1456
|
+
case "paragraph": {
|
|
1457
|
+
const s = squash2(inline(c.filter((x) => !["table", "list"].includes(tagOf2(x)))));
|
|
1458
|
+
if (s) out.push(s);
|
|
1459
|
+
out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf2(x))), depth));
|
|
1460
|
+
break;
|
|
1461
|
+
}
|
|
1462
|
+
case "list": {
|
|
1463
|
+
const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr2(n, "listType"));
|
|
1464
|
+
let i = 0;
|
|
1465
|
+
for (const it of c.filter((x) => tagOf2(x) === "item")) {
|
|
1466
|
+
i++;
|
|
1467
|
+
const body = blocks(kids2(it).filter((x) => tagOf2(x) !== "caption"), depth).join(" ");
|
|
1468
|
+
if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
1469
|
+
}
|
|
1470
|
+
break;
|
|
1471
|
+
}
|
|
1472
|
+
case "table": {
|
|
1473
|
+
const md = table(n);
|
|
1474
|
+
if (md) out.push(md);
|
|
1475
|
+
break;
|
|
1476
|
+
}
|
|
1477
|
+
case "renderMultiMedia":
|
|
1478
|
+
case "observationMedia":
|
|
1479
|
+
case "subject":
|
|
1480
|
+
break;
|
|
1481
|
+
case "section":
|
|
1482
|
+
out.push(...section(n, depth));
|
|
1483
|
+
break;
|
|
1484
|
+
case "component":
|
|
1485
|
+
case "text":
|
|
1486
|
+
case "excerpt":
|
|
1487
|
+
case "highlight":
|
|
1488
|
+
default:
|
|
1489
|
+
out.push(...blocks(c, depth));
|
|
1490
|
+
break;
|
|
1491
|
+
}
|
|
1436
1492
|
}
|
|
1437
1493
|
return out;
|
|
1438
1494
|
}
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
var
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
let data;
|
|
1446
|
-
try {
|
|
1447
|
-
const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
|
|
1448
|
-
data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
|
|
1449
|
-
} catch (e) {
|
|
1450
|
-
return miss(this.name, errMsg(e));
|
|
1451
|
-
}
|
|
1452
|
-
const rec = data.resultList?.result?.[0];
|
|
1453
|
-
if (!rec) return miss(this.name, "not in europe pmc");
|
|
1454
|
-
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
1455
|
-
if (!candidates.length) return miss(this.name, "no free copy");
|
|
1456
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1457
|
-
return download(this.name, p, candidates, ctx);
|
|
1458
|
-
}
|
|
1495
|
+
var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
|
|
1496
|
+
var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
|
|
1497
|
+
var plain2 = (nodes) => {
|
|
1498
|
+
let out = "";
|
|
1499
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
|
|
1500
|
+
return out;
|
|
1459
1501
|
};
|
|
1460
|
-
function
|
|
1502
|
+
function section(n, depth) {
|
|
1503
|
+
const code = child(n, "code");
|
|
1504
|
+
const c = code ? attr2(code, "code") : "";
|
|
1505
|
+
if (SKIP_CODES.has(c)) return [];
|
|
1506
|
+
const title = child(n, "title");
|
|
1507
|
+
let head = title ? squash2(plain2(kids2(title))) : "";
|
|
1508
|
+
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
1509
|
+
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
1461
1510
|
const out = [];
|
|
1462
|
-
|
|
1463
|
-
const
|
|
1464
|
-
|
|
1465
|
-
seen.add(url);
|
|
1466
|
-
out.push({ url, version, source: "europepmc", host, pdf });
|
|
1467
|
-
};
|
|
1468
|
-
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
1469
|
-
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
1470
|
-
if (id && r.inEPMC === "Y") {
|
|
1471
|
-
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
1472
|
-
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
1473
|
-
}
|
|
1474
|
-
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
1475
|
-
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
1476
|
-
if (u.documentStyle === "abs") continue;
|
|
1477
|
-
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
1478
|
-
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
1479
|
-
}
|
|
1511
|
+
if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
|
|
1512
|
+
const rest = kids2(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf2(x)));
|
|
1513
|
+
out.push(...blocks(rest, head ? depth + 1 : depth));
|
|
1480
1514
|
return out;
|
|
1481
1515
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
const
|
|
1491
|
-
|
|
1492
|
-
const
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
always: true,
|
|
1501
|
-
applies: (p) => p.pmid ? true : "no pmid",
|
|
1502
|
-
async run(p, ctx) {
|
|
1503
|
-
const file2 = findInboxFile(dir, p.pmid);
|
|
1504
|
-
if (!file2) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
|
|
1505
|
-
const name = file2.slice(file2.lastIndexOf("/") + 1);
|
|
1506
|
-
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
1507
|
-
const bytes = readFileSync3(file2);
|
|
1508
|
-
const r = await convertBytes(p, bytes, EXT[ext]);
|
|
1509
|
-
if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
|
|
1510
|
-
ctx.onTry?.(`${name} ok, removed from the inbox`);
|
|
1511
|
-
unlinkSync(file2);
|
|
1512
|
-
return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
|
|
1516
|
+
var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
|
|
1517
|
+
function parseSpl(xml) {
|
|
1518
|
+
const doc = parser3.parse(xml);
|
|
1519
|
+
const d = doc.find((x) => tagOf2(x) === "document");
|
|
1520
|
+
if (!d) return null;
|
|
1521
|
+
const k = kids2(d);
|
|
1522
|
+
const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
1523
|
+
const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
|
|
1524
|
+
const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
|
|
1525
|
+
const named = (nodes) => {
|
|
1526
|
+
for (const n of nodes) {
|
|
1527
|
+
if ("#text" in n) continue;
|
|
1528
|
+
if (tagOf2(n) === "manufacturedProduct") {
|
|
1529
|
+
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
1530
|
+
if (child(inner2, "name")) return inner2;
|
|
1531
|
+
}
|
|
1532
|
+
const h = named(kids2(n));
|
|
1533
|
+
if (h) return h;
|
|
1513
1534
|
}
|
|
1535
|
+
return void 0;
|
|
1514
1536
|
};
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1534
|
-
return download(this.name, p, candidates, ctx);
|
|
1535
|
-
}
|
|
1536
|
-
};
|
|
1537
|
-
function openalexCandidates(d) {
|
|
1538
|
-
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
1539
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1540
|
-
const out = [];
|
|
1541
|
-
for (const l of locs) {
|
|
1542
|
-
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
1543
|
-
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
1544
|
-
seen.add(url);
|
|
1545
|
-
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
1546
|
-
}
|
|
1537
|
+
const inner = named(k);
|
|
1538
|
+
const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
1539
|
+
const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
|
|
1540
|
+
const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
1541
|
+
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
1542
|
+
const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
|
|
1543
|
+
const org = findDeep2(k, "representedOrganization");
|
|
1544
|
+
const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
|
|
1545
|
+
const body = findDeep2(k, "structuredBody");
|
|
1546
|
+
const sections2 = body ? blocks(kids2(body), 2) : [];
|
|
1547
|
+
const docTitle = child(d, "title");
|
|
1548
|
+
const lines = [];
|
|
1549
|
+
let cur = "";
|
|
1550
|
+
for (const x of docTitle ? kids2(docTitle) : []) {
|
|
1551
|
+
if (!("#text" in x) && tagOf2(x) === "br") {
|
|
1552
|
+
lines.push(cur);
|
|
1553
|
+
cur = "";
|
|
1554
|
+
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
1547
1555
|
}
|
|
1548
|
-
|
|
1556
|
+
lines.push(cur);
|
|
1557
|
+
const highlights = lines.map(squash2).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
1558
|
+
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
1559
|
+
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
1560
|
+
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
1561
|
+
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
1562
|
+
}
|
|
1563
|
+
function headLineSpl(s) {
|
|
1564
|
+
return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
|
|
1565
|
+
}
|
|
1566
|
+
function renderLabelMd(s) {
|
|
1567
|
+
const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
|
|
1568
|
+
if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
|
|
1569
|
+
parts.push(...s.sections.flatMap((b) => [b, ""]));
|
|
1570
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1549
1571
|
}
|
|
1550
1572
|
|
|
1551
|
-
// src/
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
u.searchParams.set("tool", "fh-library");
|
|
1559
|
-
u.searchParams.set("email", contactEmail());
|
|
1560
|
-
return u.toString();
|
|
1573
|
+
// src/ctgov.ts
|
|
1574
|
+
var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
|
|
1575
|
+
var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
|
|
1576
|
+
async function fetchStudy(nctid) {
|
|
1577
|
+
const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
|
|
1578
|
+
if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
|
|
1579
|
+
return json2;
|
|
1561
1580
|
}
|
|
1562
|
-
var
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
1578
|
-
}
|
|
1581
|
+
var STATUS = {
|
|
1582
|
+
NOT_YET_RECRUITING: "Not yet recruiting",
|
|
1583
|
+
RECRUITING: "Recruiting",
|
|
1584
|
+
ENROLLING_BY_INVITATION: "Enrolling by invitation",
|
|
1585
|
+
ACTIVE_NOT_RECRUITING: "Active, not recruiting",
|
|
1586
|
+
SUSPENDED: "Suspended",
|
|
1587
|
+
TERMINATED: "Terminated",
|
|
1588
|
+
COMPLETED: "Completed",
|
|
1589
|
+
WITHDRAWN: "Withdrawn",
|
|
1590
|
+
UNKNOWN: "Unknown status",
|
|
1591
|
+
AVAILABLE: "Available",
|
|
1592
|
+
NO_LONGER_AVAILABLE: "No longer available",
|
|
1593
|
+
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
1594
|
+
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
1595
|
+
WITHHELD: "Withheld"
|
|
1579
1596
|
};
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1597
|
+
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
1598
|
+
var words = (v) => {
|
|
1599
|
+
const s = String(v ?? "");
|
|
1600
|
+
if (!s) return "";
|
|
1601
|
+
if (STATUS[s]) return STATUS[s];
|
|
1602
|
+
if (PHASE[s]) return PHASE[s];
|
|
1603
|
+
const w = s.toLowerCase().replace(/_/g, " ");
|
|
1604
|
+
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
1605
|
+
};
|
|
1606
|
+
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
1607
|
+
var list = (a) => Array.isArray(a) ? a : [];
|
|
1608
|
+
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
1609
|
+
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
1610
|
+
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
1611
|
+
function parseStudy(json2) {
|
|
1612
|
+
let d;
|
|
1588
1613
|
try {
|
|
1589
|
-
|
|
1614
|
+
d = JSON.parse(json2);
|
|
1590
1615
|
} catch {
|
|
1591
|
-
return
|
|
1592
|
-
}
|
|
1593
|
-
};
|
|
1594
|
-
function tdmCandidates(work, tokens2 = {}, env = process.env) {
|
|
1595
|
-
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
1596
|
-
const seen = /* @__PURE__ */ new Set();
|
|
1597
|
-
const candidates = [];
|
|
1598
|
-
const missingTokens = /* @__PURE__ */ new Set();
|
|
1599
|
-
for (const l of links) {
|
|
1600
|
-
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
1601
|
-
if (seen.has(l.URL)) {
|
|
1602
|
-
const prev = candidates.find((c) => c.url === l.URL);
|
|
1603
|
-
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
1604
|
-
continue;
|
|
1605
|
-
}
|
|
1606
|
-
seen.add(l.URL);
|
|
1607
|
-
const rule = TOKENED.find((r) => r.host.test(hostOf3(l.URL)));
|
|
1608
|
-
let headers;
|
|
1609
|
-
if (rule) {
|
|
1610
|
-
const token = tokens2[rule.key]?.trim() || env[rule.env]?.trim();
|
|
1611
|
-
if (!token) {
|
|
1612
|
-
missingTokens.add(rule.key);
|
|
1613
|
-
continue;
|
|
1614
|
-
}
|
|
1615
|
-
headers = { [rule.header]: token };
|
|
1616
|
-
}
|
|
1617
|
-
if (/text\/plain/.test(ct)) continue;
|
|
1618
|
-
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf3(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
1619
|
-
}
|
|
1620
|
-
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
1621
|
-
candidates.sort((a, b) => rank(a) - rank(b));
|
|
1622
|
-
return { candidates, missingTokens: [...missingTokens] };
|
|
1623
|
-
}
|
|
1624
|
-
function tdmTier(tokens2 = {}) {
|
|
1625
|
-
return {
|
|
1626
|
-
name: "tdm",
|
|
1627
|
-
applies: (p) => p.doi ? true : "no doi",
|
|
1628
|
-
async run(p, ctx) {
|
|
1629
|
-
let work;
|
|
1630
|
-
try {
|
|
1631
|
-
work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
|
|
1632
|
-
} catch (e) {
|
|
1633
|
-
return miss(this.name, errMsg(e));
|
|
1634
|
-
}
|
|
1635
|
-
const { candidates, missingTokens } = tdmCandidates(work, tokens2);
|
|
1636
|
-
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
1637
|
-
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
1638
|
-
return download(this.name, p, candidates, ctx);
|
|
1639
|
-
}
|
|
1640
|
-
};
|
|
1641
|
-
}
|
|
1642
|
-
var tdm = tdmTier();
|
|
1643
|
-
|
|
1644
|
-
// src/tiers/unpaywall.ts
|
|
1645
|
-
var unpaywall = {
|
|
1646
|
-
name: "unpaywall",
|
|
1647
|
-
applies: (p) => p.doi ? true : "no doi",
|
|
1648
|
-
async run(p, ctx) {
|
|
1649
|
-
const mail = contactEmail();
|
|
1650
|
-
let data;
|
|
1651
|
-
try {
|
|
1652
|
-
data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
1653
|
-
} catch (e) {
|
|
1654
|
-
return miss(this.name, errMsg(e));
|
|
1655
|
-
}
|
|
1656
|
-
const candidates = unpaywallCandidates(data);
|
|
1657
|
-
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
1658
|
-
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
1659
|
-
return download(this.name, p, candidates, ctx);
|
|
1616
|
+
return null;
|
|
1660
1617
|
}
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
const
|
|
1664
|
-
const
|
|
1618
|
+
const ps = d?.protocolSection;
|
|
1619
|
+
if (!ps) return null;
|
|
1620
|
+
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
1621
|
+
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
1622
|
+
const nctid = String(idm.nctId ?? "").toUpperCase();
|
|
1665
1623
|
const out = [];
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
seen.add(url);
|
|
1670
|
-
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
1671
|
-
}
|
|
1672
|
-
}
|
|
1673
|
-
return out;
|
|
1674
|
-
}
|
|
1675
|
-
|
|
1676
|
-
// src/tiers/index.ts
|
|
1677
|
-
var BUILTIN = [
|
|
1678
|
-
{ name: "inbox", make: () => inbox },
|
|
1679
|
-
{ name: "pmc", make: () => pmc },
|
|
1680
|
-
{ name: "europepmc", make: () => europepmc },
|
|
1681
|
-
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
1682
|
-
{ name: "unpaywall", make: () => unpaywall },
|
|
1683
|
-
{ name: "openalex", make: () => openalex },
|
|
1684
|
-
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
1685
|
-
];
|
|
1686
|
-
|
|
1687
|
-
// src/tiers/url.ts
|
|
1688
|
-
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
1689
|
-
function urlTier(name, template, headers) {
|
|
1690
|
-
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
1691
|
-
return {
|
|
1692
|
-
name,
|
|
1693
|
-
applies(p) {
|
|
1694
|
-
const missing = used.filter((k) => !p[k]);
|
|
1695
|
-
return missing.length ? `no ${missing.join("/")}` : true;
|
|
1696
|
-
},
|
|
1697
|
-
async run(p, ctx) {
|
|
1698
|
-
if (!used.length) return miss(name, "template has no placeholder");
|
|
1699
|
-
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
1700
|
-
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
1701
|
-
}
|
|
1624
|
+
const sec = (title, lines) => {
|
|
1625
|
+
const body = lines.filter((l) => !!l && l.trim() !== "");
|
|
1626
|
+
if (body.length) out.push(`## ${title}`, ...body);
|
|
1702
1627
|
};
|
|
1703
|
-
}
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
"
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
"
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
"
|
|
1727
|
-
"
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
"
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1628
|
+
const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
|
|
1629
|
+
sec("Identification", [
|
|
1630
|
+
idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
|
|
1631
|
+
idm.acronym ? `Acronym: ${idm.acronym}` : null,
|
|
1632
|
+
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
1633
|
+
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
1634
|
+
]);
|
|
1635
|
+
sec("Status", [
|
|
1636
|
+
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
1637
|
+
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
1638
|
+
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
1639
|
+
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
1640
|
+
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
1641
|
+
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
1642
|
+
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
1643
|
+
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
1644
|
+
]);
|
|
1645
|
+
const rp = sp.responsibleParty;
|
|
1646
|
+
sec("Sponsor and collaborators", [
|
|
1647
|
+
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
1648
|
+
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
1649
|
+
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
1650
|
+
]);
|
|
1651
|
+
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
1652
|
+
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
1653
|
+
const di = dm.designInfo ?? {};
|
|
1654
|
+
const mi = di.maskingInfo ?? {};
|
|
1655
|
+
sec("Design", [
|
|
1656
|
+
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
1657
|
+
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
1658
|
+
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
1659
|
+
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
1660
|
+
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
1661
|
+
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
1662
|
+
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
1663
|
+
]);
|
|
1664
|
+
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
1665
|
+
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
1666
|
+
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
1667
|
+
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
1668
|
+
sec("Outcomes", [
|
|
1669
|
+
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
1670
|
+
...list(om.primaryOutcomes).map(outcome),
|
|
1671
|
+
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
1672
|
+
...list(om.secondaryOutcomes).map(outcome),
|
|
1673
|
+
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
1674
|
+
...list(om.otherOutcomes).map(outcome)
|
|
1675
|
+
]);
|
|
1676
|
+
sec("Eligibility", [
|
|
1677
|
+
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
1678
|
+
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
1679
|
+
]);
|
|
1680
|
+
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
1681
|
+
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
1682
|
+
const refs = list(rf.references);
|
|
1683
|
+
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
1684
|
+
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
1685
|
+
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
1735
1686
|
return {
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1687
|
+
nctid,
|
|
1688
|
+
title: para(idm.briefTitle) || nctid,
|
|
1689
|
+
acronym: idm.acronym ?? null,
|
|
1690
|
+
sponsor: sp.leadSponsor?.name ?? null,
|
|
1691
|
+
firstPosted: st.studyFirstPostDateStruct?.date ?? null,
|
|
1692
|
+
lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
|
|
1693
|
+
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
1694
|
+
hasResults: !!d.hasResults,
|
|
1695
|
+
publications,
|
|
1696
|
+
sections: out
|
|
1740
1697
|
};
|
|
1741
1698
|
}
|
|
1742
|
-
function
|
|
1743
|
-
const out =
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
const
|
|
1748
|
-
|
|
1699
|
+
function results(rs) {
|
|
1700
|
+
const out = ["## Results"];
|
|
1701
|
+
const pf = rs.participantFlowModule;
|
|
1702
|
+
if (pf) {
|
|
1703
|
+
out.push("### Participant flow");
|
|
1704
|
+
const groups = list(pf.groups);
|
|
1705
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
1706
|
+
if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
|
|
1707
|
+
for (const p of list(pf.periods)) {
|
|
1708
|
+
const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
|
|
1709
|
+
const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
|
|
1710
|
+
for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
|
|
1711
|
+
out.push(table2(head, rows));
|
|
1712
|
+
}
|
|
1749
1713
|
}
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
seen.add(name);
|
|
1765
|
-
const t = available.get(name);
|
|
1766
|
-
if (t) tiers.push(t);
|
|
1767
|
-
else if (name === "proxy") problems.push('FH_LIBRARY_TIERS: "proxy" listed, but FH_LIBRARY_PROXY_URL is unset');
|
|
1768
|
-
else problems.push(`FH_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when FH_LIBRARY_PROXY_URL is set)`);
|
|
1714
|
+
const bc = rs.baselineCharacteristicsModule;
|
|
1715
|
+
if (bc) {
|
|
1716
|
+
out.push("### Baseline characteristics");
|
|
1717
|
+
const groups = list(bc.groups);
|
|
1718
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
1719
|
+
for (const m of list(bc.measures)) {
|
|
1720
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
1721
|
+
const rows = [];
|
|
1722
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
1723
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
1724
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
1725
|
+
})]);
|
|
1726
|
+
out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
|
|
1727
|
+
}
|
|
1769
1728
|
}
|
|
1770
|
-
const
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
}
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
} catch {
|
|
1791
|
-
return false;
|
|
1729
|
+
const om = rs.outcomeMeasuresModule;
|
|
1730
|
+
if (om) {
|
|
1731
|
+
out.push("### Outcome measures");
|
|
1732
|
+
for (const m of list(om.outcomeMeasures)) {
|
|
1733
|
+
const groups = list(m.groups);
|
|
1734
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
1735
|
+
out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
|
|
1736
|
+
if (m.description) out.push(para(m.description));
|
|
1737
|
+
out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
|
|
1738
|
+
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
1739
|
+
const rows = [];
|
|
1740
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
1741
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
1742
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
1743
|
+
})]);
|
|
1744
|
+
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
1745
|
+
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
1746
|
+
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
1747
|
+
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
1748
|
+
}
|
|
1792
1749
|
}
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1750
|
+
const ae = rs.adverseEventsModule;
|
|
1751
|
+
if (ae) {
|
|
1752
|
+
out.push("### Adverse events");
|
|
1753
|
+
const groups = list(ae.eventGroups);
|
|
1754
|
+
out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
|
|
1755
|
+
const summary = [
|
|
1756
|
+
["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
|
|
1757
|
+
["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
|
|
1758
|
+
["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
|
|
1759
|
+
].filter((r) => r.slice(1).some(Boolean));
|
|
1760
|
+
out.push(table2(["", ...groups.map((g) => g.title)], summary));
|
|
1761
|
+
const events = (title, evs) => {
|
|
1762
|
+
if (!evs.length) return;
|
|
1763
|
+
out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
|
|
1764
|
+
const s = list(e.stats).find((x) => x.groupId === g.id);
|
|
1765
|
+
return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
|
|
1766
|
+
})])));
|
|
1767
|
+
};
|
|
1768
|
+
events("Serious adverse events", list(ae.seriousEvents));
|
|
1769
|
+
events("Other adverse events", list(ae.otherEvents));
|
|
1799
1770
|
}
|
|
1771
|
+
return out;
|
|
1800
1772
|
}
|
|
1801
|
-
function
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
rotateIfLarge(LOG_FILE);
|
|
1808
|
-
appendFileSync(LOG_FILE, line);
|
|
1809
|
-
} catch {
|
|
1810
|
-
}
|
|
1773
|
+
function headLineStudy(s) {
|
|
1774
|
+
return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
|
|
1775
|
+
}
|
|
1776
|
+
function renderStudyMd(s) {
|
|
1777
|
+
const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
|
|
1778
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1811
1779
|
}
|
|
1812
1780
|
|
|
1813
|
-
// src/stats.ts
|
|
1814
|
-
import { mkdirSync as mkdirSync4, readdirSync as readdirSync3, readFileSync as readFileSync5, statSync as statSync2, unlinkSync as unlinkSync3 } from "fs";
|
|
1815
|
-
import { join as join7 } from "path";
|
|
1816
|
-
|
|
1817
1781
|
// src/entry.ts
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1782
|
+
function emptyStatus(requested, type, id, libraryDir) {
|
|
1783
|
+
return {
|
|
1784
|
+
requested,
|
|
1785
|
+
type,
|
|
1786
|
+
id,
|
|
1787
|
+
entry: null,
|
|
1788
|
+
entry_dir: null,
|
|
1789
|
+
library_dir: libraryDir,
|
|
1790
|
+
pmid: null,
|
|
1791
|
+
setid: null,
|
|
1792
|
+
nctid: null,
|
|
1793
|
+
publications: [],
|
|
1794
|
+
pmcid: null,
|
|
1795
|
+
doi: null,
|
|
1796
|
+
title: null,
|
|
1797
|
+
first_author: null,
|
|
1798
|
+
year: null,
|
|
1799
|
+
publication_types: [],
|
|
1800
|
+
retracted: false,
|
|
1801
|
+
corrections: [],
|
|
1802
|
+
coi: null,
|
|
1803
|
+
funding: [],
|
|
1804
|
+
citable: true,
|
|
1805
|
+
comment: null,
|
|
1806
|
+
has_abstract: null,
|
|
1807
|
+
level: null,
|
|
1808
|
+
version: null,
|
|
1809
|
+
retrieved_by: null,
|
|
1810
|
+
retrieved_date: null,
|
|
1811
|
+
pubmed_checked: null,
|
|
1812
|
+
attempts: [],
|
|
1813
|
+
files: [],
|
|
1814
|
+
glyphs_unresolved: null
|
|
1815
|
+
};
|
|
1816
|
+
}
|
|
1817
|
+
var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1818
|
+
function headLine(r) {
|
|
1819
|
+
return [
|
|
1820
|
+
r.authors.join(", ") || null,
|
|
1821
|
+
r.journal,
|
|
1822
|
+
r.year,
|
|
1823
|
+
r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
|
|
1824
|
+
r.doi ? `DOI ${r.doi}` : null,
|
|
1825
|
+
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
1826
|
+
].filter(Boolean).join(" \xB7 ");
|
|
1827
|
+
}
|
|
1828
|
+
function renderAbstractMd(r) {
|
|
1829
|
+
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
1830
|
+
return `# ${r.title}
|
|
1821
1831
|
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
1829
|
-
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
1830
|
-
UpdateIn: "update_in"
|
|
1831
|
-
};
|
|
1832
|
-
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
1833
|
-
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
1834
|
-
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
1835
|
-
function efetchUrl(pmid) {
|
|
1836
|
-
const u = new URL(EFETCH);
|
|
1837
|
-
u.searchParams.set("db", "pubmed");
|
|
1838
|
-
u.searchParams.set("id", pmid);
|
|
1839
|
-
u.searchParams.set("retmode", "xml");
|
|
1840
|
-
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
1841
|
-
u.searchParams.set("tool", "fh-library");
|
|
1842
|
-
u.searchParams.set("email", contactEmail());
|
|
1843
|
-
return u.toString();
|
|
1832
|
+
${headLine(r)}
|
|
1833
|
+
|
|
1834
|
+
## Abstract
|
|
1835
|
+
|
|
1836
|
+
${body}
|
|
1837
|
+
`;
|
|
1844
1838
|
}
|
|
1845
|
-
|
|
1839
|
+
function readHeadLine(p) {
|
|
1846
1840
|
try {
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1841
|
+
const lines = readFileSync3(p.md, "utf8").split("\n").filter((l) => l.trim());
|
|
1842
|
+
return lines[1] ?? "";
|
|
1843
|
+
} catch {
|
|
1844
|
+
return "";
|
|
1850
1845
|
}
|
|
1851
1846
|
}
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1847
|
+
function metaFromRecord(r, files, converter) {
|
|
1848
|
+
return {
|
|
1849
|
+
pmid: r.pmid,
|
|
1850
|
+
setid: null,
|
|
1851
|
+
nctid: null,
|
|
1852
|
+
publications: [],
|
|
1853
|
+
pmcid: r.pmcid,
|
|
1854
|
+
doi: r.doi,
|
|
1855
|
+
title: r.title,
|
|
1856
|
+
first_author: r.firstAuthor,
|
|
1857
|
+
year: r.year,
|
|
1858
|
+
publication_types: r.publicationTypes,
|
|
1859
|
+
retracted: isRetracted(r),
|
|
1860
|
+
corrections: r.corrections,
|
|
1861
|
+
coi: r.coi,
|
|
1862
|
+
funding: r.funding,
|
|
1863
|
+
has_abstract: r.abstract.length > 0,
|
|
1864
|
+
level: "abstract",
|
|
1865
|
+
version: null,
|
|
1866
|
+
files,
|
|
1867
|
+
retrieved_by: "pubmed",
|
|
1868
|
+
retrieved_date: today(),
|
|
1869
|
+
converted_by: converter,
|
|
1870
|
+
pubmed_checked: today(),
|
|
1871
|
+
attempts: []
|
|
1872
|
+
};
|
|
1864
1873
|
}
|
|
1865
|
-
var
|
|
1866
|
-
|
|
1867
|
-
|
|
1874
|
+
var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
|
|
1875
|
+
var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
|
|
1876
|
+
function parseCorrection(v) {
|
|
1877
|
+
const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
|
|
1878
|
+
if (!m) return null;
|
|
1879
|
+
let source;
|
|
1880
|
+
try {
|
|
1881
|
+
source = JSON.parse(m[3]);
|
|
1882
|
+
} catch {
|
|
1883
|
+
return null;
|
|
1884
|
+
}
|
|
1885
|
+
return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
|
|
1868
1886
|
}
|
|
1869
|
-
function
|
|
1870
|
-
|
|
1887
|
+
function renderMeta(m) {
|
|
1888
|
+
const lines = [
|
|
1889
|
+
["pmid", m.pmid],
|
|
1890
|
+
...m.setid ? [["setid", m.setid]] : [],
|
|
1891
|
+
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
1892
|
+
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
1893
|
+
["pmcid", m.pmcid ?? ""],
|
|
1894
|
+
["doi", m.doi ?? ""],
|
|
1895
|
+
["title", JSON.stringify(m.title)],
|
|
1896
|
+
["first_author", m.first_author ?? ""],
|
|
1897
|
+
["year", m.year ?? ""],
|
|
1898
|
+
["publication_types", m.publication_types.join("; ")],
|
|
1899
|
+
["retracted", m.retracted ? "yes" : "no"],
|
|
1900
|
+
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
1901
|
+
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
1902
|
+
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
1903
|
+
// a JSON array: a statement may hold semicolons
|
|
1904
|
+
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
1905
|
+
["level", m.level],
|
|
1906
|
+
["version", m.version ?? ""],
|
|
1907
|
+
["files", m.files.join(", ")],
|
|
1908
|
+
["retrieved_by", m.retrieved_by],
|
|
1909
|
+
["retrieved_date", m.retrieved_date],
|
|
1910
|
+
["converted_by", m.converted_by],
|
|
1911
|
+
["pubmed_checked", m.pubmed_checked],
|
|
1912
|
+
["attempts", m.attempts.join(", ")]
|
|
1913
|
+
];
|
|
1914
|
+
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
1871
1915
|
}
|
|
1872
|
-
function
|
|
1873
|
-
const
|
|
1874
|
-
const
|
|
1875
|
-
|
|
1876
|
-
const
|
|
1877
|
-
|
|
1916
|
+
function parseFrontmatter(md) {
|
|
1917
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(md);
|
|
1918
|
+
const fm = {}, all = {};
|
|
1919
|
+
for (const line of (m?.[1] ?? "").split("\n")) {
|
|
1920
|
+
const i = line.indexOf(":");
|
|
1921
|
+
if (i <= 0) continue;
|
|
1922
|
+
const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
|
|
1923
|
+
fm[k] = v;
|
|
1924
|
+
(all[k] ??= []).push(v);
|
|
1878
1925
|
}
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
const
|
|
1887
|
-
|
|
1888
|
-
if (
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
const first = authorList[0];
|
|
1894
|
-
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
1895
|
-
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
1896
|
-
let year = text(pub.Year).trim() || null;
|
|
1897
|
-
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
1898
|
-
const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
|
|
1899
|
-
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
1900
|
-
text: inlineText2(text(a))
|
|
1901
|
-
})).filter((a) => a.text);
|
|
1902
|
-
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
1903
|
-
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
1904
|
-
if (!type) return [];
|
|
1905
|
-
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
|
|
1906
|
-
});
|
|
1907
|
-
const funding = [];
|
|
1908
|
-
for (const g of art.GrantList?.Grant ?? []) {
|
|
1909
|
-
const a = plain(g.Agency);
|
|
1910
|
-
if (a && !funding.includes(a)) funding.push(a);
|
|
1926
|
+
return { fm, all };
|
|
1927
|
+
}
|
|
1928
|
+
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
1929
|
+
function splitAttempts(v) {
|
|
1930
|
+
if (!v) return [];
|
|
1931
|
+
const out = [];
|
|
1932
|
+
let cur = "", depth = 0;
|
|
1933
|
+
for (const ch of v) {
|
|
1934
|
+
if (ch === "(") depth++;
|
|
1935
|
+
else if (ch === ")") depth = Math.max(0, depth - 1);
|
|
1936
|
+
if (ch === "," && depth === 0) {
|
|
1937
|
+
out.push(cur);
|
|
1938
|
+
cur = "";
|
|
1939
|
+
} else cur += ch;
|
|
1911
1940
|
}
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
authors,
|
|
1920
|
-
firstAuthor,
|
|
1921
|
-
journal: plain(art.Journal?.Title) || null,
|
|
1922
|
-
year,
|
|
1923
|
-
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
1924
|
-
abstract,
|
|
1925
|
-
corrections
|
|
1926
|
-
};
|
|
1941
|
+
out.push(cur);
|
|
1942
|
+
const merged = [];
|
|
1943
|
+
for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
|
|
1944
|
+
if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
|
|
1945
|
+
else merged.push(piece);
|
|
1946
|
+
}
|
|
1947
|
+
return merged;
|
|
1927
1948
|
}
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
1940
|
-
const first = people[0];
|
|
1941
|
-
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
|
|
1942
|
-
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
1943
|
-
return {
|
|
1944
|
-
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
1945
|
-
pmcid: null,
|
|
1946
|
-
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
1947
|
-
title: inlineText2(text(d.ArticleTitle)) || inlineText2(text(book.BookTitle)),
|
|
1948
|
-
authors,
|
|
1949
|
-
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
1950
|
-
journal: plain(book.Publisher?.PublisherName) || null,
|
|
1951
|
-
year: text(book.PubDate?.Year).trim() || null,
|
|
1952
|
-
publicationTypes: [...new Set(types)],
|
|
1953
|
-
abstract,
|
|
1954
|
-
corrections: [],
|
|
1955
|
-
coi: null,
|
|
1956
|
-
funding: []
|
|
1957
|
-
};
|
|
1958
|
-
}
|
|
1959
|
-
|
|
1960
|
-
// src/spl.ts
|
|
1961
|
-
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
1962
|
-
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
1963
|
-
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
1964
|
-
async function fetchSpl(setid) {
|
|
1965
|
-
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
1966
|
-
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
1967
|
-
return xml;
|
|
1949
|
+
var opt = (v) => v ? v : null;
|
|
1950
|
+
var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
|
|
1951
|
+
function splitPublicationTypes(v) {
|
|
1952
|
+
if (!v) return [];
|
|
1953
|
+
if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
|
|
1954
|
+
const out = [];
|
|
1955
|
+
for (const piece of list2(v)) {
|
|
1956
|
+
if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
|
|
1957
|
+
else out.push(piece);
|
|
1958
|
+
}
|
|
1959
|
+
return out;
|
|
1968
1960
|
}
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
var child = (n, tag) => kids2(n).find((x) => tagOf2(x) === tag);
|
|
1975
|
-
function findDeep2(nodes, tag) {
|
|
1976
|
-
for (const n of nodes) {
|
|
1977
|
-
if ("#text" in n) continue;
|
|
1978
|
-
if (tagOf2(n) === tag) return n;
|
|
1979
|
-
const h = findDeep2(kids2(n), tag);
|
|
1980
|
-
if (h) return h;
|
|
1961
|
+
function abstractMissing(p) {
|
|
1962
|
+
try {
|
|
1963
|
+
return readFileSync3(p.md, "utf8").includes("_No abstract available._");
|
|
1964
|
+
} catch {
|
|
1965
|
+
return false;
|
|
1981
1966
|
}
|
|
1982
|
-
return void 0;
|
|
1983
1967
|
}
|
|
1984
|
-
function
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
continue;
|
|
1990
|
-
}
|
|
1991
|
-
const t = tagOf2(n);
|
|
1992
|
-
const inner = () => inline(kids2(n));
|
|
1993
|
-
switch (t) {
|
|
1994
|
-
case "content": {
|
|
1995
|
-
const style = attr2(n, "styleCode").toLowerCase();
|
|
1996
|
-
const s = inner();
|
|
1997
|
-
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
1998
|
-
break;
|
|
1999
|
-
}
|
|
2000
|
-
case "sup":
|
|
2001
|
-
out += `^${inner()}^`;
|
|
2002
|
-
break;
|
|
2003
|
-
case "sub":
|
|
2004
|
-
out += `~${inner()}~`;
|
|
2005
|
-
break;
|
|
2006
|
-
case "br":
|
|
2007
|
-
out += " ";
|
|
2008
|
-
break;
|
|
2009
|
-
case "footnote":
|
|
2010
|
-
out += ` (${squash2(inner())})`;
|
|
2011
|
-
break;
|
|
2012
|
-
case "footnoteRef":
|
|
2013
|
-
case "renderMultiMedia":
|
|
2014
|
-
break;
|
|
2015
|
-
default:
|
|
2016
|
-
out += inner();
|
|
2017
|
-
}
|
|
1968
|
+
function correctionsFromXml(p) {
|
|
1969
|
+
try {
|
|
1970
|
+
return parsePubmedXml(readFileSync3(p.xml, "utf8"))?.corrections ?? [];
|
|
1971
|
+
} catch {
|
|
1972
|
+
return [];
|
|
2018
1973
|
}
|
|
2019
|
-
return out;
|
|
2020
1974
|
}
|
|
2021
|
-
function
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
else if (!["thead", "tfoot"].includes(tagOf2(x))) collect(kids2(x), into);
|
|
2028
|
-
}
|
|
2029
|
-
};
|
|
2030
|
-
const head = [], body = [], foot = [];
|
|
2031
|
-
for (const x of kids2(n)) {
|
|
2032
|
-
const t = tagOf2(x);
|
|
2033
|
-
if (t === "thead") collect(kids2(x), head);
|
|
2034
|
-
else if (t === "tfoot") collect(kids2(x), foot);
|
|
2035
|
-
else if (t === "tr") body.push(row3(x));
|
|
2036
|
-
else if (!("#text" in x) && t !== "caption") collect(kids2(x), body);
|
|
1975
|
+
function declarationsFromXml(p) {
|
|
1976
|
+
try {
|
|
1977
|
+
const r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
1978
|
+
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
1979
|
+
} catch {
|
|
1980
|
+
return { coi: null, funding: [] };
|
|
2037
1981
|
}
|
|
2038
|
-
const rows = [...head, ...body];
|
|
2039
|
-
if (!rows.length) return "";
|
|
2040
|
-
const width = Math.max(...rows.map((r) => r.length));
|
|
2041
|
-
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
2042
|
-
const cap = child(n, "caption");
|
|
2043
|
-
const title = cap ? squash2(inline(kids2(cap))) : "";
|
|
2044
|
-
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
2045
|
-
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
2046
1982
|
}
|
|
2047
|
-
function
|
|
2048
|
-
const
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
1983
|
+
function readMeta(p) {
|
|
1984
|
+
const { fm, all } = parseFrontmatter(readFileSync3(p.meta, "utf8"));
|
|
1985
|
+
let title = fm.title ?? "";
|
|
1986
|
+
if (title.startsWith('"')) {
|
|
1987
|
+
try {
|
|
1988
|
+
title = JSON.parse(title);
|
|
1989
|
+
} catch {
|
|
2054
1990
|
}
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
let i = 0;
|
|
2067
|
-
for (const it of c.filter((x) => tagOf2(x) === "item")) {
|
|
2068
|
-
i++;
|
|
2069
|
-
const body = blocks(kids2(it).filter((x) => tagOf2(x) !== "caption"), depth).join(" ");
|
|
2070
|
-
if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
2071
|
-
}
|
|
2072
|
-
break;
|
|
2073
|
-
}
|
|
2074
|
-
case "table": {
|
|
2075
|
-
const md = table(n);
|
|
2076
|
-
if (md) out.push(md);
|
|
2077
|
-
break;
|
|
1991
|
+
}
|
|
1992
|
+
const legacy = fm.pubmed_checked === void 0;
|
|
1993
|
+
const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
|
|
1994
|
+
let coi = null, funding = [];
|
|
1995
|
+
if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
|
|
1996
|
+
else {
|
|
1997
|
+
if (fm.coi.startsWith('"')) {
|
|
1998
|
+
try {
|
|
1999
|
+
coi = JSON.parse(fm.coi);
|
|
2000
|
+
} catch {
|
|
2001
|
+
coi = fm.coi;
|
|
2078
2002
|
}
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
case "subject":
|
|
2082
|
-
break;
|
|
2083
|
-
case "section":
|
|
2084
|
-
out.push(...section(n, depth));
|
|
2085
|
-
break;
|
|
2086
|
-
case "component":
|
|
2087
|
-
case "text":
|
|
2088
|
-
case "excerpt":
|
|
2089
|
-
case "highlight":
|
|
2090
|
-
default:
|
|
2091
|
-
out.push(...blocks(c, depth));
|
|
2092
|
-
break;
|
|
2093
|
-
}
|
|
2003
|
+
} else coi = fm.coi || null;
|
|
2004
|
+
funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
|
|
2094
2005
|
}
|
|
2095
|
-
return
|
|
2006
|
+
return {
|
|
2007
|
+
pmid: fm.pmid ?? "",
|
|
2008
|
+
setid: opt(fm.setid),
|
|
2009
|
+
nctid: opt(fm.nctid),
|
|
2010
|
+
publications: list2(fm.publications),
|
|
2011
|
+
pmcid: opt(fm.pmcid),
|
|
2012
|
+
doi: opt(fm.doi),
|
|
2013
|
+
title,
|
|
2014
|
+
first_author: opt(fm.first_author),
|
|
2015
|
+
year: opt(fm.year),
|
|
2016
|
+
publication_types: splitPublicationTypes(fm.publication_types),
|
|
2017
|
+
retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
|
|
2018
|
+
corrections,
|
|
2019
|
+
coi,
|
|
2020
|
+
funding,
|
|
2021
|
+
has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
|
|
2022
|
+
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
2023
|
+
version: opt(fm.version),
|
|
2024
|
+
files: list2(fm.files),
|
|
2025
|
+
retrieved_by: fm.retrieved_by ?? "",
|
|
2026
|
+
retrieved_date: fm.retrieved_date ?? "",
|
|
2027
|
+
converted_by: fm.converted_by ?? "",
|
|
2028
|
+
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
2029
|
+
attempts: splitAttempts(fm.attempts)
|
|
2030
|
+
};
|
|
2096
2031
|
}
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
|
|
2102
|
-
return out;
|
|
2103
|
-
};
|
|
2104
|
-
function section(n, depth) {
|
|
2105
|
-
const code = child(n, "code");
|
|
2106
|
-
const c = code ? attr2(code, "code") : "";
|
|
2107
|
-
if (SKIP_CODES.has(c)) return [];
|
|
2108
|
-
const title = child(n, "title");
|
|
2109
|
-
let head = title ? squash2(plain2(kids2(title))) : "";
|
|
2110
|
-
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
2111
|
-
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
2112
|
-
const out = [];
|
|
2113
|
-
if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
|
|
2114
|
-
const rest = kids2(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf2(x)));
|
|
2115
|
-
out.push(...blocks(rest, head ? depth + 1 : depth));
|
|
2116
|
-
return out;
|
|
2032
|
+
function writeAtomic(file3, data) {
|
|
2033
|
+
const tmp = `${file3}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2034
|
+
writeFileSync(tmp, data);
|
|
2035
|
+
renameSync(tmp, file3);
|
|
2117
2036
|
}
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
const k = kids2(d);
|
|
2124
|
-
const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
2125
|
-
const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
|
|
2126
|
-
const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
|
|
2127
|
-
const named = (nodes) => {
|
|
2128
|
-
for (const n of nodes) {
|
|
2129
|
-
if ("#text" in n) continue;
|
|
2130
|
-
if (tagOf2(n) === "manufacturedProduct") {
|
|
2131
|
-
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
2132
|
-
if (child(inner2, "name")) return inner2;
|
|
2133
|
-
}
|
|
2134
|
-
const h = named(kids2(n));
|
|
2135
|
-
if (h) return h;
|
|
2136
|
-
}
|
|
2137
|
-
return void 0;
|
|
2138
|
-
};
|
|
2139
|
-
const inner = named(k);
|
|
2140
|
-
const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
2141
|
-
const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
|
|
2142
|
-
const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
2143
|
-
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
2144
|
-
const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
|
|
2145
|
-
const org = findDeep2(k, "representedOrganization");
|
|
2146
|
-
const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
|
|
2147
|
-
const body = findDeep2(k, "structuredBody");
|
|
2148
|
-
const sections2 = body ? blocks(kids2(body), 2) : [];
|
|
2149
|
-
const docTitle = child(d, "title");
|
|
2150
|
-
const lines = [];
|
|
2151
|
-
let cur = "";
|
|
2152
|
-
for (const x of docTitle ? kids2(docTitle) : []) {
|
|
2153
|
-
if (!("#text" in x) && tagOf2(x) === "br") {
|
|
2154
|
-
lines.push(cur);
|
|
2155
|
-
cur = "";
|
|
2156
|
-
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
2037
|
+
function writeMeta(p, m) {
|
|
2038
|
+
writeAtomic(p.meta, renderMeta(m));
|
|
2039
|
+
try {
|
|
2040
|
+
unlinkSync(join4(p.entryDir, `${p.entry}-missing.md`));
|
|
2041
|
+
} catch {
|
|
2157
2042
|
}
|
|
2158
|
-
lines.push(cur);
|
|
2159
|
-
const highlights = lines.map(squash2).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
2160
|
-
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
2161
|
-
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
2162
|
-
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
2163
|
-
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
2164
|
-
}
|
|
2165
|
-
function headLineSpl(s) {
|
|
2166
|
-
return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
|
|
2167
2043
|
}
|
|
2168
|
-
function
|
|
2169
|
-
const
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
return
|
|
2044
|
+
function copyEntry(from, to) {
|
|
2045
|
+
const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
|
|
2046
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
2047
|
+
mkdirSync2(tmp, { recursive: true });
|
|
2048
|
+
const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
|
|
2049
|
+
for (const n of readdirSync(from.entryDir)) if (join4(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join4(from.entryDir, n), join4(tmp, rename(n)));
|
|
2050
|
+
const m = readMeta(from);
|
|
2051
|
+
const copy = { ...m, files: m.files.map(rename) };
|
|
2052
|
+
writeFileSync(join4(tmp, basename(to.meta)), renderMeta(copy));
|
|
2053
|
+
mkdirSync2(dirname(to.entryDir), { recursive: true });
|
|
2054
|
+
if (existsSync4(to.entryDir)) renameSync(to.entryDir, old);
|
|
2055
|
+
renameSync(tmp, to.entryDir);
|
|
2056
|
+
rmSync(old, { recursive: true, force: true });
|
|
2057
|
+
return copy;
|
|
2182
2058
|
}
|
|
2183
|
-
var
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
COMPLETED: "Completed",
|
|
2191
|
-
WITHDRAWN: "Withdrawn",
|
|
2192
|
-
UNKNOWN: "Unknown status",
|
|
2193
|
-
AVAILABLE: "Available",
|
|
2194
|
-
NO_LONGER_AVAILABLE: "No longer available",
|
|
2195
|
-
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
2196
|
-
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
2197
|
-
WITHHELD: "Withheld"
|
|
2198
|
-
};
|
|
2199
|
-
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
2200
|
-
var words = (v) => {
|
|
2201
|
-
const s = String(v ?? "");
|
|
2202
|
-
if (!s) return "";
|
|
2203
|
-
if (STATUS[s]) return STATUS[s];
|
|
2204
|
-
if (PHASE[s]) return PHASE[s];
|
|
2205
|
-
const w = s.toLowerCase().replace(/_/g, " ");
|
|
2206
|
-
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
2207
|
-
};
|
|
2208
|
-
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
2209
|
-
var list = (a) => Array.isArray(a) ? a : [];
|
|
2210
|
-
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
2211
|
-
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
2212
|
-
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
2213
|
-
function parseStudy(json2) {
|
|
2214
|
-
let d;
|
|
2215
|
-
try {
|
|
2216
|
-
d = JSON.parse(json2);
|
|
2217
|
-
} catch {
|
|
2218
|
-
return null;
|
|
2059
|
+
var NOTHING_TO_VERIFY = "no abstract and no full text: nothing to verify";
|
|
2060
|
+
function citability(m) {
|
|
2061
|
+
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
2062
|
+
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
2063
|
+
if (m.retracted) {
|
|
2064
|
+
const r = m.corrections.filter(isRetractionNotice);
|
|
2065
|
+
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
2219
2066
|
}
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
const
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
2235
|
-
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
2236
|
-
]);
|
|
2237
|
-
sec("Status", [
|
|
2238
|
-
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
2239
|
-
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
2240
|
-
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
2241
|
-
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
2242
|
-
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
2243
|
-
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
2244
|
-
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
2245
|
-
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
2246
|
-
]);
|
|
2247
|
-
const rp = sp.responsibleParty;
|
|
2248
|
-
sec("Sponsor and collaborators", [
|
|
2249
|
-
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
2250
|
-
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
2251
|
-
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
2252
|
-
]);
|
|
2253
|
-
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
2254
|
-
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
2255
|
-
const di = dm.designInfo ?? {};
|
|
2256
|
-
const mi = di.maskingInfo ?? {};
|
|
2257
|
-
sec("Design", [
|
|
2258
|
-
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
2259
|
-
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
2260
|
-
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
2261
|
-
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
2262
|
-
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
2263
|
-
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
2264
|
-
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
2265
|
-
]);
|
|
2266
|
-
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
2267
|
-
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
2268
|
-
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
2269
|
-
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
2270
|
-
sec("Outcomes", [
|
|
2271
|
-
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
2272
|
-
...list(om.primaryOutcomes).map(outcome),
|
|
2273
|
-
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
2274
|
-
...list(om.secondaryOutcomes).map(outcome),
|
|
2275
|
-
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
2276
|
-
...list(om.otherOutcomes).map(outcome)
|
|
2277
|
-
]);
|
|
2278
|
-
sec("Eligibility", [
|
|
2279
|
-
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
2280
|
-
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
2281
|
-
]);
|
|
2282
|
-
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
2283
|
-
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
2284
|
-
const refs = list(rf.references);
|
|
2285
|
-
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
2286
|
-
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
2287
|
-
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
2067
|
+
if (m.level !== "full_text" && !m.has_abstract) return { citable: false, comment: NOTHING_TO_VERIFY };
|
|
2068
|
+
const notes = [
|
|
2069
|
+
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
2070
|
+
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
2071
|
+
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
2072
|
+
];
|
|
2073
|
+
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
2074
|
+
}
|
|
2075
|
+
function statusFromMeta(p, m, base) {
|
|
2076
|
+
const full = existsSync4(p.fulltextMd) ? readFileSync3(p.fulltextMd, "utf8") : null;
|
|
2077
|
+
const cit = citability(m);
|
|
2078
|
+
if (full?.includes(`
|
|
2079
|
+
${CORRECTED_HEADING}
|
|
2080
|
+
`) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
|
|
2288
2081
|
return {
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2082
|
+
...base,
|
|
2083
|
+
entry: p.entry,
|
|
2084
|
+
entry_dir: p.entryDir,
|
|
2085
|
+
pmid: m.pmid || null,
|
|
2086
|
+
setid: m.setid,
|
|
2087
|
+
nctid: m.nctid,
|
|
2088
|
+
publications: m.publications,
|
|
2089
|
+
pmcid: m.pmcid,
|
|
2090
|
+
doi: m.doi,
|
|
2091
|
+
title: m.title || null,
|
|
2092
|
+
first_author: m.first_author,
|
|
2093
|
+
year: m.year,
|
|
2094
|
+
publication_types: m.publication_types,
|
|
2095
|
+
retracted: m.retracted,
|
|
2096
|
+
corrections: m.corrections,
|
|
2097
|
+
coi: m.coi,
|
|
2098
|
+
funding: m.funding,
|
|
2099
|
+
...cit,
|
|
2100
|
+
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
2101
|
+
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
2102
|
+
// there is abstract text).
|
|
2103
|
+
has_abstract: m.has_abstract,
|
|
2104
|
+
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
2105
|
+
version: m.version,
|
|
2106
|
+
retrieved_by: m.retrieved_by || null,
|
|
2107
|
+
retrieved_date: m.retrieved_date || null,
|
|
2108
|
+
pubmed_checked: m.pubmed_checked || null,
|
|
2109
|
+
attempts: m.attempts,
|
|
2110
|
+
files: m.files,
|
|
2111
|
+
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
2299
2112
|
};
|
|
2300
2113
|
}
|
|
2301
|
-
function
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2114
|
+
function readStatus(p, base) {
|
|
2115
|
+
return statusFromMeta(p, readMeta(p), base);
|
|
2116
|
+
}
|
|
2117
|
+
function entryExists(p) {
|
|
2118
|
+
return existsSync4(p.meta);
|
|
2119
|
+
}
|
|
2120
|
+
function writeEntry(p, r, xml, converter) {
|
|
2121
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2122
|
+
writeAtomic(p.xml, xml);
|
|
2123
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2124
|
+
const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
|
|
2125
|
+
writeMeta(p, meta);
|
|
2126
|
+
return meta;
|
|
2127
|
+
}
|
|
2128
|
+
function refreshRecord(p, m, r, xml) {
|
|
2129
|
+
writeAtomic(p.xml, xml);
|
|
2130
|
+
syncCorrectedBlock(p);
|
|
2131
|
+
const next = {
|
|
2132
|
+
...m,
|
|
2133
|
+
pmcid: r.pmcid ?? m.pmcid,
|
|
2134
|
+
doi: r.doi ?? m.doi,
|
|
2135
|
+
publication_types: r.publicationTypes,
|
|
2136
|
+
retracted: isRetracted(r),
|
|
2137
|
+
corrections: r.corrections,
|
|
2138
|
+
coi: r.coi ?? m.coi,
|
|
2139
|
+
funding: r.funding.length ? r.funding : m.funding,
|
|
2140
|
+
// the paper's own statements stay when PubMed has none
|
|
2141
|
+
pubmed_checked: today()
|
|
2142
|
+
};
|
|
2143
|
+
writeMeta(p, next);
|
|
2144
|
+
return next;
|
|
2145
|
+
}
|
|
2146
|
+
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
2147
|
+
const t = new Date(date).getTime();
|
|
2148
|
+
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
2149
|
+
}
|
|
2150
|
+
var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2151
|
+
var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
|
|
2152
|
+
function declarationsFromMd(md) {
|
|
2153
|
+
let coi = null;
|
|
2154
|
+
const funding = [];
|
|
2155
|
+
const lines = md.split("\n");
|
|
2156
|
+
let section2 = "";
|
|
2157
|
+
for (let i = 0; i < lines.length; i++) {
|
|
2158
|
+
const l = lines[i].trim();
|
|
2159
|
+
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
2160
|
+
if (h) {
|
|
2161
|
+
section2 = h[1];
|
|
2162
|
+
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
2163
|
+
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
2164
|
+
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
2165
|
+
continue;
|
|
2329
2166
|
}
|
|
2167
|
+
if (section2 !== "Declarations" || !l) continue;
|
|
2168
|
+
const f = FUNDING_RE.exec(l);
|
|
2169
|
+
const c = COI_RE.exec(l);
|
|
2170
|
+
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
2171
|
+
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
2330
2172
|
}
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
out.push(
|
|
2340
|
-
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
2341
|
-
const rows = [];
|
|
2342
|
-
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
2343
|
-
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
2344
|
-
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
2345
|
-
})]);
|
|
2346
|
-
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
2347
|
-
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
2348
|
-
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
2349
|
-
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
2173
|
+
return { coi, funding };
|
|
2174
|
+
}
|
|
2175
|
+
var CORRECTED_HEADING = "## Corrected in PubMed abstract";
|
|
2176
|
+
function correctedSentences(r) {
|
|
2177
|
+
const out = [];
|
|
2178
|
+
for (const a of r.abstract) {
|
|
2179
|
+
for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
|
|
2180
|
+
const i = s.indexOf("[corrected]");
|
|
2181
|
+
if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
|
|
2350
2182
|
}
|
|
2351
2183
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2184
|
+
return out.filter(Boolean);
|
|
2185
|
+
}
|
|
2186
|
+
function withCorrectedBlock(md, sentences) {
|
|
2187
|
+
const base = md.split(`
|
|
2188
|
+
${CORRECTED_HEADING}
|
|
2189
|
+
`)[0].trimEnd();
|
|
2190
|
+
if (!sentences.length) return base + "\n";
|
|
2191
|
+
return `${base}
|
|
2192
|
+
|
|
2193
|
+
${CORRECTED_HEADING}
|
|
2194
|
+
|
|
2195
|
+
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
2196
|
+
|
|
2197
|
+
${sentences.join("\n\n")}
|
|
2198
|
+
`;
|
|
2199
|
+
}
|
|
2200
|
+
function syncCorrectedBlock(p) {
|
|
2201
|
+
if (!existsSync4(p.fulltextMd) || !existsSync4(p.xml)) return;
|
|
2202
|
+
let r;
|
|
2203
|
+
try {
|
|
2204
|
+
r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
2205
|
+
} catch {
|
|
2206
|
+
return;
|
|
2372
2207
|
}
|
|
2373
|
-
return
|
|
2208
|
+
if (!r) return;
|
|
2209
|
+
const md = readFileSync3(p.fulltextMd, "utf8");
|
|
2210
|
+
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
2211
|
+
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
2374
2212
|
}
|
|
2375
|
-
function
|
|
2376
|
-
|
|
2213
|
+
function applyFulltext(p, m, hit, converter) {
|
|
2214
|
+
const raw = p.fulltextRaw(hit.format);
|
|
2215
|
+
writeAtomic(raw, hit.raw);
|
|
2216
|
+
writeAtomic(p.fulltextMd, hit.md);
|
|
2217
|
+
syncCorrectedBlock(p);
|
|
2218
|
+
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
2219
|
+
const own = declarationsFromMd(hit.md);
|
|
2220
|
+
const next = {
|
|
2221
|
+
...m,
|
|
2222
|
+
level: "full_text",
|
|
2223
|
+
version: hit.version,
|
|
2224
|
+
files,
|
|
2225
|
+
retrieved_by: hit.tier,
|
|
2226
|
+
retrieved_date: today(),
|
|
2227
|
+
converted_by: converter,
|
|
2228
|
+
coi: m.coi ?? own.coi,
|
|
2229
|
+
funding: m.funding.length ? m.funding : own.funding
|
|
2230
|
+
};
|
|
2231
|
+
writeMeta(p, next);
|
|
2232
|
+
return next;
|
|
2377
2233
|
}
|
|
2378
|
-
function
|
|
2379
|
-
|
|
2380
|
-
|
|
2234
|
+
function recordAttempts(p, m, labels) {
|
|
2235
|
+
if (!labels.length) return m;
|
|
2236
|
+
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
2237
|
+
const replaced = new Set(labels.map(tierOf));
|
|
2238
|
+
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
2239
|
+
writeMeta(p, next);
|
|
2240
|
+
return next;
|
|
2381
2241
|
}
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2242
|
+
function writeLabelEntry(p, s, xml, converter) {
|
|
2243
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2244
|
+
const raw = p.fulltextRaw("xml");
|
|
2245
|
+
writeAtomic(raw, xml);
|
|
2246
|
+
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
2247
|
+
const meta = {
|
|
2248
|
+
pmid: "",
|
|
2249
|
+
setid: s.setid,
|
|
2250
|
+
nctid: null,
|
|
2251
|
+
publications: [],
|
|
2252
|
+
pmcid: null,
|
|
2253
|
+
doi: null,
|
|
2254
|
+
title: s.title,
|
|
2255
|
+
first_author: s.labeler,
|
|
2256
|
+
year: s.effective?.slice(0, 4) ?? null,
|
|
2257
|
+
publication_types: ["Drug Label"],
|
|
2258
|
+
retracted: false,
|
|
2259
|
+
corrections: [],
|
|
2260
|
+
coi: null,
|
|
2261
|
+
funding: [],
|
|
2262
|
+
has_abstract: false,
|
|
2263
|
+
level: "full_text",
|
|
2264
|
+
version: s.version ? `spl ${s.version}` : null,
|
|
2265
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2266
|
+
retrieved_by: "dailymed",
|
|
2267
|
+
retrieved_date: today(),
|
|
2268
|
+
converted_by: converter,
|
|
2269
|
+
pubmed_checked: today(),
|
|
2270
|
+
attempts: []
|
|
2271
|
+
};
|
|
2272
|
+
writeMeta(p, meta);
|
|
2273
|
+
return meta;
|
|
2274
|
+
}
|
|
2275
|
+
function writeStudyEntry(p, s, json2, converter) {
|
|
2276
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2277
|
+
const raw = p.fulltextRaw("json");
|
|
2278
|
+
writeAtomic(raw, json2);
|
|
2279
|
+
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
2280
|
+
const meta = {
|
|
2281
|
+
pmid: "",
|
|
2393
2282
|
setid: null,
|
|
2394
|
-
nctid:
|
|
2395
|
-
publications:
|
|
2283
|
+
nctid: s.nctid,
|
|
2284
|
+
publications: s.publications,
|
|
2396
2285
|
pmcid: null,
|
|
2397
2286
|
doi: null,
|
|
2398
|
-
title:
|
|
2399
|
-
first_author:
|
|
2400
|
-
year: null,
|
|
2401
|
-
publication_types: [],
|
|
2287
|
+
title: s.title,
|
|
2288
|
+
first_author: s.sponsor,
|
|
2289
|
+
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
2290
|
+
publication_types: ["Trial Registration"],
|
|
2402
2291
|
retracted: false,
|
|
2403
2292
|
corrections: [],
|
|
2404
2293
|
coi: null,
|
|
2405
2294
|
funding: [],
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
pubmed_checked:
|
|
2414
|
-
attempts: []
|
|
2415
|
-
files: [],
|
|
2416
|
-
glyphs_unresolved: null
|
|
2295
|
+
has_abstract: false,
|
|
2296
|
+
level: "full_text",
|
|
2297
|
+
version: s.lastUpdate,
|
|
2298
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
2299
|
+
retrieved_by: "ctgov",
|
|
2300
|
+
retrieved_date: today(),
|
|
2301
|
+
converted_by: converter,
|
|
2302
|
+
pubmed_checked: today(),
|
|
2303
|
+
attempts: []
|
|
2417
2304
|
};
|
|
2305
|
+
writeMeta(p, meta);
|
|
2306
|
+
return meta;
|
|
2418
2307
|
}
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
2428
|
-
].filter(Boolean).join(" \xB7 ");
|
|
2308
|
+
function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
|
|
2309
|
+
mkdirSync2(p.entryDir, { recursive: true });
|
|
2310
|
+
const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
|
|
2311
|
+
writeAtomic(raw, json2);
|
|
2312
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
2313
|
+
const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
|
|
2314
|
+
writeMeta(p, meta);
|
|
2315
|
+
return meta;
|
|
2429
2316
|
}
|
|
2430
|
-
function renderAbstractMd(r) {
|
|
2431
|
-
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
2432
|
-
return `# ${r.title}
|
|
2433
2317
|
|
|
2434
|
-
|
|
2318
|
+
// src/walls.ts
|
|
2319
|
+
var WALL2 = /^(challenge not cleared|empty response)/i;
|
|
2320
|
+
var WALL_STREAK = 2;
|
|
2321
|
+
var host = (url) => {
|
|
2322
|
+
try {
|
|
2323
|
+
return new URL(url).hostname.replace(/^www\./, "").toLowerCase();
|
|
2324
|
+
} catch {
|
|
2325
|
+
return "";
|
|
2326
|
+
}
|
|
2327
|
+
};
|
|
2328
|
+
var file = (dir, h) => join5(dir, "walls", `${h.replace(/[^a-z0-9.-]/g, "_")}.json`);
|
|
2329
|
+
function read(dir, h) {
|
|
2330
|
+
try {
|
|
2331
|
+
return JSON.parse(readFileSync4(file(dir, h), "utf8"));
|
|
2332
|
+
} catch {
|
|
2333
|
+
return {};
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
function walled(url, rung2, dir = FH_LIBRARY_DIR) {
|
|
2337
|
+
const h = host(url);
|
|
2338
|
+
if (!h) return null;
|
|
2339
|
+
const r = read(dir, h)[rung2];
|
|
2340
|
+
return r && r.streak >= WALL_STREAK && daysSince(r.date) < settings().retry_after_days ? r : null;
|
|
2341
|
+
}
|
|
2342
|
+
function recordRung(url, rung2, ok, reason = "", dir = FH_LIBRARY_DIR) {
|
|
2343
|
+
const h = host(url);
|
|
2344
|
+
if (!h) return;
|
|
2345
|
+
const why = reason.replace(/^(browser [^:]+|brightdata): /, "");
|
|
2346
|
+
if (!ok && !WALL2.test(why)) return;
|
|
2347
|
+
const all = read(dir, h);
|
|
2348
|
+
if (ok) {
|
|
2349
|
+
if (!all[rung2]) return;
|
|
2350
|
+
delete all[rung2];
|
|
2351
|
+
} else all[rung2] = { streak: (all[rung2]?.streak ?? 0) + 1, date: today(), reason: why };
|
|
2352
|
+
try {
|
|
2353
|
+
mkdirSync3(join5(dir, "walls"), { recursive: true });
|
|
2354
|
+
writeAtomic(file(dir, h), JSON.stringify(all) + "\n");
|
|
2355
|
+
} catch {
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
var wallNote = (r) => `${r.reason} ${r.streak}\xD7 on this host, last ${r.date}`;
|
|
2435
2359
|
|
|
2436
|
-
|
|
2360
|
+
// src/tiers/tier.ts
|
|
2361
|
+
var looked = (t, p, ctx) => ctx.lookups?.get(t.name) ?? t.lookup(p);
|
|
2362
|
+
function oaVersion(v) {
|
|
2363
|
+
if (v === "acceptedVersion") return "accepted_manuscript";
|
|
2364
|
+
if (v === "submittedVersion") return "preprint";
|
|
2365
|
+
return "published";
|
|
2366
|
+
}
|
|
2367
|
+
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
2368
|
+
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
2437
2369
|
|
|
2438
|
-
|
|
2439
|
-
|
|
2370
|
+
// src/download.ts
|
|
2371
|
+
var hostOf2 = (url) => {
|
|
2372
|
+
try {
|
|
2373
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
2374
|
+
} catch {
|
|
2375
|
+
return url;
|
|
2376
|
+
}
|
|
2377
|
+
};
|
|
2378
|
+
var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
|
|
2379
|
+
function order(candidates) {
|
|
2380
|
+
const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
|
|
2381
|
+
return [...candidates].sort((a, b) => rank(a) - rank(b));
|
|
2440
2382
|
}
|
|
2441
|
-
function
|
|
2383
|
+
async function download(tier, p, candidates, ctx) {
|
|
2384
|
+
const usable = candidates.filter((c) => !NCBI.test(c.url));
|
|
2385
|
+
if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
|
|
2386
|
+
const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
|
|
2387
|
+
if (!ordered.length) return miss(tier, "only copies already tried");
|
|
2388
|
+
const reasons = [];
|
|
2389
|
+
for (const c of ordered) {
|
|
2390
|
+
ctx.tried.add(c.url);
|
|
2391
|
+
let r = await tryOne(p, c, tier);
|
|
2392
|
+
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
2393
|
+
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
2394
|
+
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
2395
|
+
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
2396
|
+
for (const b of browserRungs()) {
|
|
2397
|
+
if (r.ok || c.headers || c.service || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
2398
|
+
const w = walled(target.url, `browser ${b.label}`);
|
|
2399
|
+
if (w) {
|
|
2400
|
+
ctx.onTry?.(`${target.url} skipped browser ${b.label}: ${wallNote(w)}`);
|
|
2401
|
+
continue;
|
|
2402
|
+
}
|
|
2403
|
+
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
2404
|
+
recordRung(target.url, `browser ${b.label}`, r.ok, r.ok ? "" : r.reason);
|
|
2405
|
+
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
2406
|
+
if (!r.ok && bare(r.reason).startsWith(MISMATCH)) ctx.onAlert?.(`browser ${b.label}: ${bare(r.reason)}`);
|
|
2407
|
+
}
|
|
2408
|
+
const uw = walled(target.url, "brightdata");
|
|
2409
|
+
if (uw && !r.ok && !c.headers && !c.service && unlockerConfigured()) ctx.onTry?.(`${target.url} skipped brightdata: ${wallNote(uw)}`);
|
|
2410
|
+
else if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
2411
|
+
r = await tryOne(p, target, tier, "unlocker");
|
|
2412
|
+
recordRung(target.url, "brightdata", r.ok, r.ok ? "" : r.reason);
|
|
2413
|
+
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
2414
|
+
if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
|
|
2415
|
+
}
|
|
2416
|
+
if (r.ok) return r;
|
|
2417
|
+
reasons.push(`${r.reason}: ${hostOf2(c.url)}`);
|
|
2418
|
+
}
|
|
2419
|
+
return miss(tier, reasons.join("; "));
|
|
2420
|
+
}
|
|
2421
|
+
var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
|
|
2422
|
+
async function tryOne(p, c, tier, via = "plain") {
|
|
2423
|
+
let got;
|
|
2442
2424
|
try {
|
|
2443
|
-
|
|
2444
|
-
|
|
2425
|
+
got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
|
|
2426
|
+
} catch (e) {
|
|
2427
|
+
const body = e?.body;
|
|
2428
|
+
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
2429
|
+
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
2430
|
+
}
|
|
2431
|
+
const { bytes, contentType } = got;
|
|
2432
|
+
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
2433
|
+
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
2434
|
+
const r = await convertBytes(p, bytes, contentType);
|
|
2435
|
+
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
2436
|
+
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
2437
|
+
}
|
|
2438
|
+
var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
|
|
2439
|
+
function serviceMessage(bytes, contentType = "") {
|
|
2440
|
+
const text3 = bytes.toString("utf8").trim();
|
|
2441
|
+
if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
|
|
2442
|
+
try {
|
|
2443
|
+
const j = JSON.parse(text3);
|
|
2444
|
+
const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
|
|
2445
|
+
if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
|
|
2445
2446
|
} catch {
|
|
2446
|
-
return "";
|
|
2447
2447
|
}
|
|
2448
|
+
const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
2449
|
+
return (visible || text3).slice(0, 160);
|
|
2450
|
+
}
|
|
2451
|
+
function describe(reason, format, md) {
|
|
2452
|
+
const words2 = md.split(/\s+/).filter(Boolean).length;
|
|
2453
|
+
const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
|
|
2454
|
+
const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
|
|
2455
|
+
if (reason === "no methods/results") return `no article body (${seen})`;
|
|
2456
|
+
if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
|
|
2457
|
+
return reason;
|
|
2458
|
+
}
|
|
2459
|
+
async function convertBytes(p, bytes, contentType = "") {
|
|
2460
|
+
let format;
|
|
2461
|
+
let md;
|
|
2462
|
+
if (looksLikePdf(bytes)) {
|
|
2463
|
+
format = "pdf";
|
|
2464
|
+
const stats2 = {};
|
|
2465
|
+
try {
|
|
2466
|
+
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
|
|
2467
|
+
} catch (e) {
|
|
2468
|
+
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
2469
|
+
}
|
|
2470
|
+
if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
2471
|
+
} else {
|
|
2472
|
+
const text3 = bytes.toString("utf8");
|
|
2473
|
+
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
2474
|
+
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
2475
|
+
format = "xml";
|
|
2476
|
+
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
2477
|
+
if (!doc.title) doc.title = p.title;
|
|
2478
|
+
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
2479
|
+
md = renderFulltextMd(doc, p.headLine);
|
|
2480
|
+
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
2481
|
+
format = "html";
|
|
2482
|
+
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
2483
|
+
md = `# ${p.title}
|
|
2484
|
+
|
|
2485
|
+
${p.headLine}
|
|
2486
|
+
|
|
2487
|
+
${htmlToMarkdown(body)}`;
|
|
2488
|
+
} else {
|
|
2489
|
+
return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
|
|
2493
|
+
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
2494
|
+
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
2495
|
+
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
2496
|
+
return { ok: true, format, md };
|
|
2497
|
+
}
|
|
2498
|
+
var ELSEVIER_TAGS = [
|
|
2499
|
+
[/ja:article\b/g, "article"],
|
|
2500
|
+
[/ja:body\b/g, "body"],
|
|
2501
|
+
[/ja:head\b/g, "front"],
|
|
2502
|
+
[/ce:sections\b/g, "sec-group"],
|
|
2503
|
+
[/ce:section-title\b/g, "title"],
|
|
2504
|
+
[/ce:section\b/g, "sec"],
|
|
2505
|
+
[/ce:simple-para\b/g, "p"],
|
|
2506
|
+
[/ce:para\b/g, "p"],
|
|
2507
|
+
[/ce:abstract-sec\b/g, "sec"],
|
|
2508
|
+
[/ce:abstract\b/g, "abstract"],
|
|
2509
|
+
[/ce:title\b/g, "article-title"],
|
|
2510
|
+
[/ce:italic\b/g, "italic"],
|
|
2511
|
+
[/ce:bold\b/g, "bold"],
|
|
2512
|
+
[/ce:sup\b/g, "sup"],
|
|
2513
|
+
[/ce:inf\b/g, "sub"],
|
|
2514
|
+
[/ce:cross-refs?\b/g, "xref"],
|
|
2515
|
+
[/ce:list-item\b/g, "list-item"],
|
|
2516
|
+
[/ce:list\b/g, "list"],
|
|
2517
|
+
[/ce:label\b/g, "label"],
|
|
2518
|
+
[/ce:caption\b/g, "caption"],
|
|
2519
|
+
[/ce:table\b/g, "table-wrap"],
|
|
2520
|
+
[/ce:figure\b/g, "fig"],
|
|
2521
|
+
[/ce:bib-reference\b/g, "ref"],
|
|
2522
|
+
[/ce:bibliography\b/g, "ref-list"],
|
|
2523
|
+
[/sb:reference\b/g, "mixed-citation"],
|
|
2524
|
+
[/ce:acknowledgment\b/g, "ack"],
|
|
2525
|
+
[/ce:inter-ref\b/g, "ext-link"]
|
|
2526
|
+
];
|
|
2527
|
+
function elsevierToJats(xml) {
|
|
2528
|
+
let out = xml;
|
|
2529
|
+
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
2530
|
+
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
2531
|
+
}
|
|
2532
|
+
|
|
2533
|
+
// src/tiers/core.ts
|
|
2534
|
+
var WINDOW_MS = 6e4;
|
|
2535
|
+
var PER_WINDOW = 10;
|
|
2536
|
+
var MAX_WAIT_MS = 65e3;
|
|
2537
|
+
var TIMEOUT_MS3 = 1e4;
|
|
2538
|
+
var calls = [];
|
|
2539
|
+
async function pace() {
|
|
2540
|
+
const now = Date.now();
|
|
2541
|
+
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
2542
|
+
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
2543
|
+
calls.push(Date.now());
|
|
2544
|
+
}
|
|
2545
|
+
async function coreFetch(url, key) {
|
|
2546
|
+
const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
|
|
2547
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
2548
|
+
await pace();
|
|
2549
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
|
|
2550
|
+
if (res.ok) return await res.text();
|
|
2551
|
+
const body = await res.text().catch(() => "");
|
|
2552
|
+
if (res.status === 429 && attempt === 0) {
|
|
2553
|
+
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
2554
|
+
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
2555
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
2556
|
+
continue;
|
|
2557
|
+
}
|
|
2558
|
+
throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
|
|
2559
|
+
}
|
|
2560
|
+
throw new Error("HTTP 429");
|
|
2448
2561
|
}
|
|
2449
|
-
function
|
|
2562
|
+
function coreTier(apiKey) {
|
|
2563
|
+
const key = apiKey?.trim();
|
|
2450
2564
|
return {
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
2459
|
-
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
version: null,
|
|
2468
|
-
files,
|
|
2469
|
-
retrieved_by: "pubmed",
|
|
2470
|
-
retrieved_date: today(),
|
|
2471
|
-
converted_by: converter,
|
|
2472
|
-
pubmed_checked: today(),
|
|
2473
|
-
attempts: []
|
|
2565
|
+
name: "core",
|
|
2566
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2567
|
+
async run(p, ctx) {
|
|
2568
|
+
if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
|
|
2569
|
+
let data;
|
|
2570
|
+
try {
|
|
2571
|
+
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
2572
|
+
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
2573
|
+
} catch (e) {
|
|
2574
|
+
return miss(this.name, errMsg(e));
|
|
2575
|
+
}
|
|
2576
|
+
const candidates = coreCandidates(data);
|
|
2577
|
+
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
2578
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2579
|
+
return download(this.name, p, candidates, ctx);
|
|
2580
|
+
}
|
|
2474
2581
|
};
|
|
2475
2582
|
}
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
const
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2583
|
+
function coreCandidates(d) {
|
|
2584
|
+
const out = [];
|
|
2585
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2586
|
+
const add = (url, pdf, host2) => {
|
|
2587
|
+
if (!url || seen.has(url)) return;
|
|
2588
|
+
seen.add(url);
|
|
2589
|
+
out.push({ url, version: "published", source: "core", host: host2, pdf });
|
|
2590
|
+
};
|
|
2591
|
+
for (const w of d.results ?? []) {
|
|
2592
|
+
add(w.downloadUrl, true, "core");
|
|
2593
|
+
for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
|
|
2594
|
+
for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
|
|
2486
2595
|
}
|
|
2487
|
-
return
|
|
2488
|
-
}
|
|
2489
|
-
function renderMeta(m) {
|
|
2490
|
-
const lines = [
|
|
2491
|
-
["pmid", m.pmid],
|
|
2492
|
-
...m.setid ? [["setid", m.setid]] : [],
|
|
2493
|
-
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
2494
|
-
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
2495
|
-
["pmcid", m.pmcid ?? ""],
|
|
2496
|
-
["doi", m.doi ?? ""],
|
|
2497
|
-
["title", JSON.stringify(m.title)],
|
|
2498
|
-
["first_author", m.first_author ?? ""],
|
|
2499
|
-
["year", m.year ?? ""],
|
|
2500
|
-
["publication_types", m.publication_types.join("; ")],
|
|
2501
|
-
["retracted", m.retracted ? "yes" : "no"],
|
|
2502
|
-
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
2503
|
-
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
2504
|
-
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
2505
|
-
// a JSON array: a statement may hold semicolons
|
|
2506
|
-
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
2507
|
-
["level", m.level],
|
|
2508
|
-
["version", m.version ?? ""],
|
|
2509
|
-
["files", m.files.join(", ")],
|
|
2510
|
-
["retrieved_by", m.retrieved_by],
|
|
2511
|
-
["retrieved_date", m.retrieved_date],
|
|
2512
|
-
["converted_by", m.converted_by],
|
|
2513
|
-
["pubmed_checked", m.pubmed_checked],
|
|
2514
|
-
["attempts", m.attempts.join(", ")]
|
|
2515
|
-
];
|
|
2516
|
-
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
2596
|
+
return out;
|
|
2517
2597
|
}
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2598
|
+
|
|
2599
|
+
// src/tiers/europepmc.ts
|
|
2600
|
+
var europepmc = {
|
|
2601
|
+
name: "europepmc",
|
|
2602
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
2603
|
+
lookup: (p) => fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`)}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }),
|
|
2604
|
+
async run(p, ctx) {
|
|
2605
|
+
let data;
|
|
2606
|
+
try {
|
|
2607
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2608
|
+
} catch (e) {
|
|
2609
|
+
return miss(this.name, errMsg(e));
|
|
2610
|
+
}
|
|
2611
|
+
const rec = data.resultList?.result?.[0];
|
|
2612
|
+
if (!rec) return miss(this.name, "not in europe pmc");
|
|
2613
|
+
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
2614
|
+
if (!candidates.length) return miss(this.name, "no free copy");
|
|
2615
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2616
|
+
return download(this.name, p, candidates, ctx);
|
|
2527
2617
|
}
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
2531
|
-
function splitAttempts(v) {
|
|
2532
|
-
if (!v) return [];
|
|
2618
|
+
};
|
|
2619
|
+
function europepmcCandidates(r, pmcid) {
|
|
2533
2620
|
const out = [];
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
if (
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2621
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2622
|
+
const add = (url, pdf, host2, version) => {
|
|
2623
|
+
if (!url || seen.has(url)) return;
|
|
2624
|
+
seen.add(url);
|
|
2625
|
+
out.push({ url, version, source: "europepmc", host: host2, pdf });
|
|
2626
|
+
};
|
|
2627
|
+
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
2628
|
+
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
2629
|
+
if (id && r.inEPMC === "Y") {
|
|
2630
|
+
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
2631
|
+
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
2542
2632
|
}
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
if (
|
|
2547
|
-
|
|
2633
|
+
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
2634
|
+
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
2635
|
+
if (u.documentStyle === "abs") continue;
|
|
2636
|
+
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
2637
|
+
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
2548
2638
|
}
|
|
2549
|
-
return
|
|
2639
|
+
return out;
|
|
2550
2640
|
}
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2641
|
+
|
|
2642
|
+
// src/tiers/openalex.ts
|
|
2643
|
+
var openalex = {
|
|
2644
|
+
name: "openalex",
|
|
2645
|
+
applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
|
|
2646
|
+
lookup: (p) => fetchText(`https://api.openalex.org/works/${encodeURIComponent(p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`)}?mailto=${encodeURIComponent(contactEmail())}`, { "User-Agent": LIBRARY_UA }),
|
|
2647
|
+
async run(p, ctx) {
|
|
2648
|
+
let data;
|
|
2649
|
+
try {
|
|
2650
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2651
|
+
} catch (e) {
|
|
2652
|
+
return miss(this.name, errMsg(e));
|
|
2653
|
+
}
|
|
2654
|
+
const candidates = openalexCandidates(data);
|
|
2655
|
+
if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
|
|
2656
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2657
|
+
return download(this.name, p, candidates, ctx);
|
|
2658
|
+
}
|
|
2659
|
+
};
|
|
2660
|
+
function openalexCandidates(d) {
|
|
2661
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
2662
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2556
2663
|
const out = [];
|
|
2557
|
-
for (const
|
|
2558
|
-
|
|
2559
|
-
|
|
2664
|
+
for (const l of locs) {
|
|
2665
|
+
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
2666
|
+
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
2667
|
+
seen.add(url);
|
|
2668
|
+
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
2669
|
+
}
|
|
2560
2670
|
}
|
|
2561
2671
|
return out;
|
|
2562
2672
|
}
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2673
|
+
|
|
2674
|
+
// src/tiers/pmc.ts
|
|
2675
|
+
function pmcEfetchUrl(pmcid) {
|
|
2676
|
+
const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
|
|
2677
|
+
u.searchParams.set("db", "pmc");
|
|
2678
|
+
u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
|
|
2679
|
+
u.searchParams.set("retmode", "xml");
|
|
2680
|
+
if (process.env.FH_LIBRARY_NCBI_API_KEY) u.searchParams.set("api_key", process.env.FH_LIBRARY_NCBI_API_KEY);
|
|
2681
|
+
u.searchParams.set("tool", "fh-library");
|
|
2682
|
+
u.searchParams.set("email", contactEmail());
|
|
2683
|
+
return u.toString();
|
|
2569
2684
|
}
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2685
|
+
var pmc = {
|
|
2686
|
+
name: "pmc",
|
|
2687
|
+
applies: (p) => p.pmcid ? true : "no pmcid",
|
|
2688
|
+
async run(p) {
|
|
2689
|
+
let xml;
|
|
2690
|
+
try {
|
|
2691
|
+
xml = await fetchText(pmcEfetchUrl(p.pmcid));
|
|
2692
|
+
} catch (e) {
|
|
2693
|
+
return miss(this.name, errMsg(e));
|
|
2694
|
+
}
|
|
2695
|
+
const doc = parseJats(xml);
|
|
2696
|
+
if (!doc.hasBody) return miss(this.name, "no body");
|
|
2697
|
+
const md = renderFulltextMd(doc, p.headLine);
|
|
2698
|
+
const check = checkFulltext(md, p.title);
|
|
2699
|
+
if (!check.ok) return miss(this.name, check.reason);
|
|
2700
|
+
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
2575
2701
|
}
|
|
2576
|
-
}
|
|
2577
|
-
|
|
2702
|
+
};
|
|
2703
|
+
|
|
2704
|
+
// src/tiers/tdm.ts
|
|
2705
|
+
var CROSSREF = "https://api.crossref.org/works/";
|
|
2706
|
+
var TOKENED = [
|
|
2707
|
+
{ host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
|
|
2708
|
+
{ host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
|
|
2709
|
+
];
|
|
2710
|
+
var hostOf3 = (url) => {
|
|
2578
2711
|
try {
|
|
2579
|
-
|
|
2580
|
-
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
2712
|
+
return new URL(url).hostname;
|
|
2581
2713
|
} catch {
|
|
2582
|
-
return
|
|
2714
|
+
return "";
|
|
2583
2715
|
}
|
|
2584
|
-
}
|
|
2585
|
-
function
|
|
2586
|
-
const
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2716
|
+
};
|
|
2717
|
+
function tdmCandidates(work, tokens2 = {}, env = process.env) {
|
|
2718
|
+
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
2719
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2720
|
+
const candidates = [];
|
|
2721
|
+
const missingTokens = /* @__PURE__ */ new Set();
|
|
2722
|
+
for (const l of links) {
|
|
2723
|
+
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
2724
|
+
if (seen.has(l.URL)) {
|
|
2725
|
+
const prev = candidates.find((c) => c.url === l.URL);
|
|
2726
|
+
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
2727
|
+
continue;
|
|
2592
2728
|
}
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
coi = JSON.parse(fm.coi);
|
|
2602
|
-
} catch {
|
|
2603
|
-
coi = fm.coi;
|
|
2729
|
+
seen.add(l.URL);
|
|
2730
|
+
const rule = TOKENED.find((r) => r.host.test(hostOf3(l.URL)));
|
|
2731
|
+
let headers;
|
|
2732
|
+
if (rule) {
|
|
2733
|
+
const token = tokens2[rule.key]?.trim() || env[rule.env]?.trim();
|
|
2734
|
+
if (!token) {
|
|
2735
|
+
missingTokens.add(rule.key);
|
|
2736
|
+
continue;
|
|
2604
2737
|
}
|
|
2605
|
-
|
|
2606
|
-
|
|
2738
|
+
headers = { [rule.header]: token };
|
|
2739
|
+
}
|
|
2740
|
+
if (/text\/plain/.test(ct)) continue;
|
|
2741
|
+
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf3(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
2607
2742
|
}
|
|
2743
|
+
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
2744
|
+
candidates.sort((a, b) => rank(a) - rank(b));
|
|
2745
|
+
return { candidates, missingTokens: [...missingTokens] };
|
|
2746
|
+
}
|
|
2747
|
+
function tdmTier(tokens2 = {}) {
|
|
2608
2748
|
return {
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
2625
|
-
version: opt(fm.version),
|
|
2626
|
-
files: list2(fm.files),
|
|
2627
|
-
retrieved_by: fm.retrieved_by ?? "",
|
|
2628
|
-
retrieved_date: fm.retrieved_date ?? "",
|
|
2629
|
-
converted_by: fm.converted_by ?? "",
|
|
2630
|
-
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
2631
|
-
attempts: splitAttempts(fm.attempts)
|
|
2749
|
+
name: "tdm",
|
|
2750
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2751
|
+
lookup: (p) => fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }),
|
|
2752
|
+
async run(p, ctx) {
|
|
2753
|
+
let work;
|
|
2754
|
+
try {
|
|
2755
|
+
work = JSON.parse(await looked(this, p, ctx));
|
|
2756
|
+
} catch (e) {
|
|
2757
|
+
return miss(this.name, errMsg(e));
|
|
2758
|
+
}
|
|
2759
|
+
const { candidates, missingTokens } = tdmCandidates(work, tokens2);
|
|
2760
|
+
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
2761
|
+
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
2762
|
+
return download(this.name, p, candidates, ctx);
|
|
2763
|
+
}
|
|
2632
2764
|
};
|
|
2633
2765
|
}
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2766
|
+
var tdm = tdmTier();
|
|
2767
|
+
|
|
2768
|
+
// src/tiers/unpaywall.ts
|
|
2769
|
+
var unpaywall = {
|
|
2770
|
+
name: "unpaywall",
|
|
2771
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2772
|
+
lookup: (p) => fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(contactEmail())}`, { "User-Agent": LIBRARY_UA }),
|
|
2773
|
+
async run(p, ctx) {
|
|
2774
|
+
let data;
|
|
2775
|
+
try {
|
|
2776
|
+
data = JSON.parse(await looked(this, p, ctx));
|
|
2777
|
+
} catch (e) {
|
|
2778
|
+
return miss(this.name, errMsg(e));
|
|
2779
|
+
}
|
|
2780
|
+
const candidates = unpaywallCandidates(data);
|
|
2781
|
+
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
2782
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2783
|
+
return download(this.name, p, candidates, ctx);
|
|
2644
2784
|
}
|
|
2645
|
-
}
|
|
2646
|
-
function
|
|
2647
|
-
const
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
if (existsSync5(to.entryDir)) renameSync2(to.entryDir, old);
|
|
2657
|
-
renameSync2(tmp, to.entryDir);
|
|
2658
|
-
rmSync(old, { recursive: true, force: true });
|
|
2659
|
-
return copy;
|
|
2660
|
-
}
|
|
2661
|
-
var NOTHING_TO_VERIFY = "no abstract and no full text: nothing to verify";
|
|
2662
|
-
function citability(m) {
|
|
2663
|
-
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
2664
|
-
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
2665
|
-
if (m.retracted) {
|
|
2666
|
-
const r = m.corrections.filter(isRetractionNotice);
|
|
2667
|
-
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
2785
|
+
};
|
|
2786
|
+
function unpaywallCandidates(d) {
|
|
2787
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
|
|
2788
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2789
|
+
const out = [];
|
|
2790
|
+
for (const l of locs) {
|
|
2791
|
+
for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
|
|
2792
|
+
if (!url || seen.has(url)) continue;
|
|
2793
|
+
seen.add(url);
|
|
2794
|
+
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
2795
|
+
}
|
|
2668
2796
|
}
|
|
2669
|
-
|
|
2670
|
-
const notes = [
|
|
2671
|
-
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
2672
|
-
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
2673
|
-
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
2674
|
-
];
|
|
2675
|
-
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
2797
|
+
return out;
|
|
2676
2798
|
}
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2799
|
+
|
|
2800
|
+
// src/tiers/index.ts
|
|
2801
|
+
var BUILTIN = [
|
|
2802
|
+
{ name: "pmc", make: () => pmc },
|
|
2803
|
+
{ name: "europepmc", make: () => europepmc },
|
|
2804
|
+
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
2805
|
+
{ name: "unpaywall", make: () => unpaywall },
|
|
2806
|
+
{ name: "openalex", make: () => openalex },
|
|
2807
|
+
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
2808
|
+
];
|
|
2809
|
+
|
|
2810
|
+
// src/tiers/url.ts
|
|
2811
|
+
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
2812
|
+
function urlTier(name, template, headers) {
|
|
2813
|
+
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
2683
2814
|
return {
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
first_author: m.first_author,
|
|
2695
|
-
year: m.year,
|
|
2696
|
-
publication_types: m.publication_types,
|
|
2697
|
-
retracted: m.retracted,
|
|
2698
|
-
corrections: m.corrections,
|
|
2699
|
-
coi: m.coi,
|
|
2700
|
-
funding: m.funding,
|
|
2701
|
-
...cit,
|
|
2702
|
-
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
2703
|
-
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
2704
|
-
// there is abstract text).
|
|
2705
|
-
has_abstract: m.has_abstract,
|
|
2706
|
-
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
2707
|
-
version: m.version,
|
|
2708
|
-
retrieved_by: m.retrieved_by || null,
|
|
2709
|
-
retrieved_date: m.retrieved_date || null,
|
|
2710
|
-
pubmed_checked: m.pubmed_checked || null,
|
|
2711
|
-
attempts: m.attempts,
|
|
2712
|
-
files: m.files,
|
|
2713
|
-
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
2815
|
+
name,
|
|
2816
|
+
applies(p) {
|
|
2817
|
+
const missing = used.filter((k) => !p[k]);
|
|
2818
|
+
return missing.length ? `no ${missing.join("/")}` : true;
|
|
2819
|
+
},
|
|
2820
|
+
async run(p, ctx) {
|
|
2821
|
+
if (!used.length) return miss(name, "template has no placeholder");
|
|
2822
|
+
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
2823
|
+
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
2824
|
+
}
|
|
2714
2825
|
};
|
|
2715
2826
|
}
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2827
|
+
|
|
2828
|
+
// src/config.ts
|
|
2829
|
+
var VARIABLES = [
|
|
2830
|
+
"FH_LIBRARY_DIR",
|
|
2831
|
+
"FH_LIBRARY_ENV",
|
|
2832
|
+
"FH_LIBRARY_TIERS",
|
|
2833
|
+
"FH_LIBRARY_CONTACT_EMAIL",
|
|
2834
|
+
"FH_LIBRARY_ELSEVIER_API_KEY",
|
|
2835
|
+
"FH_LIBRARY_WILEY_TDM_TOKEN",
|
|
2836
|
+
"FH_LIBRARY_CORE_API_KEY",
|
|
2837
|
+
"FH_LIBRARY_PROXY_URL",
|
|
2838
|
+
"FH_LIBRARY_PROXY_HEADERS",
|
|
2839
|
+
"FH_LIBRARY_PLAYWRIGHT_1",
|
|
2840
|
+
"FH_LIBRARY_PLAYWRIGHT_2",
|
|
2841
|
+
"FH_LIBRARY_CHROME_1",
|
|
2842
|
+
"FH_LIBRARY_PLAYWRIGHT_HEADLESS",
|
|
2843
|
+
"FH_LIBRARY_VERIFY_URL",
|
|
2844
|
+
"FH_LIBRARY_VERIFY_API_KEY",
|
|
2845
|
+
"FH_LIBRARY_VERIFY_MODEL",
|
|
2846
|
+
"FH_LIBRARY_VERIFY_THRESHOLD",
|
|
2847
|
+
"FH_LIBRARY_VERIFY_TIMEOUT_SECS",
|
|
2848
|
+
"FH_LIBRARY_RETRY_AFTER_DAYS",
|
|
2849
|
+
"FH_LIBRARY_RETRY_BLOCKED",
|
|
2850
|
+
"FH_LIBRARY_REFRESH_AFTER_DAYS",
|
|
2851
|
+
"FH_LIBRARY_SEARCH_CACHE_DAYS",
|
|
2852
|
+
"FH_LIBRARY_NCBI_API_KEY",
|
|
2853
|
+
"FH_LIBRARY_BRIGHTDATA_API_TOKEN",
|
|
2854
|
+
"FH_LIBRARY_BRIGHTDATA_ZONE"
|
|
2855
|
+
];
|
|
2856
|
+
var sources = (env = process.env) => Object.fromEntries(VARIABLES.map((v) => [v, sourceOf(v, env)]));
|
|
2857
|
+
function settings(env = process.env) {
|
|
2858
|
+
return {
|
|
2859
|
+
retry_after_days: num("FH_LIBRARY_RETRY_AFTER_DAYS", 1, env),
|
|
2860
|
+
retry_blocked: bool("FH_LIBRARY_RETRY_BLOCKED", false, env),
|
|
2861
|
+
refresh_after_days: num("FH_LIBRARY_REFRESH_AFTER_DAYS", 30, env),
|
|
2862
|
+
search_cache_days: num("FH_LIBRARY_SEARCH_CACHE_DAYS", 7, env)
|
|
2744
2863
|
};
|
|
2745
|
-
writeMeta(p, next);
|
|
2746
|
-
return next;
|
|
2747
|
-
}
|
|
2748
|
-
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
2749
|
-
const t = new Date(date).getTime();
|
|
2750
|
-
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
2751
2864
|
}
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
|
|
2757
|
-
|
|
2758
|
-
|
|
2759
|
-
for (let i = 0; i < lines.length; i++) {
|
|
2760
|
-
const l = lines[i].trim();
|
|
2761
|
-
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
2762
|
-
if (h) {
|
|
2763
|
-
section2 = h[1];
|
|
2764
|
-
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
2765
|
-
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
2766
|
-
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
2767
|
-
continue;
|
|
2768
|
-
}
|
|
2769
|
-
if (section2 !== "Declarations" || !l) continue;
|
|
2770
|
-
const f = FUNDING_RE.exec(l);
|
|
2771
|
-
const c = COI_RE.exec(l);
|
|
2772
|
-
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
2773
|
-
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
2865
|
+
function proxyHeaders(spec) {
|
|
2866
|
+
const out = {};
|
|
2867
|
+
for (const part of spec.split(";")) {
|
|
2868
|
+
const i = part.indexOf(":");
|
|
2869
|
+
if (i < 0) continue;
|
|
2870
|
+
const name = part.slice(0, i).trim(), value = part.slice(i + 1).trim();
|
|
2871
|
+
if (name && value) out[name] = value;
|
|
2774
2872
|
}
|
|
2775
|
-
return
|
|
2873
|
+
return out;
|
|
2776
2874
|
}
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
const
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2875
|
+
function resolveTiers(env = process.env) {
|
|
2876
|
+
const problems = [];
|
|
2877
|
+
const available = /* @__PURE__ */ new Map();
|
|
2878
|
+
const tokens2 = { elsevier: str("FH_LIBRARY_ELSEVIER_API_KEY", env) || void 0, wiley: str("FH_LIBRARY_WILEY_TDM_TOKEN", env) || void 0 };
|
|
2879
|
+
for (const t of BUILTIN) available.set(t.name, t.make({ tokens: tokens2, api_key: str("FH_LIBRARY_CORE_API_KEY", env) || void 0 }));
|
|
2880
|
+
const proxyUrl = str("FH_LIBRARY_PROXY_URL", env);
|
|
2881
|
+
if (proxyUrl) available.set("proxy", urlTier("proxy", proxyUrl, proxyHeaders(str("FH_LIBRARY_PROXY_HEADERS", env))));
|
|
2882
|
+
const listed = str("FH_LIBRARY_TIERS", env);
|
|
2883
|
+
const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
|
|
2884
|
+
const tiers = [], seen = /* @__PURE__ */ new Set();
|
|
2885
|
+
for (const name of order2) {
|
|
2886
|
+
if (seen.has(name)) continue;
|
|
2887
|
+
seen.add(name);
|
|
2888
|
+
const t = available.get(name);
|
|
2889
|
+
if (t) tiers.push(t);
|
|
2890
|
+
else if (name === "proxy") problems.push('FH_LIBRARY_TIERS: "proxy" listed, but FH_LIBRARY_PROXY_URL is unset');
|
|
2891
|
+
else problems.push(`FH_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when FH_LIBRARY_PROXY_URL is set)`);
|
|
2785
2892
|
}
|
|
2786
|
-
|
|
2893
|
+
const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
|
|
2894
|
+
return { tiers, all, source: sourceOf("FH_LIBRARY_TIERS", env), problems };
|
|
2787
2895
|
}
|
|
2788
|
-
function withCorrectedBlock(md, sentences) {
|
|
2789
|
-
const base = md.split(`
|
|
2790
|
-
${CORRECTED_HEADING}
|
|
2791
|
-
`)[0].trimEnd();
|
|
2792
|
-
if (!sentences.length) return base + "\n";
|
|
2793
|
-
return `${base}
|
|
2794
|
-
|
|
2795
|
-
${CORRECTED_HEADING}
|
|
2796
|
-
|
|
2797
|
-
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
2798
2896
|
|
|
2799
|
-
|
|
2800
|
-
|
|
2897
|
+
// src/log.ts
|
|
2898
|
+
import { appendFileSync, mkdirSync as mkdirSync4, renameSync as renameSync2, statSync } from "fs";
|
|
2899
|
+
import { hostname } from "os";
|
|
2900
|
+
import { join as join6 } from "path";
|
|
2901
|
+
var HOST = hostname();
|
|
2902
|
+
var LOG_FILE = join6(FH_LIBRARY_DIR, "library.log");
|
|
2903
|
+
function stamp() {
|
|
2904
|
+
const d = /* @__PURE__ */ new Date();
|
|
2905
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
2906
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
2801
2907
|
}
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
let
|
|
2908
|
+
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
2909
|
+
function rotateIfLarge(file3, max = LOG_MAX_BYTES) {
|
|
2910
|
+
let size = 0;
|
|
2805
2911
|
try {
|
|
2806
|
-
|
|
2912
|
+
size = statSync(file3).size;
|
|
2807
2913
|
} catch {
|
|
2808
|
-
return;
|
|
2914
|
+
return false;
|
|
2915
|
+
}
|
|
2916
|
+
if (size <= max) return false;
|
|
2917
|
+
try {
|
|
2918
|
+
renameSync2(file3, `${file3}.1`);
|
|
2919
|
+
return true;
|
|
2920
|
+
} catch {
|
|
2921
|
+
return false;
|
|
2809
2922
|
}
|
|
2810
|
-
if (!r) return;
|
|
2811
|
-
const md = readFileSync4(p.fulltextMd, "utf8");
|
|
2812
|
-
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
2813
|
-
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
2814
|
-
}
|
|
2815
|
-
function applyFulltext(p, m, hit, converter) {
|
|
2816
|
-
const raw = p.fulltextRaw(hit.format);
|
|
2817
|
-
writeAtomic(raw, hit.raw);
|
|
2818
|
-
writeAtomic(p.fulltextMd, hit.md);
|
|
2819
|
-
syncCorrectedBlock(p);
|
|
2820
|
-
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
2821
|
-
const own = declarationsFromMd(hit.md);
|
|
2822
|
-
const next = {
|
|
2823
|
-
...m,
|
|
2824
|
-
level: "full_text",
|
|
2825
|
-
version: hit.version,
|
|
2826
|
-
files,
|
|
2827
|
-
retrieved_by: hit.tier,
|
|
2828
|
-
retrieved_date: today(),
|
|
2829
|
-
converted_by: converter,
|
|
2830
|
-
coi: m.coi ?? own.coi,
|
|
2831
|
-
funding: m.funding.length ? m.funding : own.funding
|
|
2832
|
-
};
|
|
2833
|
-
writeMeta(p, next);
|
|
2834
|
-
return next;
|
|
2835
|
-
}
|
|
2836
|
-
function recordAttempts(p, m, labels) {
|
|
2837
|
-
if (!labels.length) return m;
|
|
2838
|
-
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
2839
|
-
const replaced = new Set(labels.map(tierOf));
|
|
2840
|
-
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
2841
|
-
writeMeta(p, next);
|
|
2842
|
-
return next;
|
|
2843
|
-
}
|
|
2844
|
-
function writeLabelEntry(p, s, xml, converter) {
|
|
2845
|
-
mkdirSync3(p.entryDir, { recursive: true });
|
|
2846
|
-
const raw = p.fulltextRaw("xml");
|
|
2847
|
-
writeAtomic(raw, xml);
|
|
2848
|
-
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
2849
|
-
const meta = {
|
|
2850
|
-
pmid: "",
|
|
2851
|
-
setid: s.setid,
|
|
2852
|
-
nctid: null,
|
|
2853
|
-
publications: [],
|
|
2854
|
-
pmcid: null,
|
|
2855
|
-
doi: null,
|
|
2856
|
-
title: s.title,
|
|
2857
|
-
first_author: s.labeler,
|
|
2858
|
-
year: s.effective?.slice(0, 4) ?? null,
|
|
2859
|
-
publication_types: ["Drug Label"],
|
|
2860
|
-
retracted: false,
|
|
2861
|
-
corrections: [],
|
|
2862
|
-
coi: null,
|
|
2863
|
-
funding: [],
|
|
2864
|
-
has_abstract: false,
|
|
2865
|
-
level: "full_text",
|
|
2866
|
-
version: s.version ? `spl ${s.version}` : null,
|
|
2867
|
-
files: [basename(raw), basename(p.fulltextMd)],
|
|
2868
|
-
retrieved_by: "dailymed",
|
|
2869
|
-
retrieved_date: today(),
|
|
2870
|
-
converted_by: converter,
|
|
2871
|
-
pubmed_checked: today(),
|
|
2872
|
-
attempts: []
|
|
2873
|
-
};
|
|
2874
|
-
writeMeta(p, meta);
|
|
2875
|
-
return meta;
|
|
2876
|
-
}
|
|
2877
|
-
function writeStudyEntry(p, s, json2, converter) {
|
|
2878
|
-
mkdirSync3(p.entryDir, { recursive: true });
|
|
2879
|
-
const raw = p.fulltextRaw("json");
|
|
2880
|
-
writeAtomic(raw, json2);
|
|
2881
|
-
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
2882
|
-
const meta = {
|
|
2883
|
-
pmid: "",
|
|
2884
|
-
setid: null,
|
|
2885
|
-
nctid: s.nctid,
|
|
2886
|
-
publications: s.publications,
|
|
2887
|
-
pmcid: null,
|
|
2888
|
-
doi: null,
|
|
2889
|
-
title: s.title,
|
|
2890
|
-
first_author: s.sponsor,
|
|
2891
|
-
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
2892
|
-
publication_types: ["Trial Registration"],
|
|
2893
|
-
retracted: false,
|
|
2894
|
-
corrections: [],
|
|
2895
|
-
coi: null,
|
|
2896
|
-
funding: [],
|
|
2897
|
-
has_abstract: false,
|
|
2898
|
-
level: "full_text",
|
|
2899
|
-
version: s.lastUpdate,
|
|
2900
|
-
files: [basename(raw), basename(p.fulltextMd)],
|
|
2901
|
-
retrieved_by: "ctgov",
|
|
2902
|
-
retrieved_date: today(),
|
|
2903
|
-
converted_by: converter,
|
|
2904
|
-
pubmed_checked: today(),
|
|
2905
|
-
attempts: []
|
|
2906
|
-
};
|
|
2907
|
-
writeMeta(p, meta);
|
|
2908
|
-
return meta;
|
|
2909
2923
|
}
|
|
2910
|
-
function
|
|
2911
|
-
|
|
2912
|
-
const
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2924
|
+
function log(tool, id, message) {
|
|
2925
|
+
const clean2 = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
2926
|
+
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean2}
|
|
2927
|
+
`;
|
|
2928
|
+
try {
|
|
2929
|
+
mkdirSync4(FH_LIBRARY_DIR, { recursive: true });
|
|
2930
|
+
rotateIfLarge(LOG_FILE);
|
|
2931
|
+
appendFileSync(LOG_FILE, line);
|
|
2932
|
+
} catch {
|
|
2933
|
+
}
|
|
2918
2934
|
}
|
|
2919
2935
|
|
|
2920
2936
|
// src/stats.ts
|
|
2937
|
+
import { mkdirSync as mkdirSync5, readdirSync as readdirSync2, readFileSync as readFileSync5, statSync as statSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
2938
|
+
import { join as join7 } from "path";
|
|
2921
2939
|
var HEARTBEAT_MS = 5 * 6e4;
|
|
2922
2940
|
var STALE_MS = 20 * 6e4;
|
|
2923
2941
|
var stats = null;
|
|
2924
|
-
var
|
|
2942
|
+
var file2 = null;
|
|
2925
2943
|
function startStats(pkg2, libraryDir = FH_LIBRARY_DIR) {
|
|
2926
2944
|
stats = { host: HOST, pid: process.pid, package: pkg2.name, version: pkg2.version, started: (/* @__PURE__ */ new Date()).toISOString(), verify: { hits: 0, misses: 0 }, fetch: { hits: 0, misses: 0 }, search: { hits: 0, misses: 0 }, pubmed: { hits: 0, misses: 0 } };
|
|
2927
|
-
|
|
2945
|
+
file2 = join7(libraryDir, ".servers", `${HOST}-${process.pid}.json`);
|
|
2928
2946
|
sweep(join7(libraryDir, ".servers"));
|
|
2929
2947
|
publish();
|
|
2930
2948
|
setInterval(publish, HEARTBEAT_MS).unref();
|
|
2931
2949
|
const stop = () => {
|
|
2932
2950
|
try {
|
|
2933
|
-
if (
|
|
2951
|
+
if (file2) unlinkSync2(file2);
|
|
2934
2952
|
} catch {
|
|
2935
2953
|
}
|
|
2936
2954
|
};
|
|
@@ -2942,7 +2960,7 @@ function sweep(dir, staleMs = STALE_MS) {
|
|
|
2942
2960
|
let n = 0;
|
|
2943
2961
|
let names;
|
|
2944
2962
|
try {
|
|
2945
|
-
names =
|
|
2963
|
+
names = readdirSync2(dir);
|
|
2946
2964
|
} catch {
|
|
2947
2965
|
return 0;
|
|
2948
2966
|
}
|
|
@@ -2951,7 +2969,7 @@ function sweep(dir, staleMs = STALE_MS) {
|
|
|
2951
2969
|
const p = join7(dir, name);
|
|
2952
2970
|
try {
|
|
2953
2971
|
if (Date.now() - statSync2(p).mtimeMs > staleMs) {
|
|
2954
|
-
|
|
2972
|
+
unlinkSync2(p);
|
|
2955
2973
|
n++;
|
|
2956
2974
|
}
|
|
2957
2975
|
} catch {
|
|
@@ -2966,16 +2984,16 @@ function bump(kind, hit, n = 1) {
|
|
|
2966
2984
|
}
|
|
2967
2985
|
var currentStats = () => stats;
|
|
2968
2986
|
function publish() {
|
|
2969
|
-
if (!stats || !
|
|
2987
|
+
if (!stats || !file2) return;
|
|
2970
2988
|
try {
|
|
2971
|
-
|
|
2972
|
-
writeAtomic(
|
|
2989
|
+
mkdirSync5(join7(file2, ".."), { recursive: true });
|
|
2990
|
+
writeAtomic(file2, JSON.stringify(stats, null, 2) + "\n");
|
|
2973
2991
|
} catch {
|
|
2974
2992
|
}
|
|
2975
2993
|
}
|
|
2976
2994
|
|
|
2977
2995
|
// src/tools.ts
|
|
2978
|
-
import { existsSync as
|
|
2996
|
+
import { existsSync as existsSync12, readFileSync as readFileSync13 } from "fs";
|
|
2979
2997
|
import { z } from "zod";
|
|
2980
2998
|
|
|
2981
2999
|
// src/cascade.ts
|
|
@@ -3007,10 +3025,18 @@ function shouldSkip(attempts, tier, retryAfterDays, now = /* @__PURE__ */ new Da
|
|
|
3007
3025
|
async function fulltextCascade(p, attempts, onEvent, tiers = resolveTiers().tiers, retryAfterDays = settings().retry_after_days, retryBlocked = settings().retry_blocked) {
|
|
3008
3026
|
const misses = [];
|
|
3009
3027
|
const tried = /* @__PURE__ */ new Set();
|
|
3028
|
+
const lookups = /* @__PURE__ */ new Map();
|
|
3010
3029
|
for (const tier of tiers) {
|
|
3011
|
-
if (!tier.
|
|
3030
|
+
if (!tier.lookup || shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked) || tier.applies(p) !== true) continue;
|
|
3031
|
+
const q = tier.lookup(p);
|
|
3032
|
+
q.catch(() => {
|
|
3033
|
+
});
|
|
3034
|
+
lookups.set(tier.name, q);
|
|
3035
|
+
}
|
|
3036
|
+
for (const tier of tiers) {
|
|
3037
|
+
if (shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked)) continue;
|
|
3012
3038
|
const applies = tier.applies(p);
|
|
3013
|
-
const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
|
|
3039
|
+
const ctx = { tried, lookups, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
|
|
3014
3040
|
const r = applies === true ? await tier.run(p, ctx) : { ...miss(tier.name, applies), skipped: true };
|
|
3015
3041
|
onEvent?.(r);
|
|
3016
3042
|
if (r.ok) return { hit: r, misses };
|
|
@@ -3050,11 +3076,11 @@ function parseId(raw) {
|
|
|
3050
3076
|
}
|
|
3051
3077
|
|
|
3052
3078
|
// src/info.ts
|
|
3053
|
-
import { existsSync as
|
|
3079
|
+
import { existsSync as existsSync6, readdirSync as readdirSync3, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
|
|
3054
3080
|
import { dirname as dirname2, join as join8 } from "path";
|
|
3055
3081
|
|
|
3056
3082
|
// src/verify.ts
|
|
3057
|
-
import { existsSync as
|
|
3083
|
+
import { existsSync as existsSync5, readFileSync as readFileSync6 } from "fs";
|
|
3058
3084
|
var DEFAULT_MODEL = "pplx-decider-flash-v1.2";
|
|
3059
3085
|
var QUESTION = "Does the paper state this, in substance? Ignore section labels and definitions of terms in parentheses. Every fact and number of the statement must be stated in the paper or follow from its numbers; a rounded number counts. If any part of the statement is absent from the paper or contradicted by it, answer false.";
|
|
3060
3086
|
function verifyConfig(env = process.env) {
|
|
@@ -3156,12 +3182,12 @@ async function readPaper(raw, libraryDir = FH_LIBRARY_DIR) {
|
|
|
3156
3182
|
if (p.error) return { id: p.requested, error: p.error };
|
|
3157
3183
|
const paths = entryPaths(libraryDir, p.type, p.id);
|
|
3158
3184
|
if (!entryExists(paths)) return { id: p.requested, error: "not in library" };
|
|
3159
|
-
const full =
|
|
3185
|
+
const full = existsSync5(paths.fulltextMd);
|
|
3160
3186
|
if (full) return { id: p.requested, basis: "full_text", text: readFileSync6(paths.fulltextMd, "utf8") };
|
|
3161
3187
|
const meta = readMeta(paths);
|
|
3162
3188
|
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync6(paths.md, "utf8") };
|
|
3163
3189
|
}
|
|
3164
|
-
async function verifyClaim(claim, id, cfg, decide,
|
|
3190
|
+
async function verifyClaim(claim, id, cfg, decide, read2 = readPaper) {
|
|
3165
3191
|
const out = { claim, id, verdict: "error", reason: null, score: null, basis: null, windows: 0, glyphs_unresolved: 0, superseded: null, model: cfg?.model ?? null };
|
|
3166
3192
|
const words2 = claim.trim().split(/\s+/).filter(Boolean).length;
|
|
3167
3193
|
if (!words2) return { ...out, error: "claim empty" };
|
|
@@ -3170,7 +3196,7 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
3170
3196
|
if (frag) return { ...out, error: frag };
|
|
3171
3197
|
const groups = citationGroups(claim);
|
|
3172
3198
|
if (groups.length) return { ...out, error: groups.length > 1 ? `compound claim: ${groups.length} citation groups ${groups.join(" ")} \u2014 one finding per call, without its sources` : `claim names a source ${groups[0]} \u2014 send the finding without its sources; the paper is the id` };
|
|
3173
|
-
const paper = await
|
|
3199
|
+
const paper = await read2(id);
|
|
3174
3200
|
out.id = paper.id;
|
|
3175
3201
|
if ("error" in paper) return { ...out, error: paper.error };
|
|
3176
3202
|
if (!cfg || !decide) return { ...out, error: "no verify model configured (FH_LIBRARY_VERIFY_URL)" };
|
|
@@ -3200,8 +3226,8 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
3200
3226
|
|
|
3201
3227
|
// src/info.ts
|
|
3202
3228
|
function* metaFiles(dir) {
|
|
3203
|
-
if (!
|
|
3204
|
-
for (const name of
|
|
3229
|
+
if (!existsSync6(dir)) return;
|
|
3230
|
+
for (const name of readdirSync3(dir)) {
|
|
3205
3231
|
const p = join8(dir, name);
|
|
3206
3232
|
if (statSync3(p).isDirectory()) yield* metaFiles(p);
|
|
3207
3233
|
else if (name.endsWith("-meta.md")) yield p;
|
|
@@ -3220,10 +3246,10 @@ function field(md, key) {
|
|
|
3220
3246
|
const m = new RegExp(`^${key}:\\s*(.*)$`, "m").exec(md);
|
|
3221
3247
|
return m ? m[1].trim() : "";
|
|
3222
3248
|
}
|
|
3223
|
-
function libraryInfo(libraryDir,
|
|
3249
|
+
function libraryInfo(libraryDir, host2, withIds, tiers = resolveTiers()) {
|
|
3224
3250
|
const info = {
|
|
3225
3251
|
library_dir: libraryDir,
|
|
3226
|
-
host,
|
|
3252
|
+
host: host2,
|
|
3227
3253
|
tiers: tiers.all,
|
|
3228
3254
|
config: { ...settings(), problems: tiers.problems, sources: sources() },
|
|
3229
3255
|
download: downloadInfo(),
|
|
@@ -3241,7 +3267,7 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3241
3267
|
updates: 0
|
|
3242
3268
|
};
|
|
3243
3269
|
const ids = [];
|
|
3244
|
-
for (const type of
|
|
3270
|
+
for (const type of existsSync6(libraryDir) ? readdirSync3(libraryDir) : []) {
|
|
3245
3271
|
const typeDir = join8(libraryDir, type);
|
|
3246
3272
|
if (!statSync3(typeDir).isDirectory()) continue;
|
|
3247
3273
|
for (const meta of metaFiles(typeDir)) {
|
|
@@ -3255,7 +3281,7 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3255
3281
|
t.entries++;
|
|
3256
3282
|
if (level === "full_text" || level === "abstract" || level === "title") t[level]++;
|
|
3257
3283
|
const dir = dirname2(meta);
|
|
3258
|
-
for (const f of
|
|
3284
|
+
for (const f of readdirSync3(dir)) {
|
|
3259
3285
|
try {
|
|
3260
3286
|
const st = statSync3(join8(dir, f));
|
|
3261
3287
|
if (st.isFile()) info.size_bytes += st.size;
|
|
@@ -3412,15 +3438,18 @@ function search(md, query, maxHits, opts = {}) {
|
|
|
3412
3438
|
const df = /* @__PURE__ */ new Map();
|
|
3413
3439
|
for (const x of scored) for (const s of x.matched) df.set(s, (df.get(s) ?? 0) + 1);
|
|
3414
3440
|
const rarity = (x) => x.matched.reduce((n, s) => n + 1 / df.get(s), 0);
|
|
3415
|
-
const
|
|
3416
|
-
const
|
|
3441
|
+
const pool = partial ? scored : full;
|
|
3442
|
+
const pick = pool.sort((a, b) => b.matched.length - a.matched.length || (partial ? rarity(b) - rarity(a) : 0) || b.hits - a.hits || a.p.text.length - b.p.text.length).slice(0, maxHits);
|
|
3443
|
+
const passages = pick.map(({ p, matched }) => {
|
|
3417
3444
|
const w = clip(p, matched, window);
|
|
3418
|
-
const out = { section: p.section, text: w.text
|
|
3445
|
+
const out = { section: p.section, text: w.text };
|
|
3419
3446
|
if (partial) out.matched = terms.filter((t) => matched.includes(stem(t)));
|
|
3420
3447
|
if (w.clipped) out.clipped = true;
|
|
3421
3448
|
return out;
|
|
3422
3449
|
});
|
|
3423
|
-
|
|
3450
|
+
const r = { passages, partial, missing_terms };
|
|
3451
|
+
if (pool.length > passages.length) r.more = pool.length - passages.length;
|
|
3452
|
+
return r;
|
|
3424
3453
|
}
|
|
3425
3454
|
function paragraphs(md, scope) {
|
|
3426
3455
|
const lines = md.split("\n");
|
|
@@ -3665,7 +3694,7 @@ function parseCsl(json2) {
|
|
|
3665
3694
|
|
|
3666
3695
|
// src/urlfetch.ts
|
|
3667
3696
|
import { createHash as createHash2 } from "crypto";
|
|
3668
|
-
import { existsSync as
|
|
3697
|
+
import { existsSync as existsSync7, mkdirSync as mkdirSync6, readFileSync as readFileSync8 } from "fs";
|
|
3669
3698
|
import { join as join9 } from "path";
|
|
3670
3699
|
var TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src)$/i;
|
|
3671
3700
|
function normalizeUrl(raw) {
|
|
@@ -3708,7 +3737,7 @@ function renderUrlMeta(m) {
|
|
|
3708
3737
|
return lines.map(([k, v]) => `${k}: ${v}`).join("\n") + "\n";
|
|
3709
3738
|
}
|
|
3710
3739
|
function readUrlMeta(p) {
|
|
3711
|
-
if (!
|
|
3740
|
+
if (!existsSync7(p.meta)) return null;
|
|
3712
3741
|
const md = readFileSync8(p.meta, "utf8");
|
|
3713
3742
|
const f = (k) => {
|
|
3714
3743
|
const m = new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md);
|
|
@@ -3735,7 +3764,7 @@ function readUrlMeta(p) {
|
|
|
3735
3764
|
};
|
|
3736
3765
|
}
|
|
3737
3766
|
function writeUrlMeta(p, m) {
|
|
3738
|
-
|
|
3767
|
+
mkdirSync6(p.entryDir, { recursive: true });
|
|
3739
3768
|
writeAtomic(p.meta, renderUrlMeta(m));
|
|
3740
3769
|
}
|
|
3741
3770
|
var SHELL_WORDS = 100;
|
|
@@ -3782,18 +3811,27 @@ var climb = async (url, onTry, onAlert) => {
|
|
|
3782
3811
|
let shell = !r.ok ? r.shell : void 0;
|
|
3783
3812
|
for (const b of browserRungs()) {
|
|
3784
3813
|
if (!escalates(r)) break;
|
|
3814
|
+
const w = walled(target, `browser ${b.label}`);
|
|
3815
|
+
if (w) {
|
|
3816
|
+
onTry?.(`skipped browser ${b.label}: ${wallNote(w)}`);
|
|
3817
|
+
continue;
|
|
3818
|
+
}
|
|
3785
3819
|
r = await rung(target, { browser: b.endpoint, label: b.label });
|
|
3820
|
+
recordRung(target, `browser ${b.label}`, r.ok, r.ok ? "" : r.reason);
|
|
3786
3821
|
if (!r.ok && r.shell && !shell) shell = r.shell;
|
|
3787
3822
|
onTry?.(r.ok ? `ok browser ${b.label}` : `failed browser ${b.label}: ${r.reason}`);
|
|
3788
3823
|
if (!r.ok && r.reason.startsWith(MISMATCH)) onAlert?.(`browser ${b.label}: ${r.reason}`);
|
|
3789
3824
|
}
|
|
3790
|
-
|
|
3825
|
+
const uw = !r.ok && unlockerConfigured() ? walled(target, "brightdata") : null;
|
|
3826
|
+
if (uw) onTry?.(`skipped brightdata: ${wallNote(uw)}`);
|
|
3827
|
+
else if (!r.ok && unlockerConfigured() && (r.via !== "fetch" || /^(HTTP 403|wall page|script shell)/.test(r.reason))) {
|
|
3791
3828
|
r = await rung(target, "brightdata");
|
|
3829
|
+
recordRung(target, "brightdata", r.ok, r.ok ? "" : r.reason);
|
|
3792
3830
|
onTry?.(r.ok ? "ok brightdata" : `failed brightdata: ${r.reason}`);
|
|
3793
3831
|
if (!r.ok && /^unlocker HTTP/.test(r.reason)) onAlert?.(`brightdata: ${r.reason}`);
|
|
3794
3832
|
}
|
|
3795
3833
|
if (!r.ok && shell) {
|
|
3796
|
-
onTry?.(
|
|
3834
|
+
onTry?.("shell kept: no rung rendered the page, the script shell is served");
|
|
3797
3835
|
return shell;
|
|
3798
3836
|
}
|
|
3799
3837
|
return r.ok || r.via === "fetch" ? r : { ...r, first: firstReason };
|
|
@@ -3886,6 +3924,25 @@ function logId(url) {
|
|
|
3886
3924
|
const bare2 = url.replace(/^https?:\/\//i, "");
|
|
3887
3925
|
return bare2.length > ID_WIDTH ? bare2.slice(0, ID_WIDTH - 1) + "\u2026" : bare2;
|
|
3888
3926
|
}
|
|
3927
|
+
function libraryIdForUrl(raw) {
|
|
3928
|
+
let u;
|
|
3929
|
+
try {
|
|
3930
|
+
u = new URL(raw.trim());
|
|
3931
|
+
} catch {
|
|
3932
|
+
return null;
|
|
3933
|
+
}
|
|
3934
|
+
const host2 = u.hostname.toLowerCase().replace(/^www\./, ""), path = u.pathname;
|
|
3935
|
+
let m;
|
|
3936
|
+
if (host2 === "pubmed.ncbi.nlm.nih.gov" && (m = /^\/(\d{1,8})\/?$/.exec(path))) return `pmid:${m[1]}`;
|
|
3937
|
+
if (host2 === "ncbi.nlm.nih.gov" && (m = /^\/pubmed\/(\d{1,8})\/?$/.exec(path))) return `pmid:${m[1]}`;
|
|
3938
|
+
if (host2 === "clinicaltrials.gov" && (m = /^\/(?:study|ct2\/show)\/(NCT\d{8})\/?$/i.exec(path))) return `nct:${m[1].slice(3)}`;
|
|
3939
|
+
if (host2 === "dailymed.nlm.nih.gov" && /\/(drugInfo|lookup)\.cfm$/i.test(path)) {
|
|
3940
|
+
const id = u.searchParams.get("setid");
|
|
3941
|
+
if (id && /^[0-9a-f-]{36}$/i.test(id)) return `dailymed:${id.toLowerCase()}`;
|
|
3942
|
+
}
|
|
3943
|
+
if ((host2 === "doi.org" || host2 === "dx.doi.org") && (m = /^\/(10\.\S+)$/.exec(decodeURIComponent(path)))) return `doi:${m[1]}`;
|
|
3944
|
+
return null;
|
|
3945
|
+
}
|
|
3889
3946
|
var failureText = (url, reason) => {
|
|
3890
3947
|
const m = /^HTTP (\d{3})\b/.exec(reason);
|
|
3891
3948
|
return m ? `Failed to fetch ${url} - status code ${m[1]}` : `Failed to fetch ${url}: ${reason}`;
|
|
@@ -3898,17 +3955,17 @@ function titleLine(m, md) {
|
|
|
3898
3955
|
}
|
|
3899
3956
|
function serve(requested, p, m, args, cached) {
|
|
3900
3957
|
const maxLength = args.max_length ?? 5e3, startIndex = args.start_index ?? 0;
|
|
3901
|
-
const
|
|
3958
|
+
const read2 = (file3) => existsSync7(file3) ? readFileSync8(file3, "utf8") : "";
|
|
3902
3959
|
if (args.title_only) return { ok: true, text: `Contents of ${requested}:
|
|
3903
|
-
${titleLine(m,
|
|
3960
|
+
${titleLine(m, read2(p.md))}`, cached, meta: m };
|
|
3904
3961
|
const rawFile = m.files.find((f) => !f.endsWith("-meta.md") && !f.endsWith("-page.md"));
|
|
3905
3962
|
let body, prefix = "";
|
|
3906
|
-
if (m.format === "markdown") body =
|
|
3963
|
+
if (m.format === "markdown") body = read2(p.md);
|
|
3907
3964
|
else if (m.format === "text" || args.raw) {
|
|
3908
|
-
body = rawFile ?
|
|
3965
|
+
body = rawFile ? read2(join9(p.entryDir, rawFile)) : read2(p.md);
|
|
3909
3966
|
prefix = `Content type ${m.content_type || "(not declared)"} cannot be simplified to markdown, but here is the raw content:
|
|
3910
3967
|
`;
|
|
3911
|
-
} else body =
|
|
3968
|
+
} else body = read2(p.md);
|
|
3912
3969
|
return { ok: true, text: fetchAnswer(requested, body, prefix, maxLength, startIndex), cached, meta: m };
|
|
3913
3970
|
}
|
|
3914
3971
|
var inflight = /* @__PURE__ */ new Map();
|
|
@@ -3932,7 +3989,7 @@ async function libraryFetch(libraryDir, requested, args, converter, logs = {}, f
|
|
|
3932
3989
|
return;
|
|
3933
3990
|
}
|
|
3934
3991
|
const page = await convert(r.bytes, r.contentType);
|
|
3935
|
-
|
|
3992
|
+
mkdirSync6(p.entryDir, { recursive: true });
|
|
3936
3993
|
const files = [`${p.entry}-meta.md`];
|
|
3937
3994
|
if (page.md !== null) {
|
|
3938
3995
|
writeAtomic(p.md, page.md);
|
|
@@ -3954,7 +4011,7 @@ async function libraryFetch(libraryDir, requested, args, converter, logs = {}, f
|
|
|
3954
4011
|
|
|
3955
4012
|
// src/verifycache.ts
|
|
3956
4013
|
import { createHash as createHash3 } from "crypto";
|
|
3957
|
-
import { existsSync as
|
|
4014
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync7, readFileSync as readFileSync9 } from "fs";
|
|
3958
4015
|
import { dirname as dirname3, join as join10 } from "path";
|
|
3959
4016
|
var sha = (s) => createHash3("sha1").update(s).digest("hex");
|
|
3960
4017
|
var normalizeClaim = (claim) => claim.replace(/\s+/g, " ").trim();
|
|
@@ -3968,35 +4025,35 @@ ${model}
|
|
|
3968
4025
|
${threshold}
|
|
3969
4026
|
${text3}`).slice(0, 16);
|
|
3970
4027
|
var verdictFile = (libraryDir, key) => join10(libraryDir, "verify", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
|
|
3971
|
-
function readVerdict(
|
|
3972
|
-
if (!
|
|
4028
|
+
function readVerdict(file3) {
|
|
4029
|
+
if (!existsSync8(file3)) return null;
|
|
3973
4030
|
try {
|
|
3974
|
-
return JSON.parse(readFileSync9(
|
|
4031
|
+
return JSON.parse(readFileSync9(file3, "utf8"));
|
|
3975
4032
|
} catch {
|
|
3976
4033
|
return null;
|
|
3977
4034
|
}
|
|
3978
4035
|
}
|
|
3979
|
-
function lookup(
|
|
3980
|
-
const c = readVerdict(
|
|
4036
|
+
function lookup(file3, maxDays) {
|
|
4037
|
+
const c = readVerdict(file3);
|
|
3981
4038
|
return c && daysSince(c.date) < maxDays ? c : null;
|
|
3982
4039
|
}
|
|
3983
|
-
function remember(
|
|
3984
|
-
|
|
3985
|
-
writeAtomic(
|
|
4040
|
+
function remember(file3, c) {
|
|
4041
|
+
mkdirSync7(dirname3(file3), { recursive: true });
|
|
4042
|
+
writeAtomic(file3, JSON.stringify(c, null, 2) + "\n");
|
|
3986
4043
|
}
|
|
3987
|
-
async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR,
|
|
3988
|
-
const paper = await
|
|
4044
|
+
async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR, read2 = readPaper) {
|
|
4045
|
+
const paper = await read2(id, libraryDir);
|
|
3989
4046
|
const once = () => paper;
|
|
3990
4047
|
if ("error" in paper || !cfg) return { ...await verifyClaim(claim, id, cfg, decide, once), cached: false };
|
|
3991
4048
|
const text3 = textHash(paper.text);
|
|
3992
|
-
const
|
|
3993
|
-
const hit = lookup(
|
|
4049
|
+
const file3 = verdictFile(libraryDir, cacheKey(claim, paper.id, cfg.model, cfg.threshold, text3));
|
|
4050
|
+
const hit = lookup(file3, settings().refresh_after_days);
|
|
3994
4051
|
if (hit) {
|
|
3995
4052
|
return { claim, id: paper.id, verdict: hit.verdict, reason: hit.reason, score: hit.score, basis: hit.basis, windows: hit.windows, glyphs_unresolved: hit.glyphs_unresolved, superseded: hit.superseded, model: hit.model, cached: true };
|
|
3996
4053
|
}
|
|
3997
4054
|
const r = await verifyClaim(claim, id, cfg, decide, once);
|
|
3998
4055
|
if (!r.error && r.score !== null) {
|
|
3999
|
-
remember(
|
|
4056
|
+
remember(file3, { key: file3.slice(-21, -5), id: paper.id, claim: normalizeClaim(claim), model: cfg.model, threshold: cfg.threshold, text: text3, date: today(), verdict: r.verdict, reason: r.reason, score: r.score, basis: r.basis, windows: r.windows, glyphs_unresolved: r.glyphs_unresolved, superseded: r.superseded });
|
|
4000
4057
|
}
|
|
4001
4058
|
return { ...r, cached: false };
|
|
4002
4059
|
}
|
|
@@ -4018,10 +4075,34 @@ function leanObject(o) {
|
|
|
4018
4075
|
return out;
|
|
4019
4076
|
}
|
|
4020
4077
|
var leanJson = (o) => compact2(lean(o));
|
|
4078
|
+
var STATUS_DROP = /* @__PURE__ */ new Set(["requested", "type", "id", "entry", "retrieved_date", "pubmed_checked", "attempts", "coi", "funding", "title", "first_author", "doi"]);
|
|
4079
|
+
function statusAnswer(s) {
|
|
4080
|
+
const o = {};
|
|
4081
|
+
for (const [k, v] of Object.entries(s)) if (!STATUS_DROP.has(k)) o[k] = v;
|
|
4082
|
+
if (Array.isArray(o.publication_types)) o.publication_types = reviewTypes(o.publication_types);
|
|
4083
|
+
return leanJson(o);
|
|
4084
|
+
}
|
|
4085
|
+
function verdictAnswer(r) {
|
|
4086
|
+
if (r.error) return compact2({ verdict: "error", error: r.error });
|
|
4087
|
+
const o = { verdict: r.verdict, score: r.score === null ? null : Math.round(r.score * 100) / 100, basis: r.basis ?? null };
|
|
4088
|
+
if (r.reason) o.reason = r.reason;
|
|
4089
|
+
if (r.superseded) o.superseded = true;
|
|
4090
|
+
if (r.glyphs_unresolved) o.glyphs_unresolved = r.glyphs_unresolved;
|
|
4091
|
+
return compact2(o);
|
|
4092
|
+
}
|
|
4093
|
+
function readAnswer(o) {
|
|
4094
|
+
const { requested: _r, entry: _e, ...rest } = o;
|
|
4095
|
+
if (rest.partial === false) delete rest.partial;
|
|
4096
|
+
return leanJson(rest);
|
|
4097
|
+
}
|
|
4098
|
+
function searchAnswer(r) {
|
|
4099
|
+
return compact2({ total: r.total, columns: r.columns, rows: r.rows });
|
|
4100
|
+
}
|
|
4101
|
+
var reviewTypes = (types) => types.filter((t) => t !== "Journal Article" && !/^Research Support\b/.test(t));
|
|
4021
4102
|
|
|
4022
4103
|
// src/trials.ts
|
|
4023
4104
|
import { createHash as createHash4 } from "crypto";
|
|
4024
|
-
import { existsSync as
|
|
4105
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync8, readFileSync as readFileSync10 } from "fs";
|
|
4025
4106
|
import { dirname as dirname4, join as join11 } from "path";
|
|
4026
4107
|
var searchCacheDays = () => settings().search_cache_days;
|
|
4027
4108
|
var API2 = "https://clinicaltrials.gov/api/v2/studies";
|
|
@@ -4037,7 +4118,7 @@ var COLUMNS = [
|
|
|
4037
4118
|
// "Terminated — slow accrual": a trial past its completion date is not "ongoing" (ER.md 28.7)
|
|
4038
4119
|
"phase",
|
|
4039
4120
|
"enrollment",
|
|
4040
|
-
// "
|
|
4121
|
+
// "48" actual, "~100" estimated — participant count (28.5)
|
|
4041
4122
|
"start",
|
|
4042
4123
|
"primary_completion",
|
|
4043
4124
|
"primary_outcome",
|
|
@@ -4096,7 +4177,7 @@ function parseRows(json2) {
|
|
|
4096
4177
|
String(id.briefTitle ?? ""),
|
|
4097
4178
|
status,
|
|
4098
4179
|
phases.length ? phases.map((x) => PHASE2[x] ?? x).join(" / ") : null,
|
|
4099
|
-
typeof en.count === "number" ? `${
|
|
4180
|
+
typeof en.count === "number" ? `${String(en.type).toUpperCase() === "ESTIMATED" ? "~" : ""}${en.count}` : null,
|
|
4100
4181
|
st.startDateStruct?.date ?? null,
|
|
4101
4182
|
st.primaryCompletionDateStruct?.date ?? null,
|
|
4102
4183
|
primary,
|
|
@@ -4111,10 +4192,10 @@ var liveFetch = (url) => fetchText(url, { "User-Agent": LIBRARY_UA, Accept: "app
|
|
|
4111
4192
|
async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFetch) {
|
|
4112
4193
|
const q = normalizeQuery(raw);
|
|
4113
4194
|
if ("error" in q) return { ok: false, error: q.error };
|
|
4114
|
-
const
|
|
4115
|
-
if (
|
|
4195
|
+
const file3 = searchFile(libraryDir, searchKey(q));
|
|
4196
|
+
if (existsSync9(file3)) {
|
|
4116
4197
|
try {
|
|
4117
|
-
const r = JSON.parse(readFileSync10(
|
|
4198
|
+
const r = JSON.parse(readFileSync10(file3, "utf8"));
|
|
4118
4199
|
if (daysSince(r.searched) < searchCacheDays()) return { ok: true, result: r, cached: true };
|
|
4119
4200
|
} catch {
|
|
4120
4201
|
}
|
|
@@ -4133,8 +4214,8 @@ async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFe
|
|
|
4133
4214
|
}
|
|
4134
4215
|
const result = { source: "clinicaltrials", query: q, total: parsed.total, columns: COLUMNS, rows: parsed.rows, searched: today() };
|
|
4135
4216
|
try {
|
|
4136
|
-
|
|
4137
|
-
writeAtomic(
|
|
4217
|
+
mkdirSync8(dirname4(file3), { recursive: true });
|
|
4218
|
+
writeAtomic(file3, JSON.stringify(result, null, 2) + "\n");
|
|
4138
4219
|
} catch {
|
|
4139
4220
|
}
|
|
4140
4221
|
return { ok: true, result, cached: false };
|
|
@@ -4146,7 +4227,7 @@ function trialsLogId(q) {
|
|
|
4146
4227
|
|
|
4147
4228
|
// src/pubmedsearch.ts
|
|
4148
4229
|
import { createHash as createHash5 } from "crypto";
|
|
4149
|
-
import { existsSync as
|
|
4230
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync9, readFileSync as readFileSync11 } from "fs";
|
|
4150
4231
|
import { dirname as dirname5 } from "path";
|
|
4151
4232
|
var MAX_RESULTS2 = 100;
|
|
4152
4233
|
var DEFAULT_RESULTS2 = 20;
|
|
@@ -4215,7 +4296,7 @@ function parseEsummary(json2, ids, libraryDir) {
|
|
|
4215
4296
|
const year = /\b(\d{4})\b/.exec(String(s.pubdate ?? ""))?.[1] ?? null;
|
|
4216
4297
|
const first = String(s.sortfirstauthor ?? (Array.isArray(s.authors) && s.authors[0]?.name) ?? "").replace(/\s+[A-Z]{1,3}$/, "").trim() || null;
|
|
4217
4298
|
const pmc2 = (Array.isArray(s.articleids) ? s.articleids : []).find((a) => a.idtype === "pmc")?.value ?? null;
|
|
4218
|
-
const types = Array.isArray(s.pubtype) ? s.pubtype.
|
|
4299
|
+
const types = Array.isArray(s.pubtype) ? reviewTypes(s.pubtype.map(String)).join("; ") : "";
|
|
4219
4300
|
const paths = entryPaths(libraryDir, "pmid", id);
|
|
4220
4301
|
const level = entryExists(paths) ? String(readStatus(paths, emptyStatus(`pmid:${id}`, "pmid", id, libraryDir)).level) : null;
|
|
4221
4302
|
rows.push([id, String(s.title ?? "").replace(/\.$/, "") || null, first, year, s.source ? String(s.source) : null, types || null, pmc2 ? String(pmc2) : null, level]);
|
|
@@ -4226,10 +4307,10 @@ var live = (url) => fetchText(url, { "User-Agent": "fh-library", Accept: "applic
|
|
|
4226
4307
|
async function searchArticles(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = live) {
|
|
4227
4308
|
const q = normalizeQuery2(raw);
|
|
4228
4309
|
if ("error" in q) return { ok: false, error: q.error };
|
|
4229
|
-
const
|
|
4230
|
-
if (
|
|
4310
|
+
const file3 = searchFile(libraryDir, searchKey2(q));
|
|
4311
|
+
if (existsSync10(file3)) {
|
|
4231
4312
|
try {
|
|
4232
|
-
const r = JSON.parse(readFileSync11(
|
|
4313
|
+
const r = JSON.parse(readFileSync11(file3, "utf8"));
|
|
4233
4314
|
if (daysSince(r.searched) < searchCacheDays()) return { ok: true, result: r, cached: true };
|
|
4234
4315
|
} catch {
|
|
4235
4316
|
}
|
|
@@ -4250,8 +4331,8 @@ async function searchArticles(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = live
|
|
|
4250
4331
|
}
|
|
4251
4332
|
const result = { source: "pubmed", query: q, total, columns: COLUMNS2, rows, searched: today() };
|
|
4252
4333
|
try {
|
|
4253
|
-
|
|
4254
|
-
writeAtomic(
|
|
4334
|
+
mkdirSync9(dirname5(file3), { recursive: true });
|
|
4335
|
+
writeAtomic(file3, JSON.stringify(result, null, 2) + "\n");
|
|
4255
4336
|
} catch {
|
|
4256
4337
|
}
|
|
4257
4338
|
return { ok: true, result, cached: false };
|
|
@@ -4307,11 +4388,10 @@ function recordSummary(json2) {
|
|
|
4307
4388
|
start: dated2(st.startDateStruct),
|
|
4308
4389
|
primary_completion: dated2(st.primaryCompletionDateStruct),
|
|
4309
4390
|
completion: dated2(st.completionDateStruct),
|
|
4310
|
-
first_posted: st.studyFirstPostDateStruct?.date ?? null,
|
|
4311
4391
|
last_update: st.lastUpdatePostDateStruct?.date ?? null,
|
|
4312
4392
|
results_first_posted: st.resultsFirstPostDateStruct?.date ?? null,
|
|
4313
4393
|
sponsor: para2(sp.leadSponsor?.name),
|
|
4314
|
-
collaborators:
|
|
4394
|
+
collaborators: collaboratorsLine(list3(sp.collaborators).map((c) => para2(c.name)).filter(Boolean)),
|
|
4315
4395
|
conditions: joined2(list3(ps.conditionsModule?.conditions).map((c) => para2(c))),
|
|
4316
4396
|
interventions: joined2(list3(ai.interventions).map((i) => para2(`${i.type ? `${words(i.type)}: ` : ""}${i.name ?? ""}`))),
|
|
4317
4397
|
primary_outcomes: list3(om.primaryOutcomes).map(outcome).filter(Boolean),
|
|
@@ -4323,22 +4403,68 @@ function recordSummary(json2) {
|
|
|
4323
4403
|
summary: para2(de.briefSummary, 400)
|
|
4324
4404
|
};
|
|
4325
4405
|
}
|
|
4406
|
+
function collaboratorsLine(names) {
|
|
4407
|
+
if (!names.length) return null;
|
|
4408
|
+
return names.slice(0, 3).join("; ") + (names.length > 3 ? ` +${names.length - 3}` : "");
|
|
4409
|
+
}
|
|
4410
|
+
var ORDER = ["Outcome measures", "Adverse events", "Groups", "Participant flow", "Baseline characteristics"];
|
|
4411
|
+
function reorderResults(part) {
|
|
4412
|
+
const chunks = part.split(/^(?=### )/m).map((c) => c.trim()).filter(Boolean);
|
|
4413
|
+
const groups = /* @__PURE__ */ new Map();
|
|
4414
|
+
const sections2 = [];
|
|
4415
|
+
for (const c of chunks) {
|
|
4416
|
+
const name = /^### (.+)$/m.exec(c)?.[1].trim() ?? "";
|
|
4417
|
+
const lines = c.split("\n").slice(1);
|
|
4418
|
+
let a = lines.findIndex((l) => /^- \*\*[^*]+\*\*/.test(l));
|
|
4419
|
+
let b = a < 0 ? -1 : lines.findIndex((l, i) => i > a && (/^\|/.test(l) || /^\*\*/.test(l) || /^#### /.test(l)));
|
|
4420
|
+
if (a >= 0 && b < 0) b = lines.length;
|
|
4421
|
+
if (a < 0) {
|
|
4422
|
+
sections2.push({ name, body: lines.join("\n").trim() });
|
|
4423
|
+
continue;
|
|
4424
|
+
}
|
|
4425
|
+
const kept = [];
|
|
4426
|
+
let cur = [];
|
|
4427
|
+
const flush = () => {
|
|
4428
|
+
if (!cur.length) return;
|
|
4429
|
+
const block = cur.join("\n").trim();
|
|
4430
|
+
const title = /^- \*\*([^*]+)\*\*/.exec(block)[1];
|
|
4431
|
+
if (!groups.has(title)) groups.set(title, block);
|
|
4432
|
+
else if (groups.get(title) !== block) kept.push(block);
|
|
4433
|
+
cur = [];
|
|
4434
|
+
};
|
|
4435
|
+
for (const l of lines.slice(a, b)) {
|
|
4436
|
+
if (/^- \*\*[^*]+\*\*/.test(l)) flush();
|
|
4437
|
+
cur.push(l);
|
|
4438
|
+
}
|
|
4439
|
+
flush();
|
|
4440
|
+
const body = [...lines.slice(0, a), ...kept, ...lines.slice(b)].join("\n").replace(/\n{3,}/g, "\n\n").trim();
|
|
4441
|
+
sections2.push({ name, body });
|
|
4442
|
+
}
|
|
4443
|
+
if (groups.size) sections2.push({ name: "Groups", body: [...groups.values()].join("\n\n") });
|
|
4444
|
+
const rank = (n) => {
|
|
4445
|
+
const i = ORDER.indexOf(n);
|
|
4446
|
+
return i < 0 ? ORDER.length : i;
|
|
4447
|
+
};
|
|
4448
|
+
return sections2.map((x, i) => ({ ...x, i })).sort((x, y) => rank(x.name) - rank(y.name) || x.i - y.i).filter((x) => x.body).map((x) => `### ${x.name}
|
|
4449
|
+
|
|
4450
|
+
${x.body}`).join("\n\n");
|
|
4451
|
+
}
|
|
4326
4452
|
function resultsText(nct, md, maxChars) {
|
|
4327
4453
|
const i = md.search(/^## Results\s*$/m);
|
|
4328
4454
|
if (i < 0) return { nct, has_results: false, text: null, truncated: false, outcomes: [], note: "no posted results in the registry record" };
|
|
4329
|
-
const part = md.slice(i).replace(/^## Results\s*\n/, "").trim();
|
|
4455
|
+
const part = reorderResults(md.slice(i).replace(/^## Results\s*\n/, "").replace(/\n## [\s\S]*$/, "").trim());
|
|
4330
4456
|
const outcomes = [...part.matchAll(/^#### (.+)$/gm)].map((m) => m[1].trim());
|
|
4331
4457
|
const truncated = part.length > maxChars;
|
|
4332
4458
|
return { nct, has_results: true, text: truncated ? part.slice(0, maxChars).trimEnd() + "\n\n[truncated]" : part, truncated, outcomes, note: truncated ? "cut at max_chars; library_read(id, section: <an outcome title>) returns one measure whole" : null };
|
|
4333
4459
|
}
|
|
4334
4460
|
|
|
4335
4461
|
// src/missing.ts
|
|
4336
|
-
import { existsSync as
|
|
4462
|
+
import { existsSync as existsSync11, mkdirSync as mkdirSync10, readFileSync as readFileSync12 } from "fs";
|
|
4337
4463
|
import { join as join12 } from "path";
|
|
4338
4464
|
var missingFile = (p) => join12(p.entryDir, `${p.entry}-missing.md`);
|
|
4339
4465
|
function readMissing(p) {
|
|
4340
4466
|
const f = missingFile(p);
|
|
4341
|
-
if (!
|
|
4467
|
+
if (!existsSync11(f)) return null;
|
|
4342
4468
|
const md = readFileSync12(f, "utf8");
|
|
4343
4469
|
const get = (k) => /^$/.test(k) ? "" : (new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md)?.[1] ?? "").trim();
|
|
4344
4470
|
const checked = get("checked"), reason = get("reason");
|
|
@@ -4350,7 +4476,7 @@ function freshMissing(p, retryDays) {
|
|
|
4350
4476
|
}
|
|
4351
4477
|
function rememberMissing(p, reason) {
|
|
4352
4478
|
const m = { checked: today(), reason };
|
|
4353
|
-
|
|
4479
|
+
mkdirSync10(p.entryDir, { recursive: true });
|
|
4354
4480
|
writeAtomic(missingFile(p), `checked: ${m.checked}
|
|
4355
4481
|
reason: ${m.reason}
|
|
4356
4482
|
`);
|
|
@@ -4382,7 +4508,7 @@ function splitArticles(xml) {
|
|
|
4382
4508
|
}
|
|
4383
4509
|
return out;
|
|
4384
4510
|
}
|
|
4385
|
-
var row2 = (pmid, s, level) => s ? [pmid, s.title, s.first_author, s.year, s.publication_types.
|
|
4511
|
+
var row2 = (pmid, s, level) => s ? [pmid, s.title ? s.title.replace(/\.$/, "") : s.title, s.first_author, s.year, reviewTypes(s.publication_types).join("; ") || null, s.retracted, level] : [pmid, null, null, null, null, null, level];
|
|
4386
4512
|
async function fetchArticles(ids, libraryDir = FH_LIBRARY_DIR, converter = "fh-library", fetchImpl = fetchText) {
|
|
4387
4513
|
const result = { columns: COLUMNS3, rows: [], new: 0, cached: 0, not_found: 0, invalid: 0 };
|
|
4388
4514
|
const order2 = [];
|
|
@@ -4475,7 +4601,7 @@ Tools
|
|
|
4475
4601
|
Hints
|
|
4476
4602
|
- library_read and library_verify store a missing record themselves; the library_read answer then carries added: true. No library_add necessary first.
|
|
4477
4603
|
- citable false \u2014 the paper is retracted (a retraction notice, or PubMed's publication type), or holds nothing to read: no abstract and no full text. comment says which; corrections lists the notices as {type, pmid, source}.
|
|
4478
|
-
- level \u2014 what is stored: "full_text", "abstract" (no full text could be found;
|
|
4604
|
+
- level \u2014 what is stored: "full_text", "abstract" (no full text could be found; the log names each source tried and why it failed) or "title" (not even an abstract; citable false).
|
|
4479
4605
|
- error "pmid not found", "doi not found", "ctgov: not a study record", "dailymed: not an SPL document" \u2014 no service knows the ID. Remembered for a day (FH_LIBRARY_RETRY_AFTER_DAYS): the same call answers from disk with the date it was checked; after that the service is asked again. "invalid id" \u2014 the ID itself is malformed; the same call gives the same answer.
|
|
4480
4606
|
- Calls may run in parallel: library_add, library_read and library_verify. Two calls on the same missing record share one add.
|
|
4481
4607
|
- library_fetch takes any URL that is not a paper, label or trial record; those have IDs, and library_add, library_read and library_verify work on the stored document. A second pass over the same review reads every page from disk; a failed fetch is remembered for a day.
|
|
@@ -4500,14 +4626,14 @@ Reading a document
|
|
|
4500
4626
|
- partial true \u2014 no passage holds every term; missing_terms are the words the document never uses. The same query without them finds the passage if there is one.
|
|
4501
4627
|
- Numbers are often in tables; a matching table row comes with its header row.
|
|
4502
4628
|
- section and query together search inside that section only.
|
|
4503
|
-
- section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name.
|
|
4629
|
+
- section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name, else the statements of the PubMed or Crossref record.
|
|
4504
4630
|
|
|
4505
|
-
Status fields:
|
|
4631
|
+
Status fields: pmid, pmcid, setid, nctid, publications, year, publication_types, retracted, corrections, citable, comment, has_abstract, level, version, retrieved_by, glyphs_unresolved, error. Funding and conflicts of interest: library_read section "Declarations". Answers carry only what the call did not send: no echo of the ID, claim or query; a field that is null or an empty list is left out. library_verify always has verdict, score (null: no judgement) and basis.
|
|
4506
4632
|
|
|
4507
|
-
Where records come from: PubMed for the record and abstract; the full text from
|
|
4633
|
+
Where records come from: PubMed for the record and abstract; the full text from PubMed Central, Europe PMC, publishers' text-mining links, Unpaywall, OpenAlex, CORE, and a proxy URL template from the configuration. Every action is appended as one line to <library>/library.log.`;
|
|
4508
4634
|
var idSchema = z.string().describe('ID as "type:value": a paper "pmid:30955454" (bare digits count as pmid) or "doi:10.5812/asjsm.110753" (a paper PubMed does not index; a DOI PubMed indexes is taken as its PMID), a drug label "dailymed:<set ID>" (the set ID from its DailyMed page URL), or a trial record "nct:05173896" (the eight digits; nct:NCT05173896 and a bare NCT05173896 are accepted).');
|
|
4509
4635
|
var levelWord = (m) => m.level === "abstract" && !m.has_abstract ? "title" : m.level;
|
|
4510
|
-
var json = (s) => ({ content: [{ type: "text", text:
|
|
4636
|
+
var json = (s) => ({ content: [{ type: "text", text: statusAnswer(s) }] });
|
|
4511
4637
|
function noticeTag(m) {
|
|
4512
4638
|
const n = (t) => m.corrections.filter((c) => c.type === t).length;
|
|
4513
4639
|
const parts = [
|
|
@@ -4815,7 +4941,7 @@ function registerTools(server2) {
|
|
|
4815
4941
|
);
|
|
4816
4942
|
server2.tool(
|
|
4817
4943
|
"library_read",
|
|
4818
|
-
`One piece of a stored document's Markdown, never the whole. No section/query: the outline (headings with word counts; full text or abstract only). section: that heading's own text (case-insensitive substring; a repeated heading as "Participants [3/12]"), capped at max_chars, its subsections listed. query: passages holding every term, each a window of at most 1000 chars with its section, best first, at most max_hits; partial: true with missing_terms when no passage holds them all. section and query together: inside that section. A missing paper is added first (added: true).`,
|
|
4944
|
+
`One piece of a stored document's Markdown, never the whole. No section/query: the outline (headings with word counts; full text or abstract only). section: that heading's own text (case-insensitive substring; a repeated heading as "Participants [3/12]"), capped at max_chars, its subsections listed. query: passages holding every term, each a window of at most 1000 chars with its section, best first, at most max_hits (more: how many further passages match); partial: true with missing_terms when no passage holds them all. section and query together: inside that section. A missing paper is added first (added: true).`,
|
|
4819
4945
|
{
|
|
4820
4946
|
id: idSchema,
|
|
4821
4947
|
section: z.string().optional().describe("Heading from the outline (case-insensitive substring)."),
|
|
@@ -4825,7 +4951,7 @@ function registerTools(server2) {
|
|
|
4825
4951
|
},
|
|
4826
4952
|
async ({ id, section: section2, query, max_chars, max_hits }) => {
|
|
4827
4953
|
const parsed = parseId(id);
|
|
4828
|
-
const out = (o2) => ({ content: [{ type: "text", text:
|
|
4954
|
+
const out = (o2) => ({ content: [{ type: "text", text: readAnswer(o2) }] });
|
|
4829
4955
|
if (parsed.error) {
|
|
4830
4956
|
log("library_read", parsed.requested, `error ${parsed.error}`);
|
|
4831
4957
|
return out({ requested: parsed.requested, error: parsed.error });
|
|
@@ -4841,7 +4967,7 @@ function registerTools(server2) {
|
|
|
4841
4967
|
}
|
|
4842
4968
|
added = true;
|
|
4843
4969
|
}
|
|
4844
|
-
const full =
|
|
4970
|
+
const full = existsSync12(paths.fulltextMd);
|
|
4845
4971
|
const md = readFileSync13(full ? paths.fulltextMd : paths.md, "utf8");
|
|
4846
4972
|
const text3 = full ? "full_text" : "abstract";
|
|
4847
4973
|
const base = added ? { requested: parsed.requested, entry: paths.entry, text: text3, added: true } : { requested: parsed.requested, entry: paths.entry, text: text3 };
|
|
@@ -4862,7 +4988,7 @@ function registerTools(server2) {
|
|
|
4862
4988
|
const cap = max_chars ?? 6e3;
|
|
4863
4989
|
let sec = readSection(md, section2, cap);
|
|
4864
4990
|
let via = "";
|
|
4865
|
-
if (!sec && /abstract/i.test(section2) && full &&
|
|
4991
|
+
if (!sec && /abstract/i.test(section2) && full && existsSync12(paths.md)) {
|
|
4866
4992
|
sec = readSection(readFileSync13(paths.md, "utf8"), "abstract", cap);
|
|
4867
4993
|
via = " (from the abstract file)";
|
|
4868
4994
|
}
|
|
@@ -4870,6 +4996,18 @@ function registerTools(server2) {
|
|
|
4870
4996
|
sec = assembleDeclarations(md, cap);
|
|
4871
4997
|
via = " (assembled)";
|
|
4872
4998
|
}
|
|
4999
|
+
if (!sec && /declaration/i.test(section2)) {
|
|
5000
|
+
const st = readStatus(paths, emptyStatus(parsed.requested, parsed.type, parsed.id, FH_LIBRARY_DIR));
|
|
5001
|
+
const body = [st.coi ? `### Conflicts of interest
|
|
5002
|
+
|
|
5003
|
+
${st.coi}` : "", st.funding?.length ? `### Funding
|
|
5004
|
+
|
|
5005
|
+
${st.funding.map((f) => `- ${f}`).join("\n")}` : ""].filter(Boolean).join("\n\n");
|
|
5006
|
+
if (body) {
|
|
5007
|
+
sec = { heading: "Declarations (from the record)", words: body.split(/\s+/).length, text: body, returned_words: body.split(/\s+/).length, truncated: false };
|
|
5008
|
+
via = " (from the record)";
|
|
5009
|
+
}
|
|
5010
|
+
}
|
|
4873
5011
|
if (!sec) {
|
|
4874
5012
|
log("library_read", parsed.requested, `section "${section2}" not found`);
|
|
4875
5013
|
return out({ ...base, error: `no section matching "${section2}"`, outline: headings2() });
|
|
@@ -4911,7 +5049,7 @@ function registerTools(server2) {
|
|
|
4911
5049
|
if (!r.error && r.score !== null) bump("verify", cached);
|
|
4912
5050
|
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
4913
5051
|
log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${cached ? " cached" : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
4914
|
-
return { content: [{ type: "text", text:
|
|
5052
|
+
return { content: [{ type: "text", text: verdictAnswer(r) }] };
|
|
4915
5053
|
}
|
|
4916
5054
|
);
|
|
4917
5055
|
server2.tool(
|
|
@@ -4938,7 +5076,7 @@ function registerTools(server2) {
|
|
|
4938
5076
|
);
|
|
4939
5077
|
server2.tool(
|
|
4940
5078
|
"library_fetch",
|
|
4941
|
-
`Fetch a web page as Markdown: the standard fetch tool, cached and hardened. Use it for any URL instead of a plain fetch or a browser. A page fetched in the last ${settings().refresh_after_days} days is answered from disk in milliseconds; a page a plain fetch cannot get (403, rate limit, bot wall, script-rendered) is fetched through the configured browsers and unlocker in this one call. PDFs come back as Markdown. To verify a link, title_only returns just its title line.`,
|
|
5079
|
+
`Fetch a web page as Markdown: the standard fetch tool, cached and hardened. Use it for any URL instead of a plain fetch or a browser. A page fetched in the last ${settings().refresh_after_days} days is answered from disk in milliseconds; a page a plain fetch cannot get (403, rate limit, bot wall, script-rendered) is fetched through the configured browsers and unlocker in this one call. PDFs come back as Markdown. To verify a link, title_only returns just its title line. A PubMed, ClinicalTrials.gov, DailyMed or doi.org URL is answered from the library's record of it.`,
|
|
4942
5080
|
{
|
|
4943
5081
|
url: z.string().describe("URL to fetch"),
|
|
4944
5082
|
max_length: z.number().int().min(1).max(999999).default(5e3).describe("Maximum number of characters to return."),
|
|
@@ -4949,6 +5087,39 @@ function registerTools(server2) {
|
|
|
4949
5087
|
async ({ url, max_length, start_index, raw, title_only }) => {
|
|
4950
5088
|
const t0 = Date.now();
|
|
4951
5089
|
const id = logId(url), put = (line) => log("library_fetch", id, `${line} ${url}`);
|
|
5090
|
+
let rid = raw ? null : libraryIdForUrl(url);
|
|
5091
|
+
if (rid?.startsWith("doi:") && !entryExists(entryPaths(FH_LIBRARY_DIR, "doi", parseId(rid).id))) {
|
|
5092
|
+
try {
|
|
5093
|
+
const pm = await pmidForDoi(parseId(rid).id);
|
|
5094
|
+
if (pm) rid = `pmid:${pm}`;
|
|
5095
|
+
} catch {
|
|
5096
|
+
}
|
|
5097
|
+
}
|
|
5098
|
+
if (rid) {
|
|
5099
|
+
const pid = parseId(rid);
|
|
5100
|
+
const paths = entryPaths(FH_LIBRARY_DIR, pid.type, pid.id);
|
|
5101
|
+
const had = entryExists(paths);
|
|
5102
|
+
if (!had && pid.type === "pmid") {
|
|
5103
|
+
const b = await fetchArticles([pid.id], FH_LIBRARY_DIR, `${pkg.name} ${pkg.version}`);
|
|
5104
|
+
if (b.rows[0]?.[6] === "not found" || !entryExists(paths)) {
|
|
5105
|
+
put(`library ${rid} error: pmid not found`);
|
|
5106
|
+
return { content: [{ type: "text", text: `Failed to fetch ${url}: pmid not found` }], isError: true };
|
|
5107
|
+
}
|
|
5108
|
+
} else if (!had) {
|
|
5109
|
+
const added = JSON.parse((await addEntry(rid)).content[0].text);
|
|
5110
|
+
if (added.error) {
|
|
5111
|
+
put(`library ${rid} error: ${added.error}`);
|
|
5112
|
+
return { content: [{ type: "text", text: `Failed to fetch ${url}: ${added.error}` }], isError: true };
|
|
5113
|
+
}
|
|
5114
|
+
}
|
|
5115
|
+
const file3 = pid.type === "pmid" ? paths.md : existsSync12(paths.fulltextMd) ? paths.fulltextMd : paths.md;
|
|
5116
|
+
const md = existsSync12(file3) ? readFileSync13(file3, "utf8") : "";
|
|
5117
|
+
const text3 = title_only ? `Contents of ${url}:
|
|
5118
|
+
${(md.split("\n").find((l) => l.trim()) ?? "").trim()}` : fetchAnswer(url, md, "", max_length, start_index);
|
|
5119
|
+
bump("fetch", had);
|
|
5120
|
+
put(`library ${rid} ${had ? "cached" : "added"} chars=${text3.length}${title_only ? " title" : ""}${start_index ? ` start=${start_index}` : ""}`);
|
|
5121
|
+
return { content: [{ type: "text", text: text3 }] };
|
|
5122
|
+
}
|
|
4952
5123
|
const r = await libraryFetch(FH_LIBRARY_DIR, url, { max_length, start_index, raw, title_only }, `${pkg.name} ${pkg.version}`, {
|
|
4953
5124
|
onTry: put,
|
|
4954
5125
|
onAlert: (line) => put(`!! ${line}`)
|
|
@@ -4974,7 +5145,7 @@ function registerTools(server2) {
|
|
|
4974
5145
|
if (r.new) bump("pubmed", false, r.new);
|
|
4975
5146
|
const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
|
|
4976
5147
|
log("library_pubmed", `pmids:${r.rows.length}`, `articles ${secs} cached=${r.cached} new=${r.new}${r.not_found ? ` not_found=${r.not_found}` : ""}${r.invalid ? ` invalid=${r.invalid}` : ""}`);
|
|
4977
|
-
return { content: [{ type: "text", text:
|
|
5148
|
+
return { content: [{ type: "text", text: compact2({ columns: r.columns, rows: r.rows }) }] };
|
|
4978
5149
|
}
|
|
4979
5150
|
);
|
|
4980
5151
|
server2.tool(
|
|
@@ -4997,12 +5168,12 @@ function registerTools(server2) {
|
|
|
4997
5168
|
bump("search", r.cached);
|
|
4998
5169
|
const q = r.result.query;
|
|
4999
5170
|
log("library_search", `pm:${logId2(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""}${q.years ? ` years=${q.years}` : ""} | ${q.query}`);
|
|
5000
|
-
return { content: [{ type: "text", text:
|
|
5171
|
+
return { content: [{ type: "text", text: searchAnswer(r.result) }] };
|
|
5001
5172
|
}
|
|
5002
5173
|
);
|
|
5003
5174
|
server2.tool(
|
|
5004
5175
|
"library_clinicaltrials_search_studies",
|
|
5005
|
-
`Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment, start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${searchCacheDays()} days. One trial in detail: library_clinicaltrials_get_study_record.`,
|
|
5176
|
+
`Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment ("~100" estimated), start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${searchCacheDays()} days. One trial in detail: library_clinicaltrials_get_study_record.`,
|
|
5006
5177
|
{
|
|
5007
5178
|
query: z.string().optional().describe("Free-text search across all fields (title, conditions, interventions, summary, \u2026); AND / OR / NOT allowed."),
|
|
5008
5179
|
intervention: z.string().optional().describe('Intervention or treatment, e.g. "tadalafil", "cognitive behavioral therapy".'),
|
|
@@ -5022,7 +5193,7 @@ function registerTools(server2) {
|
|
|
5022
5193
|
bump("search", r.cached);
|
|
5023
5194
|
const q = r.result.query, parts = [q.intervention && `intervention=${q.intervention}`, q.condition && `condition=${q.condition}`, q.query && `query=${q.query}`].filter(Boolean).join(" ");
|
|
5024
5195
|
log("library_search", `ct:${trialsLogId(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total} status=${q.status}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""} | ${parts}`);
|
|
5025
|
-
return { content: [{ type: "text", text:
|
|
5196
|
+
return { content: [{ type: "text", text: searchAnswer(r.result) }] };
|
|
5026
5197
|
}
|
|
5027
5198
|
);
|
|
5028
5199
|
const nctEntry = async (id) => {
|
|
@@ -5049,7 +5220,7 @@ function registerTools(server2) {
|
|
|
5049
5220
|
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
5050
5221
|
}
|
|
5051
5222
|
const raw = e.paths.fulltextRaw("json");
|
|
5052
|
-
const r =
|
|
5223
|
+
const r = existsSync12(raw) ? recordSummary(readFileSync13(raw, "utf8")) : null;
|
|
5053
5224
|
if (!r) {
|
|
5054
5225
|
log("library_trials", e.requested, "record error: no stored record");
|
|
5055
5226
|
return { content: [{ type: "text", text: compact2({ error: "no stored record" }) }], isError: true };
|
|
@@ -5068,7 +5239,7 @@ function registerTools(server2) {
|
|
|
5068
5239
|
log("library_trials", nct, `results error: ${e.error}`);
|
|
5069
5240
|
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
5070
5241
|
}
|
|
5071
|
-
const md =
|
|
5242
|
+
const md = existsSync12(e.paths.fulltextMd) ? readFileSync13(e.paths.fulltextMd, "utf8") : "";
|
|
5072
5243
|
const r = resultsText(e.requested.replace(/^nct:/, "NCT"), md, max_chars);
|
|
5073
5244
|
log("library_trials", e.requested, `results ${r.has_results ? `${r.outcomes.length} outcomes${r.truncated ? " truncated" : ""}` : "none"}`);
|
|
5074
5245
|
return { content: [{ type: "text", text: leanJson(r) }] };
|