ai4l-library-dev 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +2172 -2162
  2. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -46,22 +46,13 @@ function bool(name, def, env = process.env) {
46
46
  return ["true", "1", "yes", "on"].includes(v) ? true : ["false", "0", "no", "off"].includes(v) ? false : def;
47
47
  }
48
48
 
49
+ // src/env-load.ts
50
+ loadDotenv();
51
+
49
52
  // src/index.ts
50
53
  import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
51
54
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
52
55
 
53
- // src/tools.ts
54
- import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
55
- import { z } from "zod";
56
-
57
- // src/entry.ts
58
- import { copyFileSync, existsSync as existsSync2, mkdirSync, readdirSync, readFileSync as readFileSync2, renameSync, rmSync, writeFileSync } from "fs";
59
- import { join as join2 } from "path";
60
- import { basename, dirname } from "path";
61
-
62
- // src/pubmed.ts
63
- import { XMLParser } from "fast-xml-parser";
64
-
65
56
  // src/contact.ts
66
57
  var DEFAULT = "hello@forever-healthy.org";
67
58
  var contactEmail = (env = process.env) => str("AI4L_LIBRARY_CONTACT_EMAIL", env) || DEFAULT;
@@ -109,197 +100,173 @@ async function fetchText(url, headers = {}) {
109
100
  throw new Error(last);
110
101
  }
111
102
 
112
- // src/pubmed.ts
113
- var CORRECTION_TYPES = {
114
- ErratumIn: "erratum_in",
115
- RetractionIn: "retraction_in",
116
- PartialRetractionIn: "partial_retraction_in",
117
- RetractedandRepublishedIn: "retracted_and_republished_in",
118
- ExpressionOfConcernIn: "expression_of_concern_in",
119
- UpdateIn: "update_in"
120
- };
121
- var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
122
- var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
123
- var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
124
- function efetchUrl(pmid) {
125
- const u = new URL(EFETCH);
126
- u.searchParams.set("db", "pubmed");
127
- u.searchParams.set("id", pmid);
128
- u.searchParams.set("retmode", "xml");
129
- if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
130
- u.searchParams.set("tool", "ai4l-library");
131
- u.searchParams.set("email", contactEmail());
132
- return u.toString();
103
+ // src/checks.ts
104
+ var GREEK = { "\u03B1": "alpha", "\u03B2": "beta", "\xDF": "beta", "\u03B3": "gamma", "\u03B4": "delta", "\u03B5": "epsilon", "\u03B6": "zeta", "\u03B7": "eta", "\u03B8": "theta", "\u03B9": "iota", "\u03BA": "kappa", "\u03BB": "lambda", "\u03BC": "mu", "\xB5": "mu", "\u03BD": "nu", "\u03BE": "xi", "\u03C0": "pi", "\u03C1": "rho", "\u03C3": "sigma", "\u03C4": "tau", "\u03C5": "upsilon", "\u03C6": "phi", "\u03C7": "chi", "\u03C8": "psi", "\u03C9": "omega", "\u0391": "alpha", "\u0392": "beta", "\u0393": "gamma", "\u0394": "delta", "\u03A9": "omega", "\u03A3": "sigma", "\u039A": "kappa", "\u039B": "lambda", "\u03A0": "pi", "\u03A6": "phi", "\u03A8": "psi" };
105
+ var greek = (s) => s.replace(/[\u0370-\u03ff\u00b5\u00df]/g, (c) => GREEK[c] ?? c);
106
+ var norm = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
107
+ var compact = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, "");
108
+ function hasTitle(md, title) {
109
+ const t = norm(title);
110
+ if (!t) return true;
111
+ if (norm(md).includes(t.slice(0, 60))) return true;
112
+ return compact(md).includes(compact(title).slice(0, 50));
133
113
  }
134
- async function fetchPubmedXml(pmid) {
135
- try {
136
- return await fetchText(efetchUrl(pmid));
137
- } catch (e) {
138
- throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
114
+ var WALL = [
115
+ /checking your browser/i,
116
+ /recaptcha/i,
117
+ /verify (that )?you are (a )?human/i,
118
+ /human verification/i,
119
+ /access denied/i,
120
+ /enable javascript/i,
121
+ /javascript is (disabled|required)/i,
122
+ /cloudflare/i,
123
+ /captcha/i,
124
+ /sign in to (continue|access)/i,
125
+ /purchase (access|this article|pdf)/i,
126
+ /subscribe to (read|access)/i,
127
+ /cookie.{0,40}(consent|wall)/i,
128
+ /403 forbidden/i,
129
+ /page not found/i
130
+ ];
131
+ function isWallPage(text3) {
132
+ const short = text3.length < 4e3;
133
+ if (short && WALL.some((re) => re.test(text3))) return true;
134
+ const title = /<title>([^<]*)<\/title>/i.exec(text3)?.[1] ?? "";
135
+ return /checking your browser|recaptcha|just a moment|access denied|attention required|human verification/i.test(title);
136
+ }
137
+ var headings = (md) => md.split("\n").filter((l) => /^#{1,6}\s/.test(l)).map((l) => l.replace(/^#+\s*/, ""));
138
+ function proseWords(md) {
139
+ let n = 0;
140
+ for (const p of md.split(/\n\s*\n/).map((x) => x.trim()).filter((x) => x && !/^#{1,6}\s/.test(x))) {
141
+ const words2 = p.split(/\s+/).length;
142
+ if (words2 < 40 || /^(-|\*|\d+\.)\s/.test(p)) continue;
143
+ const linkChars = (p.match(/\[[^\]]*\]\([^)]*\)/g) ?? []).join("").length;
144
+ if (linkChars / p.length > 0.3) continue;
145
+ n += words2;
146
+ }
147
+ return n;
148
+ }
149
+ var MIN_PROSE = 800;
150
+ function hasMethodsAndResults(md) {
151
+ const hs = headings(md);
152
+ const has = (re) => hs.some((h) => re.test(h));
153
+ if (has(/method|material|procedure|design|experimental|patients? and|participants/i) && has(/result|finding|outcome/i)) return true;
154
+ const words2 = md.split(/\s+/).length;
155
+ if (words2 < 1200 || hs.length < 2) return false;
156
+ const SECTION = /^(abstract|summary|introduction|background|discussion|conclusions?|references|acknowledg|limitations|key points|funding|participants|statistical|supplementary|data availability|author contributions|competing interests|conflicts? of interest|disclosure)/i;
157
+ return hs.filter((h) => SECTION.test(h.trim())).length >= 2;
158
+ }
159
+ var hasDoi = (md, doi) => !!doi && compact(md).includes(compact(doi));
160
+ function checkFulltext(md, title, titleSource, doi, trusted = false) {
161
+ if (isWallPage(md)) return { ok: false, reason: "wall page" };
162
+ const inBody = hasTitle(md, title);
163
+ if (!inBody && !(titleSource && hasTitle(titleSource, title)) && !hasDoi(md, doi)) return { ok: false, reason: "title missing" };
164
+ if (!trusted && proseWords(md) < MIN_PROSE) return { ok: false, reason: "no methods/results" };
165
+ if (!hasMethodsAndResults(md)) return { ok: false, reason: "no methods/results" };
166
+ return { ok: true };
167
+ }
168
+
169
+ // src/html.ts
170
+ import TurndownService from "turndown";
171
+ import { gfm } from "@joplin/turndown-plugin-gfm";
172
+ var td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
173
+ td.use(gfm);
174
+ td.remove(["script", "style", "nav", "noscript", "iframe", "button", "input", "select", "textarea", "header", "footer", "aside"]);
175
+ var FURNITURE = /gdpr|cookie|onetrust|consent|advert|\bad-slot\b/i;
176
+ td.remove(((n) => ["div", "section"].includes(String(n.nodeName).toLowerCase()) && FURNITURE.test(`${n.className ?? ""} ${n.id ?? ""}`)));
177
+ td.addRule("drop-svg", { filter: (n) => n.nodeName.toLowerCase() === "svg", replacement: () => "" });
178
+ td.addRule("drop-images", { filter: ["img"], replacement: () => "" });
179
+ td.addRule("div-headings", {
180
+ filter: (n) => String(n.nodeName).toLowerCase() === "div" && /(^|\s)h[1-6](\s|$)/.test(String(n.className ?? "")),
181
+ replacement: (content, n) => {
182
+ const level = /(^|\s)h([1-6])(\s|$)/.exec(String(n.className))[2];
183
+ return `
184
+
185
+ ${"#".repeat(Number(level))} ${content.trim()}
186
+
187
+ `;
139
188
  }
189
+ });
190
+ function htmlToMarkdown(html) {
191
+ return td.turndown(html).replace(/\n{3,}/g, "\n\n").trim() + "\n";
140
192
  }
193
+
194
+ // src/jats.ts
195
+ import { XMLParser } from "fast-xml-parser";
141
196
  var parser = new XMLParser({
197
+ preserveOrder: true,
142
198
  ignoreAttributes: false,
143
199
  attributeNamePrefix: "@_",
144
200
  parseTagValue: false,
201
+ trimValues: false,
145
202
  processEntities: true,
146
- stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
147
- isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
203
+ htmlEntities: true
148
204
  });
149
- function text(n) {
150
- if (n == null) return "";
151
- if (typeof n === "object") return text(n["#text"]);
152
- return String(n);
153
- }
154
- var plain = (n) => decodeEntities(text(n)).trim();
155
- function inlineText(raw) {
156
- return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
157
- }
158
- function decodeEntities(s) {
159
- return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&");
160
- }
161
- function parsePubmedXml(xml) {
162
- const doc = parser.parse(xml);
163
- const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
164
- if (!article) {
165
- const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
166
- return book ? parseBook(book) : null;
167
- }
168
- const cit = article.MedlineCitation ?? {};
169
- const art = cit.Article ?? {};
170
- const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
171
- const idOf = (t) => {
172
- const hit = ids.find((i) => i["@_IdType"] === t);
173
- return hit ? text(hit).trim() || null : null;
174
- };
175
- const authorList = art.AuthorList?.[0]?.Author ?? [];
176
- const authors = authorList.map((a) => {
177
- if (a.CollectiveName) return plain(a.CollectiveName);
178
- const last = plain(a.LastName);
179
- const ini = plain(a.Initials);
180
- return [last, ini].filter(Boolean).join(" ");
181
- }).filter(Boolean);
182
- const first = authorList[0];
183
- const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
184
- const pub = art.Journal?.JournalIssue?.PubDate ?? {};
185
- let year = text(pub.Year).trim() || null;
186
- if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
187
- const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
188
- label: typeof a === "object" ? a["@_Label"] ?? null : null,
189
- text: inlineText(text(a))
190
- })).filter((a) => a.text);
191
- const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
192
- const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
193
- if (!type) return [];
194
- return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText(text(c.RefSource)) }];
195
- });
196
- const funding = [];
197
- for (const g of art.GrantList?.Grant ?? []) {
198
- const a = plain(g.Agency);
199
- if (a && !funding.includes(a)) funding.push(a);
200
- }
201
- return {
202
- coi: inlineText(text(cit.CoiStatement)) || null,
203
- funding,
204
- pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
205
- pmcid: idOf("pmc"),
206
- doi: idOf("doi"),
207
- title: inlineText(text(art.ArticleTitle)),
208
- authors,
209
- firstAuthor,
210
- journal: plain(art.Journal?.Title) || null,
211
- year,
212
- publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
213
- abstract,
214
- corrections
215
- };
216
- }
217
- function parseBook(b) {
218
- const d = b.BookDocument ?? {};
219
- const book = d.Book ?? {};
220
- const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
221
- const idOf = (t) => {
222
- const hit = ids.find((i) => i["@_IdType"] === t);
223
- return hit ? text(hit).trim() || null : null;
224
- };
225
- const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
226
- const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
227
- const people = pick?.Author ?? [];
228
- const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
229
- const first = people[0];
230
- const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText(text(a)) })).filter((a) => a.text);
231
- const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
232
- return {
233
- pmid: text(d.PMID).trim() || idOf("pubmed") || "",
234
- pmcid: null,
235
- doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
236
- title: inlineText(text(d.ArticleTitle)) || inlineText(text(book.BookTitle)),
237
- authors,
238
- firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
239
- journal: plain(book.Publisher?.PublisherName) || null,
240
- year: text(book.PubDate?.Year).trim() || null,
241
- publicationTypes: [...new Set(types)],
242
- abstract,
243
- corrections: [],
244
- coi: null,
245
- funding: []
246
- };
247
- }
248
-
249
- // src/spl.ts
250
- import { XMLParser as XMLParser2 } from "fast-xml-parser";
251
- var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
252
- var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
253
- async function fetchSpl(setid) {
254
- const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
255
- if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
256
- return xml;
257
- }
258
- var parser2 = new XMLParser2({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
259
205
  var tagOf = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
260
206
  var kids = (n) => n[tagOf(n)] ?? [];
261
207
  var attr = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
262
- var squash = (s) => s.replace(/\s+/g, " ").trim();
263
- var child = (n, tag) => kids(n).find((x) => tagOf(x) === tag);
264
- function findDeep(nodes, tag) {
265
- for (const n of nodes) {
266
- if ("#text" in n) continue;
267
- if (tagOf(n) === tag) return n;
268
- const h = findDeep(kids(n), tag);
269
- if (h) return h;
208
+ var SKIP_INLINE = /* @__PURE__ */ new Set(["inline-graphic", "graphic", "private-char"]);
209
+ function formula(n) {
210
+ const all = [n, ...descendants(n)];
211
+ const tex = all.find((x) => tagOf(x) === "tex-math");
212
+ if (tex) {
213
+ let t = plainText(kids(tex)).trim();
214
+ const m = /\\begin\{document\}([\s\S]*?)\\end\{document\}/.exec(t);
215
+ if (m) t = m[1].trim();
216
+ t = t.replace(/^\$+|\$+$/g, "").trim();
217
+ return t ? `$${t}$` : "";
270
218
  }
271
- return void 0;
219
+ const mml = all.find((x) => /^(mml:)?math$/.test(tagOf(x)));
220
+ return mml ? plainText(kids(mml)).replace(/\s+/g, "") : plainText(kids(n));
272
221
  }
273
- function inline(nodes) {
274
- let out = "";
275
- for (const n of nodes) {
276
- if ("#text" in n) {
277
- out += String(n["#text"]);
278
- continue;
222
+ function descendants(n) {
223
+ if ("#text" in n) return [];
224
+ const out = [];
225
+ for (const k of kids(n)) {
226
+ out.push(k, ...descendants(k));
227
+ }
228
+ return out;
229
+ }
230
+ function inlineText(nodes) {
231
+ let out = "";
232
+ for (const n of nodes) {
233
+ if ("#text" in n) {
234
+ out += String(n["#text"]);
235
+ continue;
279
236
  }
280
237
  const t = tagOf(n);
281
- const inner = () => inline(kids(n));
238
+ if (SKIP_INLINE.has(t)) continue;
239
+ const inner = () => inlineText(kids(n));
282
240
  switch (t) {
283
- case "content": {
284
- const style = attr(n, "styleCode").toLowerCase();
285
- const s = inner();
286
- out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
241
+ case "italic":
242
+ case "i":
243
+ out += `*${inner()}*`;
244
+ break;
245
+ case "bold":
246
+ case "b":
247
+ out += `**${inner()}**`;
287
248
  break;
288
- }
289
249
  case "sup":
290
250
  out += `^${inner()}^`;
291
251
  break;
292
252
  case "sub":
293
253
  out += `~${inner()}~`;
294
254
  break;
295
- case "br":
255
+ case "break":
296
256
  out += " ";
297
257
  break;
298
- case "footnote":
299
- out += ` (${squash(inner())})`;
258
+ case "ext-link":
259
+ case "uri": {
260
+ const href = attr(n, "xlink:href") || attr(n, "href");
261
+ const text3 = inner().trim() || href;
262
+ out += href ? `[${text3}](${href})` : text3;
300
263
  break;
301
- case "footnoteRef":
302
- case "renderMultiMedia":
264
+ }
265
+ case "inline-formula":
266
+ case "tex-math":
267
+ case "mml:math":
268
+ case "alternatives":
269
+ out += formula(n);
303
270
  break;
304
271
  default:
305
272
  out += inner();
@@ -307,33 +274,36 @@ function inline(nodes) {
307
274
  }
308
275
  return out;
309
276
  }
310
- function table(n) {
311
- const row2 = (x) => kids(x).filter((c) => ["td", "th"].includes(tagOf(c))).map((c) => squash(inline(kids(c))).replace(/\|/g, "\\|"));
312
- const collect = (nodes, into) => {
313
- for (const x of nodes) {
314
- if ("#text" in x) continue;
315
- if (tagOf(x) === "tr") into.push(row2(x));
316
- else if (!["thead", "tfoot"].includes(tagOf(x))) collect(kids(x), into);
277
+ function plainText(nodes) {
278
+ let out = "";
279
+ for (const n of nodes) out += "#text" in n ? String(n["#text"]) : plainText(kids(n));
280
+ return out;
281
+ }
282
+ var squash = (s) => s.replace(/\s+/g, " ").trim();
283
+ var cell = (s) => squash(s).replace(/\|/g, "\\|");
284
+ function renderTable(table3) {
285
+ const rows = [];
286
+ const walk = (nodes) => {
287
+ for (const n of nodes) {
288
+ const t = tagOf(n);
289
+ if (t === "tr") rows.push(kids(n).filter((c) => ["td", "th"].includes(tagOf(c))).map((c) => cell(inlineText(kids(c)))));
290
+ else if (!("#text" in n)) walk(kids(n));
317
291
  }
318
292
  };
319
- const head = [], body = [], foot = [];
320
- for (const x of kids(n)) {
321
- const t = tagOf(x);
322
- if (t === "thead") collect(kids(x), head);
323
- else if (t === "tfoot") collect(kids(x), foot);
324
- else if (t === "tr") body.push(row2(x));
325
- else if (!("#text" in x) && t !== "caption") collect(kids(x), body);
326
- }
327
- const rows = [...head, ...body];
293
+ walk(kids(table3));
328
294
  if (!rows.length) return "";
329
295
  const width = Math.max(...rows.map((r) => r.length));
330
- const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
331
- const cap = child(n, "caption");
332
- const title = cap ? squash(inline(kids(cap))) : "";
333
- const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
334
- return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
296
+ const pad = (r) => [...r, ...Array(width - r.length).fill("")];
297
+ const line = (r) => `| ${pad(r).join(" | ")} |`;
298
+ return [line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line)].join("\n");
335
299
  }
336
- function blocks(nodes, depth) {
300
+ function labelCaption(n) {
301
+ const label = n ? squash(plainText(kids(n).filter((c) => tagOf(c) === "label"))) : "";
302
+ const cap = kids(n).find((c) => tagOf(c) === "caption");
303
+ const capText = cap ? squash(inlineText(kids(cap))) : "";
304
+ return [label ? `**${label}.**` : "", capText].filter(Boolean).join(" ");
305
+ }
306
+ function renderBlocks(nodes, depth) {
337
307
  const out = [];
338
308
  for (const n of nodes) {
339
309
  if ("#text" in n) {
@@ -344,328 +314,200 @@ function blocks(nodes, depth) {
344
314
  const t = tagOf(n);
345
315
  const c = kids(n);
346
316
  switch (t) {
347
- case "paragraph": {
348
- const s = squash(inline(c.filter((x) => !["table", "list"].includes(tagOf(x)))));
317
+ case "sec":
318
+ case "app":
319
+ case "ack":
320
+ case "boxed-text":
321
+ case "notes":
322
+ case "glossary": {
323
+ const title = c.find((x) => tagOf(x) === "title");
324
+ if (title) out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash(inlineText(kids(title)))}`);
325
+ else if (t === "ack") out.push(`${"#".repeat(Math.min(depth + 1, 6))} Acknowledgements`);
326
+ out.push(...renderBlocks(c.filter((x) => tagOf(x) !== "title"), depth + 1));
327
+ break;
328
+ }
329
+ case "title":
330
+ out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash(inlineText(c))}`);
331
+ break;
332
+ case "p": {
333
+ const BLOCK = /* @__PURE__ */ new Set(["table-wrap", "fig", "list", "disp-formula", "disp-quote", "boxed-text", "supplementary-material"]);
334
+ const inline2 = c.filter((x) => "#text" in x || !BLOCK.has(tagOf(x)));
335
+ const blocks2 = c.filter((x) => !("#text" in x) && BLOCK.has(tagOf(x)));
336
+ const s = squash(inlineText(inline2));
349
337
  if (s) out.push(s);
350
- out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf(x))), depth));
338
+ out.push(...renderBlocks(blocks2, depth));
351
339
  break;
352
340
  }
353
341
  case "list": {
354
- const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr(n, "listType"));
342
+ const ordered = /order|alpha|roman|number/i.test(attr(n, "list-type"));
355
343
  let i = 0;
356
- for (const it of c.filter((x) => tagOf(x) === "item")) {
344
+ for (const li of c.filter((x) => tagOf(x) === "list-item")) {
357
345
  i++;
358
- const body = blocks(kids(it).filter((x) => tagOf(x) !== "caption"), depth).join(" ");
359
- if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
346
+ const body = renderBlocks(kids(li).filter((x) => tagOf(x) !== "label"), depth).join(" ");
347
+ out.push(`${ordered ? `${i}.` : "-"} ${body}`);
360
348
  }
361
349
  break;
362
350
  }
363
- case "table": {
364
- const md = table(n);
365
- if (md) out.push(md);
351
+ case "table-wrap": {
352
+ const head = labelCaption(n);
353
+ if (head) out.push(head);
354
+ for (const tbl of c.filter((x) => tagOf(x) === "table")) {
355
+ const md = renderTable(tbl);
356
+ if (md) out.push(md);
357
+ }
358
+ const foot = c.find((x) => tagOf(x) === "table-wrap-foot");
359
+ if (foot) out.push(...renderBlocks(kids(foot), depth));
366
360
  break;
367
361
  }
368
- case "renderMultiMedia":
369
- case "observationMedia":
370
- case "subject":
362
+ case "fig": {
363
+ const head = labelCaption(n);
364
+ out.push(head || "*(figure)*");
371
365
  break;
372
- case "section":
373
- out.push(...section(n, depth));
366
+ }
367
+ case "disp-formula": {
368
+ const s = squash(formula(n));
369
+ if (s) out.push(s);
374
370
  break;
375
- case "component":
376
- case "text":
377
- case "excerpt":
378
- case "highlight":
379
- default:
380
- out.push(...blocks(c, depth));
371
+ }
372
+ case "disp-quote":
373
+ out.push(...renderBlocks(c, depth).map((l) => `> ${l}`));
374
+ break;
375
+ case "def-list":
376
+ for (const d of c.filter((x) => tagOf(x) === "def-item")) {
377
+ const term = d["def-item"].find((x) => tagOf(x) === "term");
378
+ const def = d["def-item"].find((x) => tagOf(x) === "def");
379
+ out.push(`**${squash(inlineText(term ? kids(term) : []))}**: ${def ? renderBlocks(kids(def), depth).join(" ") : ""}`);
380
+ }
381
+ break;
382
+ case "ref-list": {
383
+ const title = c.find((x) => tagOf(x) === "title");
384
+ out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${title ? squash(inlineText(kids(title))) : "References"}`);
385
+ for (const r of c.filter((x) => tagOf(x) === "ref")) {
386
+ const rk = kids(r);
387
+ const label = squash(plainText(rk.filter((x) => tagOf(x) === "label")));
388
+ out.push(`- ${label ? `${label} ` : ""}${renderCitation(rk.filter((x) => tagOf(x) !== "label"))}`);
389
+ }
390
+ break;
391
+ }
392
+ case "supplementary-material":
393
+ case "graphic":
394
+ case "media":
395
+ case "fn-group":
396
+ case "label":
381
397
  break;
398
+ default:
399
+ out.push(...renderBlocks(c, depth));
382
400
  }
383
401
  }
384
402
  return out;
385
403
  }
386
- var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
387
- var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
388
- var plain2 = (nodes) => {
389
- let out = "";
390
- for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf(n) === "br" ? " " : plain2(kids(n));
391
- return out;
404
+ function renderCitation(nodes) {
405
+ const alt = nodes.find((x) => tagOf(x) === "citation-alternatives");
406
+ if (alt) return renderCitation(kids(alt));
407
+ const mixed = nodes.find((x) => ["mixed-citation", "citation", "nlm-citation"].includes(tagOf(x)) && !kids(x).some((y) => tagOf(y) === "person-group"));
408
+ if (mixed) return squash(inlineText(kids(mixed)));
409
+ const el = nodes.find((x) => ["element-citation", "mixed-citation", "citation", "nlm-citation"].includes(tagOf(x)));
410
+ if (!el) return squash(inlineText(nodes));
411
+ const f = kids(el);
412
+ const get = (tag) => f.filter((x) => tagOf(x) === tag).map((x) => squash(inlineText(kids(x))));
413
+ const names = f.filter((x) => tagOf(x) === "person-group").flatMap((g) => kids(g)).map((x) => {
414
+ const t = tagOf(x);
415
+ if (t === "etal") return "et al";
416
+ if (t === "collab") return squash(plainText(kids(x)));
417
+ if (t === "name" || t === "string-name") {
418
+ const sur = squash(plainText(kids(x).filter((y) => tagOf(y) === "surname")));
419
+ const giv = squash(plainText(kids(x).filter((y) => tagOf(y) === "given-names")));
420
+ return [sur, giv].filter(Boolean).join(" ");
421
+ }
422
+ return "";
423
+ }).filter(Boolean);
424
+ const vol = get("volume")[0], issue = get("issue")[0], fp = get("fpage")[0], lp = get("lpage")[0];
425
+ const pages = fp ? lp ? `${fp}-${lp}` : fp : get("elocation-id")[0];
426
+ const where = [get("source")[0], get("year")[0] ? `${get("year")[0]};` : "", vol ? `${vol}${issue ? `(${issue})` : ""}` : "", pages ? `:${pages}` : ""].filter(Boolean).join("").replace(/;$/, "");
427
+ const ids = f.filter((x) => tagOf(x) === "pub-id").map((x) => `${attr(x, "pub-id-type")}:${squash(plainText(kids(x)))}`);
428
+ return [names.join(", "), [...get("article-title"), ...get("chapter-title")][0], where, ...ids].filter(Boolean).join(". ").replace(/\.\./g, ".");
429
+ }
430
+ var FN_LABELS = {
431
+ "conflict": "Conflicts of interest",
432
+ "COI-statement": "Conflicts of interest",
433
+ "coi-statement": "Conflicts of interest",
434
+ "financial-disclosure": "Financial disclosure",
435
+ "supported-by": "Funding",
436
+ "funding": "Funding",
437
+ "equal": "Author note",
438
+ "con": "Contributions",
439
+ "contributed-by": "Contributions",
440
+ "other": ""
392
441
  };
393
- function section(n, depth) {
394
- const code = child(n, "code");
395
- const c = code ? attr(code, "code") : "";
396
- if (SKIP_CODES.has(c)) return [];
397
- const title = child(n, "title");
398
- let head = title ? squash(plain2(kids(title))) : "";
399
- if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
400
- if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
442
+ function declarations(meta, back) {
401
443
  const out = [];
402
- if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
403
- const rest = kids(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf(x)));
404
- out.push(...blocks(rest, head ? depth + 1 : depth));
444
+ const fg = meta ? findDeep(kids(meta), "funding-group") : void 0;
445
+ if (fg) {
446
+ const statements = kids(fg).filter((x) => tagOf(x) === "funding-statement").map((x) => squash(inlineText(kids(x)))).filter(Boolean);
447
+ const awards = kids(fg).filter((x) => tagOf(x) === "award-group").map((ag) => {
448
+ const src = kids(ag).filter((x) => tagOf(x) === "funding-source").map((x) => squash(plainText(kids(x)))).filter(Boolean);
449
+ const ids = kids(ag).filter((x) => tagOf(x) === "award-id").map((x) => squash(plainText(kids(x)))).filter(Boolean);
450
+ return [src.join(", "), ids.length ? `(${ids.join(", ")})` : ""].filter(Boolean).join(" ");
451
+ }).filter(Boolean);
452
+ const text3 = statements.length ? statements.join(" ") : awards.join("; ");
453
+ if (text3) out.push(`Funding: ${text3}`);
454
+ }
455
+ const fns = [];
456
+ const notes = meta ? findDeep(kids(meta), "author-notes") : void 0;
457
+ if (notes) fns.push(...kids(notes).filter((x) => tagOf(x) === "fn"));
458
+ if (back) for (const g of kids(back).filter((x) => tagOf(x) === "fn-group")) fns.push(...kids(g).filter((x) => tagOf(x) === "fn"));
459
+ for (const fn of fns) {
460
+ const text3 = renderBlocks(kids(fn).filter((x) => tagOf(x) !== "label"), 1).join(" ").trim();
461
+ if (!text3) continue;
462
+ const label = FN_LABELS[attr(fn, "fn-type")] ?? "";
463
+ out.push(label && !new RegExp(`^[*_]*${label.split(" ")[0]}`, "i").test(text3) ? `${label}: ${text3}` : text3);
464
+ }
405
465
  return out;
406
466
  }
407
- var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
408
- function parseSpl(xml) {
409
- const doc = parser2.parse(xml);
410
- const d = doc.find((x) => tagOf(x) === "document");
411
- if (!d) return null;
412
- const k = kids(d);
413
- const setid = attr(child(d, "setId") ?? {}, "root").toLowerCase();
414
- const version = attr(child(d, "versionNumber") ?? {}, "value") || null;
415
- const effective = isoDate(attr(child(d, "effectiveTime") ?? {}, "value"));
416
- const named = (nodes) => {
417
- for (const n of nodes) {
418
- if ("#text" in n) continue;
419
- if (tagOf(n) === "manufacturedProduct") {
420
- const inner2 = child(n, "manufacturedProduct") ?? n;
421
- if (child(inner2, "name")) return inner2;
422
- }
423
- const h = named(kids(n));
424
- if (h) return h;
425
- }
426
- return void 0;
427
- };
428
- const inner = named(k);
429
- const brand = inner ? squash(inline(kids(child(inner, "name") ?? { name: [] }))) || null : null;
430
- const genericNode = inner ? findDeep(kids(inner), "genericMedicine") : void 0;
431
- const generic = genericNode ? squash(inline(kids(child(genericNode, "name") ?? { name: [] }))) || null : null;
432
- const formNode = inner ? child(inner, "formCode") : void 0;
433
- const form = formNode ? attr(formNode, "displayName").toLowerCase() || null : null;
434
- const org = findDeep(k, "representedOrganization");
435
- const labeler = org ? squash(inline(kids(child(org, "name") ?? { name: [] }))) || null : null;
436
- const body = findDeep(k, "structuredBody");
437
- const sections2 = body ? blocks(kids(body), 2) : [];
438
- const docTitle = child(d, "title");
439
- const lines = [];
440
- let cur = "";
441
- for (const x of docTitle ? kids(docTitle) : []) {
442
- if (!("#text" in x) && tagOf(x) === "br") {
443
- lines.push(cur);
444
- cur = "";
445
- } else cur += "#text" in x ? String(x["#text"]) : inline([x]);
467
+ function find(nodes, path) {
468
+ let cur = nodes;
469
+ let hit;
470
+ for (const tag of path) {
471
+ hit = cur.find((x) => tagOf(x) === tag);
472
+ if (!hit) return void 0;
473
+ cur = kids(hit);
446
474
  }
447
- lines.push(cur);
448
- const highlights = lines.map(squash).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
449
- const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
450
- const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
451
- const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
452
- return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
475
+ return hit;
453
476
  }
454
- function headLineSpl(s) {
455
- return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
477
+ function findDeep(nodes, tag) {
478
+ for (const n of nodes) {
479
+ if ("#text" in n) continue;
480
+ if (tagOf(n) === tag) return n;
481
+ const hit = findDeep(kids(n), tag);
482
+ if (hit) return hit;
483
+ }
484
+ return void 0;
456
485
  }
457
- function renderLabelMd(s) {
458
- const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
459
- if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
460
- parts.push(...s.sections.flatMap((b) => [b, ""]));
461
- return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
462
- }
463
-
464
- // src/ctgov.ts
465
- var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
466
- var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
467
- async function fetchStudy(nctid) {
468
- const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
469
- if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
470
- return json2;
471
- }
472
- var STATUS = {
473
- NOT_YET_RECRUITING: "Not yet recruiting",
474
- RECRUITING: "Recruiting",
475
- ENROLLING_BY_INVITATION: "Enrolling by invitation",
476
- ACTIVE_NOT_RECRUITING: "Active, not recruiting",
477
- SUSPENDED: "Suspended",
478
- TERMINATED: "Terminated",
479
- COMPLETED: "Completed",
480
- WITHDRAWN: "Withdrawn",
481
- UNKNOWN: "Unknown status",
482
- AVAILABLE: "Available",
483
- NO_LONGER_AVAILABLE: "No longer available",
484
- TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
485
- APPROVED_FOR_MARKETING: "Approved for marketing",
486
- WITHHELD: "Withheld"
487
- };
488
- var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
489
- var words = (v) => {
490
- const s = String(v ?? "");
491
- if (!s) return "";
492
- if (STATUS[s]) return STATUS[s];
493
- if (PHASE[s]) return PHASE[s];
494
- const w = s.toLowerCase().replace(/_/g, " ");
495
- return w.charAt(0).toUpperCase() + w.slice(1);
496
- };
497
- var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
498
- var list = (a) => Array.isArray(a) ? a : [];
499
- var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
500
- var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
501
- var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
502
- function parseStudy(json2) {
503
- let d;
504
- try {
505
- d = JSON.parse(json2);
506
- } catch {
507
- return null;
508
- }
509
- const ps = d?.protocolSection;
510
- if (!ps) return null;
511
- const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
512
- const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
513
- const nctid = String(idm.nctId ?? "").toUpperCase();
514
- const out = [];
515
- const sec = (title, lines) => {
516
- const body = lines.filter((l) => !!l && l.trim() !== "");
517
- if (body.length) out.push(`## ${title}`, ...body);
518
- };
519
- const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
520
- sec("Identification", [
521
- idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
522
- idm.acronym ? `Acronym: ${idm.acronym}` : null,
523
- idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
524
- secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
525
- ]);
526
- sec("Status", [
527
- `Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
528
- st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
529
- st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
530
- st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
531
- st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
532
- st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
533
- st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
534
- `Results posted: ${d.hasResults ? "yes" : "no"}`
535
- ]);
536
- const rp = sp.responsibleParty;
537
- sec("Sponsor and collaborators", [
538
- sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
539
- list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
540
- rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
541
- ]);
542
- sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
543
- sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
544
- const di = dm.designInfo ?? {};
545
- const mi = di.maskingInfo ?? {};
546
- sec("Design", [
547
- dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
548
- list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
549
- di.allocation ? `Allocation: ${words(di.allocation)}` : null,
550
- di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
551
- di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
552
- mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
553
- dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
554
- ]);
555
- const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
556
- const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
557
- sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
558
- const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
559
- sec("Outcomes", [
560
- list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
561
- ...list(om.primaryOutcomes).map(outcome),
562
- list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
563
- ...list(om.secondaryOutcomes).map(outcome),
564
- list(om.otherOutcomes).length ? "### Other outcomes" : null,
565
- ...list(om.otherOutcomes).map(outcome)
566
- ]);
567
- sec("Eligibility", [
568
- [el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
569
- el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
570
- ]);
571
- const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
572
- sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
573
- const refs = list(rf.references);
574
- sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
575
- const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
576
- if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
577
- return {
578
- nctid,
579
- title: para(idm.briefTitle) || nctid,
580
- acronym: idm.acronym ?? null,
581
- sponsor: sp.leadSponsor?.name ?? null,
582
- firstPosted: st.studyFirstPostDateStruct?.date ?? null,
583
- lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
584
- status: st.overallStatus ? words(st.overallStatus) : null,
585
- hasResults: !!d.hasResults,
586
- publications,
587
- sections: out
588
- };
589
- }
590
- function results(rs) {
591
- const out = ["## Results"];
592
- const pf = rs.participantFlowModule;
593
- if (pf) {
594
- out.push("### Participant flow");
595
- const groups = list(pf.groups);
596
- out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
597
- if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
598
- for (const p of list(pf.periods)) {
599
- const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
600
- const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
601
- for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
602
- out.push(table2(head, rows));
603
- }
604
- }
605
- const bc = rs.baselineCharacteristicsModule;
606
- if (bc) {
607
- out.push("### Baseline characteristics");
608
- const groups = list(bc.groups);
609
- out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
610
- for (const m of list(bc.measures)) {
611
- const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
612
- const rows = [];
613
- for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
614
- const x = list(cat.measurements).find((v) => v.groupId === g.id);
615
- return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
616
- })]);
617
- out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
618
- }
619
- }
620
- const om = rs.outcomeMeasuresModule;
621
- if (om) {
622
- out.push("### Outcome measures");
623
- for (const m of list(om.outcomeMeasures)) {
624
- const groups = list(m.groups);
625
- const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
626
- out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
627
- if (m.description) out.push(para(m.description));
628
- out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
629
- if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
630
- const rows = [];
631
- for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
632
- const x = list(cat.measurements).find((v) => v.groupId === g.id);
633
- return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
634
- })]);
635
- const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
636
- if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
637
- out.push(table2(["", ...groups.map((g) => g.title)], rows));
638
- for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
639
- }
640
- }
641
- const ae = rs.adverseEventsModule;
642
- if (ae) {
643
- out.push("### Adverse events");
644
- const groups = list(ae.eventGroups);
645
- out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
646
- const summary = [
647
- ["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
648
- ["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
649
- ["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
650
- ].filter((r) => r.slice(1).some(Boolean));
651
- out.push(table2(["", ...groups.map((g) => g.title)], summary));
652
- const events = (title, evs) => {
653
- if (!evs.length) return;
654
- out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
655
- const s = list(e.stats).find((x) => x.groupId === g.id);
656
- return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
657
- })])));
658
- };
659
- events("Serious adverse events", list(ae.seriousEvents));
660
- events("Other adverse events", list(ae.otherEvents));
661
- }
662
- return out;
663
- }
664
- function headLineStudy(s) {
665
- return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
666
- }
667
- function renderStudyMd(s) {
668
- const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
486
+ function parseJats(xml) {
487
+ const doc = parser.parse(xml);
488
+ const article = find(doc, ["pmc-articleset", "article"]) ?? find(doc, ["article"]) ?? findDeep(doc, "article");
489
+ const a = article ? kids(article) : [];
490
+ const front = find(a, ["front"]);
491
+ const meta = front ? find(kids(front), ["article-meta"]) ?? front : void 0;
492
+ const titleNode = meta ? find(kids(meta), ["title-group", "article-title"]) ?? findDeep(kids(meta), "article-title") : void 0;
493
+ const abstractNode = meta ? kids(meta).find((x) => tagOf(x) === "abstract" && !attr(x, "abstract-type")) ?? findDeep(kids(meta), "abstract") : void 0;
494
+ const body = find(a, ["body"]);
495
+ const back = find(a, ["back"]);
496
+ return {
497
+ title: titleNode ? squash(inlineText(kids(titleNode))) : "",
498
+ abstract: abstractNode ? renderBlocks(kids(abstractNode).filter((x) => tagOf(x) !== "title"), 2) : [],
499
+ body: body ? renderBlocks(kids(body), 1) : [],
500
+ declarations: declarations(meta, back),
501
+ back: back ? renderBlocks(kids(back), 1) : [],
502
+ hasBody: !!body && kids(body).length > 0
503
+ };
504
+ }
505
+ function renderFulltextMd(doc, headLine2) {
506
+ const parts = [`# ${doc.title}`, "", headLine2, ""];
507
+ if (doc.abstract.length) parts.push("## Abstract", "", ...doc.abstract.flatMap((b) => [b, ""]));
508
+ parts.push(...doc.body.flatMap((b) => [b, ""]));
509
+ if (doc.declarations.length) parts.push("## Declarations", "", ...doc.declarations.flatMap((b) => [b, ""]));
510
+ parts.push(...doc.back.flatMap((b) => [b, ""]));
669
511
  return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
670
512
  }
671
513
 
@@ -953,1788 +795,1990 @@ var UNRESOLVED_GLYPH = /[\u0001-\u0008\u000b\u000c\u000e-\u001f]/g;
953
795
  var countUnresolved = (text3) => (text3.match(UNRESOLVED_GLYPH) ?? []).length;
954
796
  var looksLikePdf = (bytes) => bytes.subarray(0, 5).toString("latin1") === "%PDF-";
955
797
 
956
- // src/entry.ts
957
- function emptyStatus(requested, type, id, libraryDir) {
798
+ // src/browser.ts
799
+ import { existsSync as existsSync3, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
800
+ import { join as join3 } from "path";
801
+ import { chromium } from "playwright-core";
802
+
803
+ // src/layout.ts
804
+ import { createHash } from "crypto";
805
+ import { existsSync as existsSync2 } from "fs";
806
+ import { join as join2, resolve as resolve2 } from "path";
807
+ var AI4L_LIBRARY_DIR = (() => {
808
+ const env = process.env.AI4L_LIBRARY_DIR;
809
+ if (env && env.trim()) return resolve2(env.trim());
810
+ if (existsSync2("/library")) return "/library";
811
+ return resolve2("library");
812
+ })();
813
+ var doiSlug = (doi) => doi.toLowerCase().replace(/[^a-z0-9.-]+/g, "_");
814
+ function entryDirRel(type, num2) {
815
+ if (type === "doi") {
816
+ const h = createHash("sha1").update(num2).digest("hex");
817
+ return `${type}/${h.slice(0, 2)}/${h.slice(0, 4)}/${num2}`;
818
+ }
819
+ if (!/^\d+$/.test(num2)) return `${type}/${num2.slice(0, 2)}/${num2.slice(0, 4)}/${num2}`;
820
+ const l1 = num2.length > 6 ? num2.slice(0, -6) : "0";
821
+ const l2 = num2.length > 4 ? num2.slice(0, -4) : "0";
822
+ return `${type}/${l1}/${l2}/${num2}`;
823
+ }
824
+ function entryPaths(libraryDir, type, num2) {
825
+ if (type === "doi") num2 = doiSlug(num2);
826
+ const entry = `${type}-${num2}`;
827
+ const entryDir = join2(libraryDir, entryDirRel(type, num2));
958
828
  return {
959
- requested,
960
- type,
961
- id,
962
- entry: null,
963
- entry_dir: null,
964
- library_dir: libraryDir,
965
- pmid: null,
966
- setid: null,
967
- nctid: null,
968
- publications: [],
969
- pmcid: null,
970
- doi: null,
971
- title: null,
972
- first_author: null,
973
- year: null,
974
- publication_types: [],
975
- retracted: false,
976
- corrections: [],
977
- coi: null,
978
- funding: [],
979
- citable: true,
980
- comment: null,
981
- has_abstract: null,
982
- level: null,
983
- version: null,
984
- retrieved_by: null,
985
- retrieved_date: null,
986
- pubmed_checked: null,
987
- attempts: [],
988
- files: [],
989
- glyphs_unresolved: null
829
+ entry,
830
+ entryDir,
831
+ xml: join2(entryDir, `${entry}-abstract.xml`),
832
+ md: join2(entryDir, `${entry}-abstract.md`),
833
+ meta: join2(entryDir, `${entry}-meta.md`),
834
+ fulltextMd: join2(entryDir, `${entry}-fulltext.md`),
835
+ fulltextRaw: (format) => join2(entryDir, `${entry}-fulltext.${format}`)
990
836
  };
991
837
  }
992
- var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
993
- function headLine(r) {
994
- return [
995
- r.authors.join(", ") || null,
996
- r.journal,
997
- r.year,
998
- r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
999
- r.doi ? `DOI ${r.doi}` : null,
1000
- r.pmcid ? `PMCID ${r.pmcid}` : null
1001
- ].filter(Boolean).join(" \xB7 ");
1002
- }
1003
- function renderAbstractMd(r) {
1004
- const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
1005
- return `# ${r.title}
1006
-
1007
- ${headLine(r)}
1008
-
1009
- ## Abstract
1010
838
 
1011
- ${body}
1012
- `;
1013
- }
1014
- function readHeadLine(p) {
1015
- try {
1016
- const lines = readFileSync2(p.md, "utf8").split("\n").filter((l) => l.trim());
1017
- return lines[1] ?? "";
1018
- } catch {
1019
- return "";
839
+ // src/browser.ts
840
+ function browserConfig(env = process.env) {
841
+ const out = { browsers: [], headless: "auto" };
842
+ for (let i = 1; i <= 9; i++) {
843
+ const v = str(`AI4L_LIBRARY_BROWSER_${i}`, env);
844
+ if (v && !out.browsers.includes(v)) out.browsers.push(v);
1020
845
  }
846
+ const h = str("AI4L_LIBRARY_BROWSER_HEADLESS", env).toLowerCase();
847
+ if (h === "true" || h === "false") out.headless = h === "true";
848
+ return out;
1021
849
  }
1022
- function metaFromRecord(r, files, converter) {
1023
- return {
1024
- pmid: r.pmid,
1025
- setid: null,
1026
- nctid: null,
1027
- publications: [],
1028
- pmcid: r.pmcid,
1029
- doi: r.doi,
1030
- title: r.title,
1031
- first_author: r.firstAuthor,
1032
- year: r.year,
1033
- publication_types: r.publicationTypes,
1034
- retracted: isRetracted(r),
1035
- corrections: r.corrections,
1036
- coi: r.coi,
1037
- funding: r.funding,
1038
- has_abstract: r.abstract.length > 0,
1039
- level: "abstract",
1040
- version: null,
1041
- files,
1042
- retrieved_by: "pubmed",
1043
- retrieved_date: today(),
1044
- converted_by: converter,
1045
- pubmed_checked: today(),
1046
- attempts: []
1047
- };
1048
- }
1049
- var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
1050
- var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
1051
- function parseCorrection(v) {
1052
- const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
1053
- if (!m) return null;
1054
- let source;
850
+ var INSTALL_HINT = "npx playwright-core install chromium";
851
+ var isRemote = (endpoint) => /^wss?:\/\//i.test(endpoint);
852
+ var isCdp = (endpoint) => /^cdp:\/\//i.test(endpoint);
853
+ var cdpHttp = (endpoint) => endpoint.replace(/^cdp:\/\//i, "http://");
854
+ var rungKind = (endpoint) => isCdp(endpoint) ? "chrome" : isRemote(endpoint) ? "server" : "local";
855
+ function localChromium() {
1055
856
  try {
1056
- source = JSON.parse(m[3]);
857
+ const p = chromium.executablePath();
858
+ return existsSync3(p) ? p : null;
1057
859
  } catch {
1058
860
  return null;
1059
861
  }
1060
- return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
1061
- }
1062
- function renderMeta(m) {
1063
- const lines = [
1064
- ["pmid", m.pmid],
1065
- ...m.setid ? [["setid", m.setid]] : [],
1066
- ...m.nctid ? [["nctid", m.nctid]] : [],
1067
- ...m.nctid ? [["publications", m.publications.join(", ")]] : [],
1068
- ["pmcid", m.pmcid ?? ""],
1069
- ["doi", m.doi ?? ""],
1070
- ["title", JSON.stringify(m.title)],
1071
- ["first_author", m.first_author ?? ""],
1072
- ["year", m.year ?? ""],
1073
- ["publication_types", m.publication_types.join("; ")],
1074
- ["retracted", m.retracted ? "yes" : "no"],
1075
- ...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
1076
- ["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
1077
- ["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
1078
- // a JSON array: a statement may hold semicolons
1079
- ["has_abstract", m.has_abstract ? "yes" : "no"],
1080
- ["level", m.level],
1081
- ["version", m.version ?? ""],
1082
- ["files", m.files.join(", ")],
1083
- ["retrieved_by", m.retrieved_by],
1084
- ["retrieved_date", m.retrieved_date],
1085
- ["converted_by", m.converted_by],
1086
- ["pubmed_checked", m.pubmed_checked],
1087
- ["attempts", m.attempts.join(", ")]
1088
- ];
1089
- return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
1090
862
  }
1091
- function parseFrontmatter(md) {
1092
- const m = /^---\n([\s\S]*?)\n---/.exec(md);
1093
- const fm = {}, all = {};
1094
- for (const line of (m?.[1] ?? "").split("\n")) {
1095
- const i = line.indexOf(":");
1096
- if (i <= 0) continue;
1097
- const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
1098
- fm[k] = v;
1099
- (all[k] ??= []).push(v);
863
+ function browserLabel(endpoint) {
864
+ if (rungKind(endpoint) === "local") return "local";
865
+ try {
866
+ const h = new URL(isCdp(endpoint) ? cdpHttp(endpoint) : endpoint).hostname;
867
+ return isCdp(endpoint) ? `chrome@${h}` : h;
868
+ } catch {
869
+ return endpoint;
1100
870
  }
1101
- return { fm, all };
1102
871
  }
1103
- var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
1104
- function splitAttempts(v) {
1105
- if (!v) return [];
1106
- const out = [];
1107
- let cur = "", depth = 0;
1108
- for (const ch of v) {
1109
- if (ch === "(") depth++;
1110
- else if (ch === ")") depth = Math.max(0, depth - 1);
1111
- if (ch === "," && depth === 0) {
1112
- out.push(cur);
1113
- cur = "";
1114
- } else cur += ch;
1115
- }
1116
- out.push(cur);
1117
- const merged = [];
1118
- for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
1119
- if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
1120
- else merged.push(piece);
1121
- }
1122
- return merged;
872
+ var hasDisplay = () => !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
873
+ function browserHeadless(endpoint, cfg = browserConfig()) {
874
+ if (isCdp(endpoint)) return false;
875
+ return cfg.headless === "auto" ? isRemote(endpoint) ? false : !hasDisplay() : cfg.headless;
1123
876
  }
1124
- var opt = (v) => v ? v : null;
1125
- var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
1126
- function splitPublicationTypes(v) {
1127
- if (!v) return [];
1128
- if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
1129
- const out = [];
1130
- for (const piece of list2(v)) {
1131
- if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
1132
- else out.push(piece);
1133
- }
1134
- return out;
877
+ function browserRungs(cfg = browserConfig()) {
878
+ return cfg.browsers.map((endpoint) => ({
879
+ endpoint,
880
+ label: browserLabel(endpoint),
881
+ kind: rungKind(endpoint),
882
+ available: rungKind(endpoint) !== "local" || localChromium() !== null,
883
+ headless: browserHeadless(endpoint, cfg)
884
+ }));
1135
885
  }
1136
- function abstractMissing(p) {
886
+ var CHALLENGE = /just a moment|nur einen moment|attention required|verify you are human|human verification|checking your browser|access denied/i;
887
+ var CONNECT_TIMEOUT_MS = 2e4;
888
+ var NAV_TIMEOUT_MS = 2e4;
889
+ var CHALLENGE_WAIT_MS = 8e3;
890
+ var DOWNLOAD_WAIT_MS = 1e4;
891
+ var IDLE_CLOSE_MS = 12e4;
892
+ var ctxs = /* @__PURE__ */ new Map();
893
+ var idleTimer = null;
894
+ function profileDir() {
895
+ const dir = join3(AI4L_LIBRARY_DIR, ".browser");
896
+ mkdirSync(join3(dir, "Default"), { recursive: true });
897
+ const prefs = join3(dir, "Default", "Preferences");
898
+ let p = {};
1137
899
  try {
1138
- return readFileSync2(p.md, "utf8").includes("_No abstract available._");
900
+ p = JSON.parse(readFileSync2(prefs, "utf8"));
1139
901
  } catch {
1140
- return false;
1141
902
  }
903
+ p.plugins = { ...p.plugins ?? {}, always_open_pdf_externally: true };
904
+ p.download = { ...p.download ?? {}, prompt_for_download: false };
905
+ writeFileSync(prefs, JSON.stringify(p));
906
+ return dir;
1142
907
  }
1143
- function correctionsFromXml(p) {
1144
- try {
1145
- return parsePubmedXml(readFileSync2(p.xml, "utf8"))?.corrections ?? [];
1146
- } catch {
1147
- return [];
1148
- }
908
+ var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
909
+ var LOCAL_WINDOW_ARGS = [...LAUNCH_ARGS, "--window-position=4000,4000"];
910
+ var TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "Europe/Berlin";
911
+ var CONTEXT_OPTS = { acceptDownloads: true, viewport: null, locale: "en-US", timezoneId: TIMEZONE };
912
+ function stateFile(endpoint) {
913
+ const dir = join3(AI4L_LIBRARY_DIR, ".browser");
914
+ mkdirSync(dir, { recursive: true });
915
+ return join3(dir, `${browserLabel(endpoint).replace(/[^A-Za-z0-9.-]+/g, "_")}.json`);
1149
916
  }
1150
- function declarationsFromXml(p) {
1151
- try {
1152
- const r = parsePubmedXml(readFileSync2(p.xml, "utf8"));
1153
- return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
1154
- } catch {
1155
- return { coi: null, funding: [] };
917
+ async function context(endpoint, cfg) {
918
+ let p = ctxs.get(endpoint);
919
+ if (!p) {
920
+ p = (async () => {
921
+ const headless = browserHeadless(endpoint, cfg);
922
+ const drop = () => {
923
+ if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
924
+ };
925
+ if (isCdp(endpoint)) {
926
+ const browser = await chromium.connectOverCDP(cdpHttp(endpoint), { timeout: CONNECT_TIMEOUT_MS });
927
+ browser.on("disconnected", drop);
928
+ const ctx2 = browser.contexts()[0] ?? await browser.newContext(CONTEXT_OPTS);
929
+ return { ctx: ctx2, browser };
930
+ }
931
+ if (isRemote(endpoint)) {
932
+ const browser = await chromium.connect(endpoint, {
933
+ timeout: CONNECT_TIMEOUT_MS,
934
+ headers: { "x-playwright-launch-options": JSON.stringify({ headless, args: LAUNCH_ARGS, ignoreDefaultArgs: ["--enable-automation"] }) }
935
+ });
936
+ browser.on("disconnected", drop);
937
+ const state = stateFile(endpoint);
938
+ const ctx2 = await browser.newContext({ ...CONTEXT_OPTS, ...existsSync3(state) ? { storageState: state } : {} });
939
+ return { ctx: ctx2, browser };
940
+ }
941
+ if (!localChromium()) throw new Error(`no local Chromium (${INSTALL_HINT})`);
942
+ const ctx = await chromium.launchPersistentContext(profileDir(), {
943
+ channel: "chromium",
944
+ headless,
945
+ ...CONTEXT_OPTS,
946
+ args: headless ? LAUNCH_ARGS : LOCAL_WINDOW_ARGS,
947
+ ignoreDefaultArgs: ["--enable-automation"]
948
+ });
949
+ ctx.on("close", drop);
950
+ return { ctx, browser: null };
951
+ })();
952
+ ctxs.set(endpoint, p);
953
+ p.catch(() => {
954
+ if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
955
+ });
1156
956
  }
957
+ return (await p).ctx;
1157
958
  }
1158
- function readMeta(p) {
1159
- const { fm, all } = parseFrontmatter(readFileSync2(p.meta, "utf8"));
1160
- let title = fm.title ?? "";
1161
- if (title.startsWith('"')) {
959
+ function touchIdle() {
960
+ if (idleTimer) clearTimeout(idleTimer);
961
+ idleTimer = setTimeout(() => {
962
+ void closeBrowsers();
963
+ }, IDLE_CLOSE_MS);
964
+ idleTimer.unref();
965
+ }
966
+ async function closeBrowsers() {
967
+ const open = [...ctxs.entries()];
968
+ ctxs.clear();
969
+ for (const [endpoint, p] of open) {
1162
970
  try {
1163
- title = JSON.parse(title);
971
+ const { ctx, browser } = await p;
972
+ if (isCdp(endpoint)) {
973
+ await browser?.close().catch(() => void 0);
974
+ continue;
975
+ }
976
+ if (browser) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
977
+ await ctx.close().catch(() => void 0);
978
+ await browser?.close().catch(() => void 0);
1164
979
  } catch {
1165
980
  }
1166
981
  }
1167
- const legacy = fm.pubmed_checked === void 0;
1168
- const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
1169
- let coi = null, funding = [];
1170
- if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
1171
- else {
1172
- if (fm.coi.startsWith('"')) {
1173
- try {
1174
- coi = JSON.parse(fm.coi);
1175
- } catch {
1176
- coi = fm.coi;
982
+ }
983
+ async function browserFetch(url, endpoint, cfg = browserConfig()) {
984
+ const ctx = await context(endpoint, cfg);
985
+ touchIdle();
986
+ const page = await ctx.newPage();
987
+ try {
988
+ let pdfBest = null;
989
+ const pdfReads = [];
990
+ let status, contentType = "";
991
+ page.on("response", (r) => {
992
+ if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) {
993
+ status = r.status();
994
+ contentType = r.headers()["content-type"] ?? "";
1177
995
  }
1178
- } else coi = fm.coi || null;
1179
- funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
996
+ if (r.status() !== 200 || !/application\/pdf/i.test(r.headers()["content-type"] ?? "")) return;
997
+ pdfReads.push(r.body().then((b) => {
998
+ if (b.subarray(0, 5).toString("latin1") === "%PDF-" && (!pdfBest || b.length > pdfBest.length)) pdfBest = b;
999
+ }).catch(() => void 0));
1000
+ });
1001
+ let download2 = null;
1002
+ const dl = page.waitForEvent("download", { timeout: NAV_TIMEOUT_MS + CHALLENGE_WAIT_MS + DOWNLOAD_WAIT_MS }).then((d) => {
1003
+ download2 = d;
1004
+ return d;
1005
+ }).catch(() => null);
1006
+ let navError = null;
1007
+ const nav = page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).then((resp) => {
1008
+ if (status === void 0) {
1009
+ status = resp?.status();
1010
+ contentType = resp?.headers()["content-type"] ?? "";
1011
+ }
1012
+ }).catch((e) => {
1013
+ navError = (e instanceof Error ? e.message : String(e)).split("\n")[0].slice(0, 80);
1014
+ });
1015
+ await Promise.race([nav, dl]);
1016
+ const pause = (ms) => Promise.race([dl, new Promise((r) => setTimeout(r, ms))]);
1017
+ if (download2) {
1018
+ } else if (navError && !/download is starting|net::ERR_ABORTED/i.test(navError)) {
1019
+ await pause(2e3);
1020
+ if (!download2) throw new Error(`navigation: ${navError}`);
1021
+ } else {
1022
+ await pause(2e3);
1023
+ const t0 = Date.now();
1024
+ while (!download2 && Date.now() - t0 < CHALLENGE_WAIT_MS) {
1025
+ const title2 = await page.title().catch(() => "");
1026
+ if (!CHALLENGE.test(title2)) break;
1027
+ await pause(1e3);
1028
+ }
1029
+ if (!download2) await pause(DOWNLOAD_WAIT_MS / 5);
1030
+ }
1031
+ if (download2) {
1032
+ const d = download2;
1033
+ return { bytes: readFileSync2(await d.path()), contentType: "application/pdf", url: d.url(), via: "download" };
1034
+ }
1035
+ const title = await page.title().catch(() => "");
1036
+ if (CHALLENGE.test(title)) throw new Error(`challenge not cleared (${title.slice(0, 40)})`);
1037
+ if (status && status >= 400) throw new Error(`HTTP ${status}`);
1038
+ const docType = await page.evaluate(() => document.contentType).catch(() => "");
1039
+ const inViewer = /application\/pdf/i.test(contentType) || /application\/pdf/i.test(docType);
1040
+ if (inViewer) {
1041
+ const t1 = Date.now();
1042
+ while (Date.now() - t1 < DOWNLOAD_WAIT_MS) {
1043
+ await Promise.all(pdfReads);
1044
+ if (pdfBest && pdfBest.length > 4096) break;
1045
+ if (!pdfReads.length && Date.now() - t1 > 3e3) break;
1046
+ await new Promise((r) => setTimeout(r, 500));
1047
+ }
1048
+ if (pdfBest) return { bytes: pdfBest, contentType: "application/pdf", url: page.url(), via: "response" };
1049
+ }
1050
+ if (inViewer) {
1051
+ const r = await ctx.request.get(page.url(), { timeout: NAV_TIMEOUT_MS });
1052
+ if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
1053
+ return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
1054
+ }
1055
+ const html = await page.content();
1056
+ return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
1057
+ } finally {
1058
+ await page.close().catch(() => void 0);
1059
+ if (isRemote(endpoint)) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
1180
1060
  }
1181
- return {
1182
- pmid: fm.pmid ?? "",
1183
- setid: opt(fm.setid),
1184
- nctid: opt(fm.nctid),
1185
- publications: list2(fm.publications),
1186
- pmcid: opt(fm.pmcid),
1187
- doi: opt(fm.doi),
1188
- title,
1189
- first_author: opt(fm.first_author),
1190
- year: opt(fm.year),
1191
- publication_types: splitPublicationTypes(fm.publication_types),
1192
- retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
1193
- corrections,
1194
- coi,
1195
- funding,
1196
- has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
1197
- level: fm.level === "full_text" ? "full_text" : "abstract",
1198
- version: opt(fm.version),
1199
- files: list2(fm.files),
1200
- retrieved_by: fm.retrieved_by ?? "",
1201
- retrieved_date: fm.retrieved_date ?? "",
1202
- converted_by: fm.converted_by ?? "",
1203
- pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
1204
- attempts: splitAttempts(fm.attempts)
1205
- };
1206
- }
1207
- function writeAtomic(file, data) {
1208
- const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
1209
- writeFileSync(tmp, data);
1210
- renameSync(tmp, file);
1211
1061
  }
1212
- function writeMeta(p, m) {
1213
- writeAtomic(p.meta, renderMeta(m));
1062
+
1063
+ // src/unlock.ts
1064
+ var ENDPOINT = "https://api.brightdata.com/request";
1065
+ var TIMEOUT_MS2 = 6e4;
1066
+ var unlockerConfigured = () => !!process.env.BRIGHTDATA_API_TOKEN?.trim();
1067
+ async function unlock(url, fetchImpl = fetch) {
1068
+ const token = process.env.BRIGHTDATA_API_TOKEN?.trim();
1069
+ if (!token) throw new Error("unlocker not configured");
1070
+ const zone = process.env.BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
1071
+ const res = await fetchImpl(ENDPOINT, {
1072
+ method: "POST",
1073
+ headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
1074
+ body: JSON.stringify({ zone, url, format: "raw" }),
1075
+ signal: AbortSignal.timeout(TIMEOUT_MS2)
1076
+ });
1077
+ if (!res.ok) {
1078
+ const why = res.headers.get("x-brd-err-msg") ?? res.headers.get("x-brd-error");
1079
+ throw new Error(`unlocker HTTP ${res.status}${why ? `: ${why}` : ""}`);
1080
+ }
1081
+ return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
1214
1082
  }
1215
- function copyEntry(from, to) {
1216
- const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
1217
- rmSync(tmp, { recursive: true, force: true });
1218
- mkdirSync(tmp, { recursive: true });
1219
- const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
1220
- for (const n of readdirSync(from.entryDir)) if (join2(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join2(from.entryDir, n), join2(tmp, rename(n)));
1221
- const m = readMeta(from);
1222
- const copy = { ...m, files: m.files.map(rename) };
1223
- writeFileSync(join2(tmp, basename(to.meta)), renderMeta(copy));
1224
- mkdirSync(dirname(to.entryDir), { recursive: true });
1225
- if (existsSync2(to.entryDir)) renameSync(to.entryDir, old);
1226
- renameSync(tmp, to.entryDir);
1227
- rmSync(old, { recursive: true, force: true });
1228
- return copy;
1083
+
1084
+ // src/tiers/tier.ts
1085
+ function oaVersion(v) {
1086
+ if (v === "acceptedVersion") return "accepted_manuscript";
1087
+ if (v === "submittedVersion") return "preprint";
1088
+ return "published";
1229
1089
  }
1230
- function citability(m) {
1231
- const of = (t) => m.corrections.filter((c) => c.type === t);
1232
- const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
1233
- if (m.retracted) {
1234
- const r = m.corrections.filter(isRetractionNotice);
1235
- return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
1090
+ var miss = (tier, reason) => ({ ok: false, tier, reason });
1091
+ var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
1092
+
1093
+ // src/download.ts
1094
+ var hostOf = (url) => {
1095
+ try {
1096
+ return new URL(url).hostname.replace(/^www\./, "");
1097
+ } catch {
1098
+ return url;
1236
1099
  }
1237
- const notes = [
1238
- ...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
1239
- ...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
1240
- ...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
1241
- ];
1242
- return { citable: true, comment: notes.length ? notes.join("; ") : null };
1243
- }
1244
- function statusFromMeta(p, m, base) {
1245
- const full = existsSync2(p.fulltextMd) ? readFileSync2(p.fulltextMd, "utf8") : null;
1246
- const cit = citability(m);
1247
- if (full?.includes(`
1248
- ${CORRECTED_HEADING}
1249
- `) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
1250
- return {
1251
- ...base,
1252
- entry: p.entry,
1253
- entry_dir: p.entryDir,
1254
- pmid: m.pmid || null,
1255
- setid: m.setid,
1256
- nctid: m.nctid,
1257
- publications: m.publications,
1258
- pmcid: m.pmcid,
1259
- doi: m.doi,
1260
- title: m.title || null,
1261
- first_author: m.first_author,
1262
- year: m.year,
1263
- publication_types: m.publication_types,
1264
- retracted: m.retracted,
1265
- corrections: m.corrections,
1266
- coi: m.coi,
1267
- funding: m.funding,
1268
- ...cit,
1269
- // What the entry holds: full_text, abstract, or only the title (the meta's
1270
- // level is "abstract" until a full text arrives; has_abstract says whether
1271
- // there is abstract text).
1272
- has_abstract: m.has_abstract,
1273
- level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
1274
- version: m.version,
1275
- retrieved_by: m.retrieved_by || null,
1276
- retrieved_date: m.retrieved_date || null,
1277
- pubmed_checked: m.pubmed_checked || null,
1278
- attempts: m.attempts,
1279
- files: m.files,
1280
- glyphs_unresolved: full === null ? null : countUnresolved(full)
1281
- };
1100
+ };
1101
+ var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
1102
+ function order(candidates) {
1103
+ const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
1104
+ return [...candidates].sort((a, b) => rank(a) - rank(b));
1282
1105
  }
1283
- function readStatus(p, base) {
1284
- return statusFromMeta(p, readMeta(p), base);
1106
+ async function download(tier, p, candidates, ctx) {
1107
+ const usable = candidates.filter((c) => !NCBI.test(c.url));
1108
+ if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
1109
+ const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
1110
+ if (!ordered.length) return miss(tier, "only copies already tried");
1111
+ const reasons = [];
1112
+ for (const c of ordered) {
1113
+ ctx.tried.add(c.url);
1114
+ let r = await tryOne(p, c, tier);
1115
+ const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
1116
+ if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
1117
+ ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
1118
+ const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
1119
+ for (const b of browserRungs()) {
1120
+ if (r.ok || c.headers || c.service || !b.available || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
1121
+ r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
1122
+ ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
1123
+ }
1124
+ if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
1125
+ r = await tryOne(p, target, tier, "unlocker");
1126
+ ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
1127
+ if (!r.ok && /^unlocker HTTP/.test(bare(r.reason))) ctx.onAlert?.(`brightdata: ${bare(r.reason)}`);
1128
+ }
1129
+ if (r.ok) return r;
1130
+ reasons.push(`${r.reason}: ${hostOf(c.url)}`);
1131
+ }
1132
+ return miss(tier, reasons.join("; "));
1285
1133
  }
1286
- function entryExists(p) {
1287
- return existsSync2(p.meta);
1134
+ var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
1135
+ async function tryOne(p, c, tier, via = "plain") {
1136
+ let got;
1137
+ try {
1138
+ got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
1139
+ } catch (e) {
1140
+ const body = e?.body;
1141
+ if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
1142
+ return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
1143
+ }
1144
+ const { bytes, contentType } = got;
1145
+ if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
1146
+ if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
1147
+ const r = await convertBytes(p, bytes, contentType);
1148
+ if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
1149
+ return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
1288
1150
  }
1289
- function writeEntry(p, r, xml, converter) {
1290
- mkdirSync(p.entryDir, { recursive: true });
1291
- writeAtomic(p.xml, xml);
1292
- writeAtomic(p.md, renderAbstractMd(r));
1293
- const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
1294
- writeMeta(p, meta);
1295
- return meta;
1296
- }
1297
- function refreshRecord(p, m, r, xml) {
1298
- writeAtomic(p.xml, xml);
1299
- syncCorrectedBlock(p);
1300
- const next = {
1301
- ...m,
1302
- pmcid: r.pmcid ?? m.pmcid,
1303
- doi: r.doi ?? m.doi,
1304
- publication_types: r.publicationTypes,
1305
- retracted: isRetracted(r),
1306
- corrections: r.corrections,
1307
- coi: r.coi ?? m.coi,
1308
- funding: r.funding.length ? r.funding : m.funding,
1309
- // the paper's own statements stay when PubMed has none
1310
- pubmed_checked: today()
1311
- };
1312
- writeMeta(p, next);
1313
- return next;
1151
+ var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
1152
+ function serviceMessage(bytes, contentType = "") {
1153
+ const text3 = bytes.toString("utf8").trim();
1154
+ if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
1155
+ try {
1156
+ const j = JSON.parse(text3);
1157
+ const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
1158
+ if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
1159
+ } catch {
1160
+ }
1161
+ const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
1162
+ return (visible || text3).slice(0, 160);
1314
1163
  }
1315
- function daysSince(date, now = /* @__PURE__ */ new Date()) {
1316
- const t = new Date(date).getTime();
1317
- return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
1164
+ function describe(reason, format, md) {
1165
+ const words2 = md.split(/\s+/).filter(Boolean).length;
1166
+ const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
1167
+ const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
1168
+ if (reason === "no methods/results") return `no article body (${seen})`;
1169
+ if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
1170
+ return reason;
1318
1171
  }
1319
- var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
1320
- var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
1321
- function declarationsFromMd(md) {
1322
- let coi = null;
1323
- const funding = [];
1324
- const lines = md.split("\n");
1325
- let section2 = "";
1326
- for (let i = 0; i < lines.length; i++) {
1327
- const l = lines[i].trim();
1328
- const h = /^#{2,6}\s+(.*)$/.exec(l);
1329
- if (h) {
1330
- section2 = h[1];
1331
- const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
1332
- if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
1333
- if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
1334
- continue;
1172
+ async function convertBytes(p, bytes, contentType = "") {
1173
+ let format;
1174
+ let md;
1175
+ if (looksLikePdf(bytes)) {
1176
+ format = "pdf";
1177
+ const stats = {};
1178
+ try {
1179
+ md = await pdfToMarkdown(bytes, p.title, p.headLine, stats);
1180
+ } catch (e) {
1181
+ return { ok: false, reason: `pdf: ${errMsg(e)}` };
1335
1182
  }
1336
- if (section2 !== "Declarations" || !l) continue;
1337
- const f = FUNDING_RE.exec(l);
1338
- const c = COI_RE.exec(l);
1339
- if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
1340
- else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
1341
- }
1342
- return { coi, funding };
1343
- }
1344
- var CORRECTED_HEADING = "## Corrected in PubMed abstract";
1345
- function correctedSentences(r) {
1346
- const out = [];
1347
- for (const a of r.abstract) {
1348
- for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
1349
- const i = s.indexOf("[corrected]");
1350
- if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
1183
+ if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
1184
+ } else {
1185
+ const text3 = bytes.toString("utf8");
1186
+ if (isWallPage(text3)) return { ok: false, reason: "wall page" };
1187
+ if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
1188
+ format = "xml";
1189
+ const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
1190
+ if (!doc.title) doc.title = p.title;
1191
+ if (!doc.hasBody) return { ok: false, reason: "xml without body" };
1192
+ md = renderFulltextMd(doc, p.headLine);
1193
+ } else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
1194
+ format = "html";
1195
+ const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
1196
+ md = `# ${p.title}
1197
+
1198
+ ${p.headLine}
1199
+
1200
+ ${htmlToMarkdown(body)}`;
1201
+ } else {
1202
+ return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
1351
1203
  }
1352
1204
  }
1353
- return out.filter(Boolean);
1205
+ const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
1206
+ const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
1207
+ const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
1208
+ if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
1209
+ return { ok: true, format, md };
1210
+ }
1211
+ var ELSEVIER_TAGS = [
1212
+ [/ja:article\b/g, "article"],
1213
+ [/ja:body\b/g, "body"],
1214
+ [/ja:head\b/g, "front"],
1215
+ [/ce:sections\b/g, "sec-group"],
1216
+ [/ce:section-title\b/g, "title"],
1217
+ [/ce:section\b/g, "sec"],
1218
+ [/ce:simple-para\b/g, "p"],
1219
+ [/ce:para\b/g, "p"],
1220
+ [/ce:abstract-sec\b/g, "sec"],
1221
+ [/ce:abstract\b/g, "abstract"],
1222
+ [/ce:title\b/g, "article-title"],
1223
+ [/ce:italic\b/g, "italic"],
1224
+ [/ce:bold\b/g, "bold"],
1225
+ [/ce:sup\b/g, "sup"],
1226
+ [/ce:inf\b/g, "sub"],
1227
+ [/ce:cross-refs?\b/g, "xref"],
1228
+ [/ce:list-item\b/g, "list-item"],
1229
+ [/ce:list\b/g, "list"],
1230
+ [/ce:label\b/g, "label"],
1231
+ [/ce:caption\b/g, "caption"],
1232
+ [/ce:table\b/g, "table-wrap"],
1233
+ [/ce:figure\b/g, "fig"],
1234
+ [/ce:bib-reference\b/g, "ref"],
1235
+ [/ce:bibliography\b/g, "ref-list"],
1236
+ [/sb:reference\b/g, "mixed-citation"],
1237
+ [/ce:acknowledgment\b/g, "ack"],
1238
+ [/ce:inter-ref\b/g, "ext-link"]
1239
+ ];
1240
+ function elsevierToJats(xml) {
1241
+ let out = xml;
1242
+ for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
1243
+ return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
1354
1244
  }
1355
- function withCorrectedBlock(md, sentences) {
1356
- const base = md.split(`
1357
- ${CORRECTED_HEADING}
1358
- `)[0].trimEnd();
1359
- if (!sentences.length) return base + "\n";
1360
- return `${base}
1361
-
1362
- ${CORRECTED_HEADING}
1363
-
1364
- NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
1365
1245
 
1366
- ${sentences.join("\n\n")}
1367
- `;
1246
+ // src/tiers/core.ts
1247
+ var WINDOW_MS = 6e4;
1248
+ var PER_WINDOW = 10;
1249
+ var MAX_WAIT_MS = 65e3;
1250
+ var TIMEOUT_MS3 = 1e4;
1251
+ var calls = [];
1252
+ async function pace() {
1253
+ const now = Date.now();
1254
+ while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
1255
+ if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
1256
+ calls.push(Date.now());
1368
1257
  }
1369
- function syncCorrectedBlock(p) {
1370
- if (!existsSync2(p.fulltextMd) || !existsSync2(p.xml)) return;
1371
- let r;
1372
- try {
1373
- r = parsePubmedXml(readFileSync2(p.xml, "utf8"));
1374
- } catch {
1375
- return;
1258
+ async function coreFetch(url, key) {
1259
+ const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
1260
+ for (let attempt = 0; attempt < 2; attempt++) {
1261
+ await pace();
1262
+ const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
1263
+ if (res.ok) return await res.text();
1264
+ const body = await res.text().catch(() => "");
1265
+ if (res.status === 429 && attempt === 0) {
1266
+ const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
1267
+ const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
1268
+ await new Promise((r) => setTimeout(r, wait));
1269
+ continue;
1270
+ }
1271
+ throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
1376
1272
  }
1377
- if (!r) return;
1378
- const md = readFileSync2(p.fulltextMd, "utf8");
1379
- const next = withCorrectedBlock(md, correctedSentences(r));
1380
- if (next !== md) writeAtomic(p.fulltextMd, next);
1273
+ throw new Error("HTTP 429");
1381
1274
  }
1382
- function applyFulltext(p, m, hit, converter) {
1383
- const raw = p.fulltextRaw(hit.format);
1384
- writeAtomic(raw, hit.raw);
1385
- writeAtomic(p.fulltextMd, hit.md);
1386
- syncCorrectedBlock(p);
1387
- const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
1388
- const own = declarationsFromMd(hit.md);
1389
- const next = {
1390
- ...m,
1391
- level: "full_text",
1392
- version: hit.version,
1393
- files,
1394
- retrieved_by: hit.tier,
1395
- retrieved_date: today(),
1396
- converted_by: converter,
1397
- coi: m.coi ?? own.coi,
1398
- funding: m.funding.length ? m.funding : own.funding
1275
+ function coreTier(apiKey) {
1276
+ const key = apiKey?.trim();
1277
+ return {
1278
+ name: "core",
1279
+ applies: (p) => p.doi ? true : "no doi",
1280
+ async run(p, ctx) {
1281
+ if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
1282
+ let data;
1283
+ try {
1284
+ const q = encodeURIComponent(`doi:"${p.doi}"`);
1285
+ data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
1286
+ } catch (e) {
1287
+ return miss(this.name, errMsg(e));
1288
+ }
1289
+ const candidates = coreCandidates(data);
1290
+ if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
1291
+ ctx.onTry?.(`located ${candidates.length} copies`);
1292
+ return download(this.name, p, candidates, ctx);
1293
+ }
1399
1294
  };
1400
- writeMeta(p, next);
1401
- return next;
1402
1295
  }
1403
- function recordAttempts(p, m, labels) {
1404
- if (!labels.length) return m;
1405
- const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
1406
- const replaced = new Set(labels.map(tierOf));
1407
- const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
1408
- writeMeta(p, next);
1409
- return next;
1296
+ function coreCandidates(d) {
1297
+ const out = [];
1298
+ const seen = /* @__PURE__ */ new Set();
1299
+ const add = (url, pdf, host) => {
1300
+ if (!url || seen.has(url)) return;
1301
+ seen.add(url);
1302
+ out.push({ url, version: "published", source: "core", host, pdf });
1303
+ };
1304
+ for (const w of d.results ?? []) {
1305
+ add(w.downloadUrl, true, "core");
1306
+ for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
1307
+ for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
1308
+ }
1309
+ return out;
1410
1310
  }
1411
- function writeLabelEntry(p, s, xml, converter) {
1412
- mkdirSync(p.entryDir, { recursive: true });
1413
- const raw = p.fulltextRaw("xml");
1414
- writeAtomic(raw, xml);
1415
- writeAtomic(p.fulltextMd, renderLabelMd(s));
1416
- const meta = {
1417
- pmid: "",
1418
- setid: s.setid,
1419
- nctid: null,
1420
- publications: [],
1421
- pmcid: null,
1422
- doi: null,
1423
- title: s.title,
1424
- first_author: s.labeler,
1425
- year: s.effective?.slice(0, 4) ?? null,
1426
- publication_types: ["Drug Label"],
1427
- retracted: false,
1428
- corrections: [],
1429
- coi: null,
1430
- funding: [],
1431
- has_abstract: false,
1432
- level: "full_text",
1433
- version: s.version ? `spl ${s.version}` : null,
1434
- files: [basename(raw), basename(p.fulltextMd)],
1435
- retrieved_by: "dailymed",
1436
- retrieved_date: today(),
1437
- converted_by: converter,
1438
- pubmed_checked: today(),
1439
- attempts: []
1440
- };
1441
- writeMeta(p, meta);
1442
- return meta;
1443
- }
1444
- function writeStudyEntry(p, s, json2, converter) {
1445
- mkdirSync(p.entryDir, { recursive: true });
1446
- const raw = p.fulltextRaw("json");
1447
- writeAtomic(raw, json2);
1448
- writeAtomic(p.fulltextMd, renderStudyMd(s));
1449
- const meta = {
1450
- pmid: "",
1451
- setid: null,
1452
- nctid: s.nctid,
1453
- publications: s.publications,
1454
- pmcid: null,
1455
- doi: null,
1456
- title: s.title,
1457
- first_author: s.sponsor,
1458
- year: s.firstPosted?.slice(0, 4) ?? null,
1459
- publication_types: ["Trial Registration"],
1460
- retracted: false,
1461
- corrections: [],
1462
- coi: null,
1463
- funding: [],
1464
- has_abstract: false,
1465
- level: "full_text",
1466
- version: s.lastUpdate,
1467
- files: [basename(raw), basename(p.fulltextMd)],
1468
- retrieved_by: "ctgov",
1469
- retrieved_date: today(),
1470
- converted_by: converter,
1471
- pubmed_checked: today(),
1472
- attempts: []
1311
+
1312
+ // src/tiers/europepmc.ts
1313
+ var europepmc = {
1314
+ name: "europepmc",
1315
+ applies: (p) => p.pmid ? true : "no pmid",
1316
+ async run(p, ctx) {
1317
+ let data;
1318
+ try {
1319
+ const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
1320
+ data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
1321
+ } catch (e) {
1322
+ return miss(this.name, errMsg(e));
1323
+ }
1324
+ const rec = data.resultList?.result?.[0];
1325
+ if (!rec) return miss(this.name, "not in europe pmc");
1326
+ const candidates = europepmcCandidates(rec, p.pmcid);
1327
+ if (!candidates.length) return miss(this.name, "no free copy");
1328
+ ctx.onTry?.(`located ${candidates.length} copies`);
1329
+ return download(this.name, p, candidates, ctx);
1330
+ }
1331
+ };
1332
+ function europepmcCandidates(r, pmcid) {
1333
+ const out = [];
1334
+ const seen = /* @__PURE__ */ new Set();
1335
+ const add = (url, pdf, host, version) => {
1336
+ if (!url || seen.has(url)) return;
1337
+ seen.add(url);
1338
+ out.push({ url, version, source: "europepmc", host, pdf });
1473
1339
  };
1474
- writeMeta(p, meta);
1475
- return meta;
1476
- }
1477
- function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
1478
- mkdirSync(p.entryDir, { recursive: true });
1479
- const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
1480
- writeAtomic(raw, json2);
1481
- writeAtomic(p.md, renderAbstractMd(r));
1482
- const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
1483
- writeMeta(p, meta);
1484
- return meta;
1340
+ const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
1341
+ const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
1342
+ if (id && r.inEPMC === "Y") {
1343
+ if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
1344
+ if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
1345
+ }
1346
+ for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
1347
+ if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
1348
+ if (u.documentStyle === "abs") continue;
1349
+ if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
1350
+ add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
1351
+ }
1352
+ return out;
1485
1353
  }
1486
1354
 
1487
- // src/checks.ts
1488
- var GREEK = { "\u03B1": "alpha", "\u03B2": "beta", "\xDF": "beta", "\u03B3": "gamma", "\u03B4": "delta", "\u03B5": "epsilon", "\u03B6": "zeta", "\u03B7": "eta", "\u03B8": "theta", "\u03B9": "iota", "\u03BA": "kappa", "\u03BB": "lambda", "\u03BC": "mu", "\xB5": "mu", "\u03BD": "nu", "\u03BE": "xi", "\u03C0": "pi", "\u03C1": "rho", "\u03C3": "sigma", "\u03C4": "tau", "\u03C5": "upsilon", "\u03C6": "phi", "\u03C7": "chi", "\u03C8": "psi", "\u03C9": "omega", "\u0391": "alpha", "\u0392": "beta", "\u0393": "gamma", "\u0394": "delta", "\u03A9": "omega", "\u03A3": "sigma", "\u039A": "kappa", "\u039B": "lambda", "\u03A0": "pi", "\u03A6": "phi", "\u03A8": "psi" };
1489
- var greek = (s) => s.replace(/[\u0370-\u03ff\u00b5\u00df]/g, (c) => GREEK[c] ?? c);
1490
- var norm = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
1491
- var compact = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, "");
1492
- function hasTitle(md, title) {
1493
- const t = norm(title);
1494
- if (!t) return true;
1495
- if (norm(md).includes(t.slice(0, 60))) return true;
1496
- return compact(md).includes(compact(title).slice(0, 50));
1497
- }
1498
- var WALL = [
1499
- /checking your browser/i,
1500
- /recaptcha/i,
1501
- /verify (that )?you are (a )?human/i,
1502
- /human verification/i,
1503
- /access denied/i,
1504
- /enable javascript/i,
1505
- /javascript is (disabled|required)/i,
1506
- /cloudflare/i,
1507
- /captcha/i,
1508
- /sign in to (continue|access)/i,
1509
- /purchase (access|this article|pdf)/i,
1510
- /subscribe to (read|access)/i,
1511
- /cookie.{0,40}(consent|wall)/i,
1512
- /403 forbidden/i,
1513
- /page not found/i
1514
- ];
1515
- function isWallPage(text3) {
1516
- const short = text3.length < 4e3;
1517
- if (short && WALL.some((re) => re.test(text3))) return true;
1518
- const title = /<title>([^<]*)<\/title>/i.exec(text3)?.[1] ?? "";
1519
- return /checking your browser|recaptcha|just a moment|access denied|attention required|human verification/i.test(title);
1520
- }
1521
- var headings = (md) => md.split("\n").filter((l) => /^#{1,6}\s/.test(l)).map((l) => l.replace(/^#+\s*/, ""));
1522
- function proseWords(md) {
1523
- let n = 0;
1524
- for (const p of md.split(/\n\s*\n/).map((x) => x.trim()).filter((x) => x && !/^#{1,6}\s/.test(x))) {
1525
- const words2 = p.split(/\s+/).length;
1526
- if (words2 < 40 || /^(-|\*|\d+\.)\s/.test(p)) continue;
1527
- const linkChars = (p.match(/\[[^\]]*\]\([^)]*\)/g) ?? []).join("").length;
1528
- if (linkChars / p.length > 0.3) continue;
1529
- n += words2;
1355
+ // src/tiers/inbox.ts
1356
+ import { existsSync as existsSync4, readdirSync, readFileSync as readFileSync3, unlinkSync } from "fs";
1357
+ import { join as join4 } from "path";
1358
+ var INBOX_DIR = join4(AI4L_LIBRARY_DIR, "inbox");
1359
+ var EXT = { ".pdf": "application/pdf", ".xml": "text/xml", ".html": "text/html", ".htm": "text/html" };
1360
+ function findInboxFile(dir, pmid) {
1361
+ if (!existsSync4(dir)) return null;
1362
+ const re = new RegExp(`(^|[^0-9])${pmid}([^0-9]|$)`);
1363
+ for (const name of readdirSync(dir).sort()) {
1364
+ const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
1365
+ if (ext in EXT && re.test(name.slice(0, -ext.length))) return join4(dir, name);
1530
1366
  }
1531
- return n;
1532
- }
1533
- var MIN_PROSE = 800;
1534
- function hasMethodsAndResults(md) {
1535
- const hs = headings(md);
1536
- const has = (re) => hs.some((h) => re.test(h));
1537
- if (has(/method|material|procedure|design|experimental|patients? and|participants/i) && has(/result|finding|outcome/i)) return true;
1538
- const words2 = md.split(/\s+/).length;
1539
- if (words2 < 1200 || hs.length < 2) return false;
1540
- const SECTION = /^(abstract|summary|introduction|background|discussion|conclusions?|references|acknowledg|limitations|key points|funding|participants|statistical|supplementary|data availability|author contributions|competing interests|conflicts? of interest|disclosure)/i;
1541
- return hs.filter((h) => SECTION.test(h.trim())).length >= 2;
1367
+ return null;
1542
1368
  }
1543
- var hasDoi = (md, doi) => !!doi && compact(md).includes(compact(doi));
1544
- function checkFulltext(md, title, titleSource, doi, trusted = false) {
1545
- if (isWallPage(md)) return { ok: false, reason: "wall page" };
1546
- const inBody = hasTitle(md, title);
1547
- if (!inBody && !(titleSource && hasTitle(titleSource, title)) && !hasDoi(md, doi)) return { ok: false, reason: "title missing" };
1548
- if (!trusted && proseWords(md) < MIN_PROSE) return { ok: false, reason: "no methods/results" };
1549
- if (!hasMethodsAndResults(md)) return { ok: false, reason: "no methods/results" };
1550
- return { ok: true };
1369
+ function inboxTier(dir = INBOX_DIR) {
1370
+ return {
1371
+ name: "inbox",
1372
+ always: true,
1373
+ applies: (p) => p.pmid ? true : "no pmid",
1374
+ async run(p, ctx) {
1375
+ const file = findInboxFile(dir, p.pmid);
1376
+ if (!file) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
1377
+ const name = file.slice(file.lastIndexOf("/") + 1);
1378
+ const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
1379
+ const bytes = readFileSync3(file);
1380
+ const r = await convertBytes(p, bytes, EXT[ext]);
1381
+ if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
1382
+ ctx.onTry?.(`${name} ok, removed from the inbox`);
1383
+ unlinkSync(file);
1384
+ return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
1385
+ }
1386
+ };
1551
1387
  }
1388
+ var inbox = inboxTier();
1552
1389
 
1553
- // src/html.ts
1554
- import TurndownService from "turndown";
1555
- import { gfm } from "@joplin/turndown-plugin-gfm";
1556
- var td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
1557
- td.use(gfm);
1558
- td.remove(["script", "style", "nav", "noscript", "iframe", "button", "input", "select", "textarea", "header", "footer", "aside"]);
1559
- var FURNITURE = /gdpr|cookie|onetrust|consent|advert|\bad-slot\b/i;
1560
- td.remove(((n) => ["div", "section"].includes(String(n.nodeName).toLowerCase()) && FURNITURE.test(`${n.className ?? ""} ${n.id ?? ""}`)));
1561
- td.addRule("drop-svg", { filter: (n) => n.nodeName.toLowerCase() === "svg", replacement: () => "" });
1562
- td.addRule("drop-images", { filter: ["img"], replacement: () => "" });
1563
- td.addRule("div-headings", {
1564
- filter: (n) => String(n.nodeName).toLowerCase() === "div" && /(^|\s)h[1-6](\s|$)/.test(String(n.className ?? "")),
1565
- replacement: (content, n) => {
1566
- const level = /(^|\s)h([1-6])(\s|$)/.exec(String(n.className))[2];
1567
- return `
1568
-
1569
- ${"#".repeat(Number(level))} ${content.trim()}
1570
-
1571
- `;
1390
+ // src/tiers/openalex.ts
1391
+ var openalex = {
1392
+ name: "openalex",
1393
+ applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
1394
+ async run(p, ctx) {
1395
+ const mail = contactEmail();
1396
+ const key = p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`;
1397
+ let data;
1398
+ try {
1399
+ data = JSON.parse(await fetchText(`https://api.openalex.org/works/${encodeURIComponent(key)}?mailto=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
1400
+ } catch (e) {
1401
+ return miss(this.name, errMsg(e));
1402
+ }
1403
+ const candidates = openalexCandidates(data);
1404
+ if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
1405
+ ctx.onTry?.(`located ${candidates.length} copies`);
1406
+ return download(this.name, p, candidates, ctx);
1572
1407
  }
1573
- });
1574
- function htmlToMarkdown(html) {
1575
- return td.turndown(html).replace(/\n{3,}/g, "\n\n").trim() + "\n";
1408
+ };
1409
+ function openalexCandidates(d) {
1410
+ const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
1411
+ const seen = /* @__PURE__ */ new Set();
1412
+ const out = [];
1413
+ for (const l of locs) {
1414
+ for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
1415
+ if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
1416
+ seen.add(url);
1417
+ out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
1418
+ }
1419
+ }
1420
+ return out;
1576
1421
  }
1577
1422
 
1578
- // src/jats.ts
1579
- import { XMLParser as XMLParser3 } from "fast-xml-parser";
1580
- var parser3 = new XMLParser3({
1581
- preserveOrder: true,
1582
- ignoreAttributes: false,
1583
- attributeNamePrefix: "@_",
1584
- parseTagValue: false,
1585
- trimValues: false,
1586
- processEntities: true,
1587
- htmlEntities: true
1588
- });
1589
- var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
1590
- var kids2 = (n) => n[tagOf2(n)] ?? [];
1591
- var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
1592
- var SKIP_INLINE = /* @__PURE__ */ new Set(["inline-graphic", "graphic", "private-char"]);
1593
- function formula(n) {
1594
- const all = [n, ...descendants(n)];
1595
- const tex = all.find((x) => tagOf2(x) === "tex-math");
1596
- if (tex) {
1597
- let t = plainText(kids2(tex)).trim();
1598
- const m = /\\begin\{document\}([\s\S]*?)\\end\{document\}/.exec(t);
1599
- if (m) t = m[1].trim();
1600
- t = t.replace(/^\$+|\$+$/g, "").trim();
1601
- return t ? `$${t}$` : "";
1602
- }
1603
- const mml = all.find((x) => /^(mml:)?math$/.test(tagOf2(x)));
1604
- return mml ? plainText(kids2(mml)).replace(/\s+/g, "") : plainText(kids2(n));
1423
+ // src/tiers/pmc.ts
1424
+ function pmcEfetchUrl(pmcid) {
1425
+ const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
1426
+ u.searchParams.set("db", "pmc");
1427
+ u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
1428
+ u.searchParams.set("retmode", "xml");
1429
+ if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
1430
+ u.searchParams.set("tool", "ai4l-library");
1431
+ u.searchParams.set("email", contactEmail());
1432
+ return u.toString();
1605
1433
  }
1606
- function descendants(n) {
1607
- if ("#text" in n) return [];
1608
- const out = [];
1609
- for (const k of kids2(n)) {
1610
- out.push(k, ...descendants(k));
1434
+ var pmc = {
1435
+ name: "pmc",
1436
+ applies: (p) => p.pmcid ? true : "no pmcid",
1437
+ async run(p) {
1438
+ let xml;
1439
+ try {
1440
+ xml = await fetchText(pmcEfetchUrl(p.pmcid));
1441
+ } catch (e) {
1442
+ return miss(this.name, errMsg(e));
1443
+ }
1444
+ const doc = parseJats(xml);
1445
+ if (!doc.hasBody) return miss(this.name, "no body");
1446
+ const md = renderFulltextMd(doc, p.headLine);
1447
+ const check = checkFulltext(md, p.title);
1448
+ if (!check.ok) return miss(this.name, check.reason);
1449
+ return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
1611
1450
  }
1612
- return out;
1613
- }
1614
- function inlineText2(nodes) {
1615
- let out = "";
1616
- for (const n of nodes) {
1617
- if ("#text" in n) {
1618
- out += String(n["#text"]);
1451
+ };
1452
+
1453
+ // src/tiers/tdm.ts
1454
+ var CROSSREF = "https://api.crossref.org/works/";
1455
+ var TOKENED = [
1456
+ { host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
1457
+ { host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
1458
+ ];
1459
+ var hostOf2 = (url) => {
1460
+ try {
1461
+ return new URL(url).hostname;
1462
+ } catch {
1463
+ return "";
1464
+ }
1465
+ };
1466
+ function tdmCandidates(work, tokens = {}, env = process.env) {
1467
+ const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
1468
+ const seen = /* @__PURE__ */ new Set();
1469
+ const candidates = [];
1470
+ const missingTokens = /* @__PURE__ */ new Set();
1471
+ for (const l of links) {
1472
+ const ct = (l["content-type"] ?? "").toLowerCase();
1473
+ if (seen.has(l.URL)) {
1474
+ const prev = candidates.find((c) => c.url === l.URL);
1475
+ if (prev && /pdf/.test(ct)) prev.pdf = true;
1619
1476
  continue;
1620
1477
  }
1621
- const t = tagOf2(n);
1622
- if (SKIP_INLINE.has(t)) continue;
1623
- const inner = () => inlineText2(kids2(n));
1624
- switch (t) {
1625
- case "italic":
1626
- case "i":
1627
- out += `*${inner()}*`;
1628
- break;
1629
- case "bold":
1630
- case "b":
1631
- out += `**${inner()}**`;
1632
- break;
1633
- case "sup":
1634
- out += `^${inner()}^`;
1635
- break;
1636
- case "sub":
1637
- out += `~${inner()}~`;
1638
- break;
1639
- case "break":
1640
- out += " ";
1641
- break;
1642
- case "ext-link":
1643
- case "uri": {
1644
- const href = attr2(n, "xlink:href") || attr2(n, "href");
1645
- const text3 = inner().trim() || href;
1646
- out += href ? `[${text3}](${href})` : text3;
1647
- break;
1478
+ seen.add(l.URL);
1479
+ const rule = TOKENED.find((r) => r.host.test(hostOf2(l.URL)));
1480
+ let headers;
1481
+ if (rule) {
1482
+ const token = tokens[rule.key]?.trim() || env[rule.env]?.trim();
1483
+ if (!token) {
1484
+ missingTokens.add(rule.key);
1485
+ continue;
1648
1486
  }
1649
- case "inline-formula":
1650
- case "tex-math":
1651
- case "mml:math":
1652
- case "alternatives":
1653
- out += formula(n);
1654
- break;
1655
- default:
1656
- out += inner();
1487
+ headers = { [rule.header]: token };
1657
1488
  }
1489
+ if (/text\/plain/.test(ct)) continue;
1490
+ candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf2(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
1658
1491
  }
1659
- return out;
1660
- }
1661
- function plainText(nodes) {
1662
- let out = "";
1663
- for (const n of nodes) out += "#text" in n ? String(n["#text"]) : plainText(kids2(n));
1664
- return out;
1492
+ const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
1493
+ candidates.sort((a, b) => rank(a) - rank(b));
1494
+ return { candidates, missingTokens: [...missingTokens] };
1665
1495
  }
1666
- var squash2 = (s) => s.replace(/\s+/g, " ").trim();
1667
- var cell = (s) => squash2(s).replace(/\|/g, "\\|");
1668
- function renderTable(table3) {
1669
- const rows = [];
1670
- const walk = (nodes) => {
1671
- for (const n of nodes) {
1672
- const t = tagOf2(n);
1673
- if (t === "tr") rows.push(kids2(n).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => cell(inlineText2(kids2(c)))));
1674
- else if (!("#text" in n)) walk(kids2(n));
1496
+ function tdmTier(tokens = {}) {
1497
+ return {
1498
+ name: "tdm",
1499
+ applies: (p) => p.doi ? true : "no doi",
1500
+ async run(p, ctx) {
1501
+ let work;
1502
+ try {
1503
+ work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
1504
+ } catch (e) {
1505
+ return miss(this.name, errMsg(e));
1506
+ }
1507
+ const { candidates, missingTokens } = tdmCandidates(work, tokens);
1508
+ if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
1509
+ ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
1510
+ return download(this.name, p, candidates, ctx);
1675
1511
  }
1676
1512
  };
1677
- walk(kids2(table3));
1678
- if (!rows.length) return "";
1679
- const width = Math.max(...rows.map((r) => r.length));
1680
- const pad = (r) => [...r, ...Array(width - r.length).fill("")];
1681
- const line = (r) => `| ${pad(r).join(" | ")} |`;
1682
- return [line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line)].join("\n");
1683
- }
1684
- function labelCaption(n) {
1685
- const label = n ? squash2(plainText(kids2(n).filter((c) => tagOf2(c) === "label"))) : "";
1686
- const cap = kids2(n).find((c) => tagOf2(c) === "caption");
1687
- const capText = cap ? squash2(inlineText2(kids2(cap))) : "";
1688
- return [label ? `**${label}.**` : "", capText].filter(Boolean).join(" ");
1689
1513
  }
1690
- function renderBlocks(nodes, depth) {
1691
- const out = [];
1692
- for (const n of nodes) {
1693
- if ("#text" in n) {
1694
- const t2 = squash2(String(n["#text"]));
1695
- if (t2) out.push(t2);
1696
- continue;
1514
+ var tdm = tdmTier();
1515
+
1516
+ // src/tiers/unpaywall.ts
1517
+ var unpaywall = {
1518
+ name: "unpaywall",
1519
+ applies: (p) => p.doi ? true : "no doi",
1520
+ async run(p, ctx) {
1521
+ const mail = contactEmail();
1522
+ let data;
1523
+ try {
1524
+ data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
1525
+ } catch (e) {
1526
+ return miss(this.name, errMsg(e));
1697
1527
  }
1698
- const t = tagOf2(n);
1699
- const c = kids2(n);
1700
- switch (t) {
1701
- case "sec":
1702
- case "app":
1703
- case "ack":
1704
- case "boxed-text":
1705
- case "notes":
1706
- case "glossary": {
1707
- const title = c.find((x) => tagOf2(x) === "title");
1708
- if (title) out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash2(inlineText2(kids2(title)))}`);
1709
- else if (t === "ack") out.push(`${"#".repeat(Math.min(depth + 1, 6))} Acknowledgements`);
1710
- out.push(...renderBlocks(c.filter((x) => tagOf2(x) !== "title"), depth + 1));
1711
- break;
1712
- }
1713
- case "title":
1714
- out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash2(inlineText2(c))}`);
1715
- break;
1716
- case "p": {
1717
- const BLOCK = /* @__PURE__ */ new Set(["table-wrap", "fig", "list", "disp-formula", "disp-quote", "boxed-text", "supplementary-material"]);
1718
- const inline2 = c.filter((x) => "#text" in x || !BLOCK.has(tagOf2(x)));
1719
- const blocks2 = c.filter((x) => !("#text" in x) && BLOCK.has(tagOf2(x)));
1720
- const s = squash2(inlineText2(inline2));
1721
- if (s) out.push(s);
1722
- out.push(...renderBlocks(blocks2, depth));
1723
- break;
1724
- }
1725
- case "list": {
1726
- const ordered = /order|alpha|roman|number/i.test(attr2(n, "list-type"));
1727
- let i = 0;
1728
- for (const li of c.filter((x) => tagOf2(x) === "list-item")) {
1729
- i++;
1730
- const body = renderBlocks(kids2(li).filter((x) => tagOf2(x) !== "label"), depth).join(" ");
1731
- out.push(`${ordered ? `${i}.` : "-"} ${body}`);
1732
- }
1733
- break;
1734
- }
1735
- case "table-wrap": {
1736
- const head = labelCaption(n);
1737
- if (head) out.push(head);
1738
- for (const tbl of c.filter((x) => tagOf2(x) === "table")) {
1739
- const md = renderTable(tbl);
1740
- if (md) out.push(md);
1741
- }
1742
- const foot = c.find((x) => tagOf2(x) === "table-wrap-foot");
1743
- if (foot) out.push(...renderBlocks(kids2(foot), depth));
1744
- break;
1745
- }
1746
- case "fig": {
1747
- const head = labelCaption(n);
1748
- out.push(head || "*(figure)*");
1749
- break;
1750
- }
1751
- case "disp-formula": {
1752
- const s = squash2(formula(n));
1753
- if (s) out.push(s);
1754
- break;
1755
- }
1756
- case "disp-quote":
1757
- out.push(...renderBlocks(c, depth).map((l) => `> ${l}`));
1758
- break;
1759
- case "def-list":
1760
- for (const d of c.filter((x) => tagOf2(x) === "def-item")) {
1761
- const term = d["def-item"].find((x) => tagOf2(x) === "term");
1762
- const def = d["def-item"].find((x) => tagOf2(x) === "def");
1763
- out.push(`**${squash2(inlineText2(term ? kids2(term) : []))}**: ${def ? renderBlocks(kids2(def), depth).join(" ") : ""}`);
1764
- }
1765
- break;
1766
- case "ref-list": {
1767
- const title = c.find((x) => tagOf2(x) === "title");
1768
- out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${title ? squash2(inlineText2(kids2(title))) : "References"}`);
1769
- for (const r of c.filter((x) => tagOf2(x) === "ref")) {
1770
- const rk = kids2(r);
1771
- const label = squash2(plainText(rk.filter((x) => tagOf2(x) === "label")));
1772
- out.push(`- ${label ? `${label} ` : ""}${renderCitation(rk.filter((x) => tagOf2(x) !== "label"))}`);
1773
- }
1774
- break;
1775
- }
1776
- case "supplementary-material":
1777
- case "graphic":
1778
- case "media":
1779
- case "fn-group":
1780
- case "label":
1781
- break;
1782
- default:
1783
- out.push(...renderBlocks(c, depth));
1528
+ const candidates = unpaywallCandidates(data);
1529
+ if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
1530
+ ctx.onTry?.(`located ${candidates.length} copies`);
1531
+ return download(this.name, p, candidates, ctx);
1532
+ }
1533
+ };
1534
+ function unpaywallCandidates(d) {
1535
+ const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
1536
+ const seen = /* @__PURE__ */ new Set();
1537
+ const out = [];
1538
+ for (const l of locs) {
1539
+ for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
1540
+ if (!url || seen.has(url)) continue;
1541
+ seen.add(url);
1542
+ out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
1784
1543
  }
1785
1544
  }
1786
1545
  return out;
1787
1546
  }
1788
- function renderCitation(nodes) {
1789
- const alt = nodes.find((x) => tagOf2(x) === "citation-alternatives");
1790
- if (alt) return renderCitation(kids2(alt));
1791
- const mixed = nodes.find((x) => ["mixed-citation", "citation", "nlm-citation"].includes(tagOf2(x)) && !kids2(x).some((y) => tagOf2(y) === "person-group"));
1792
- if (mixed) return squash2(inlineText2(kids2(mixed)));
1793
- const el = nodes.find((x) => ["element-citation", "mixed-citation", "citation", "nlm-citation"].includes(tagOf2(x)));
1794
- if (!el) return squash2(inlineText2(nodes));
1795
- const f = kids2(el);
1796
- const get = (tag) => f.filter((x) => tagOf2(x) === tag).map((x) => squash2(inlineText2(kids2(x))));
1797
- const names = f.filter((x) => tagOf2(x) === "person-group").flatMap((g) => kids2(g)).map((x) => {
1798
- const t = tagOf2(x);
1799
- if (t === "etal") return "et al";
1800
- if (t === "collab") return squash2(plainText(kids2(x)));
1801
- if (t === "name" || t === "string-name") {
1802
- const sur = squash2(plainText(kids2(x).filter((y) => tagOf2(y) === "surname")));
1803
- const giv = squash2(plainText(kids2(x).filter((y) => tagOf2(y) === "given-names")));
1804
- return [sur, giv].filter(Boolean).join(" ");
1805
- }
1806
- return "";
1807
- }).filter(Boolean);
1808
- const vol = get("volume")[0], issue = get("issue")[0], fp = get("fpage")[0], lp = get("lpage")[0];
1809
- const pages = fp ? lp ? `${fp}-${lp}` : fp : get("elocation-id")[0];
1810
- const where = [get("source")[0], get("year")[0] ? `${get("year")[0]};` : "", vol ? `${vol}${issue ? `(${issue})` : ""}` : "", pages ? `:${pages}` : ""].filter(Boolean).join("").replace(/;$/, "");
1811
- const ids = f.filter((x) => tagOf2(x) === "pub-id").map((x) => `${attr2(x, "pub-id-type")}:${squash2(plainText(kids2(x)))}`);
1812
- return [names.join(", "), [...get("article-title"), ...get("chapter-title")][0], where, ...ids].filter(Boolean).join(". ").replace(/\.\./g, ".");
1813
- }
1814
- var FN_LABELS = {
1815
- "conflict": "Conflicts of interest",
1816
- "COI-statement": "Conflicts of interest",
1817
- "coi-statement": "Conflicts of interest",
1818
- "financial-disclosure": "Financial disclosure",
1819
- "supported-by": "Funding",
1820
- "funding": "Funding",
1821
- "equal": "Author note",
1822
- "con": "Contributions",
1823
- "contributed-by": "Contributions",
1824
- "other": ""
1825
- };
1826
- function declarations(meta, back) {
1827
- const out = [];
1828
- const fg = meta ? findDeep2(kids2(meta), "funding-group") : void 0;
1829
- if (fg) {
1830
- const statements = kids2(fg).filter((x) => tagOf2(x) === "funding-statement").map((x) => squash2(inlineText2(kids2(x)))).filter(Boolean);
1831
- const awards = kids2(fg).filter((x) => tagOf2(x) === "award-group").map((ag) => {
1832
- const src = kids2(ag).filter((x) => tagOf2(x) === "funding-source").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
1833
- const ids = kids2(ag).filter((x) => tagOf2(x) === "award-id").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
1834
- return [src.join(", "), ids.length ? `(${ids.join(", ")})` : ""].filter(Boolean).join(" ");
1835
- }).filter(Boolean);
1836
- const text3 = statements.length ? statements.join(" ") : awards.join("; ");
1837
- if (text3) out.push(`Funding: ${text3}`);
1838
- }
1839
- const fns = [];
1840
- const notes = meta ? findDeep2(kids2(meta), "author-notes") : void 0;
1841
- if (notes) fns.push(...kids2(notes).filter((x) => tagOf2(x) === "fn"));
1842
- if (back) for (const g of kids2(back).filter((x) => tagOf2(x) === "fn-group")) fns.push(...kids2(g).filter((x) => tagOf2(x) === "fn"));
1843
- for (const fn of fns) {
1844
- const text3 = renderBlocks(kids2(fn).filter((x) => tagOf2(x) !== "label"), 1).join(" ").trim();
1845
- if (!text3) continue;
1846
- const label = FN_LABELS[attr2(fn, "fn-type")] ?? "";
1847
- out.push(label && !new RegExp(`^[*_]*${label.split(" ")[0]}`, "i").test(text3) ? `${label}: ${text3}` : text3);
1848
- }
1849
- return out;
1850
- }
1851
- function find(nodes, path) {
1852
- let cur = nodes;
1853
- let hit;
1854
- for (const tag of path) {
1855
- hit = cur.find((x) => tagOf2(x) === tag);
1856
- if (!hit) return void 0;
1857
- cur = kids2(hit);
1858
- }
1859
- return hit;
1860
- }
1861
- function findDeep2(nodes, tag) {
1862
- for (const n of nodes) {
1863
- if ("#text" in n) continue;
1864
- if (tagOf2(n) === tag) return n;
1865
- const hit = findDeep2(kids2(n), tag);
1866
- if (hit) return hit;
1867
- }
1868
- return void 0;
1869
- }
1870
- function parseJats(xml) {
1871
- const doc = parser3.parse(xml);
1872
- const article = find(doc, ["pmc-articleset", "article"]) ?? find(doc, ["article"]) ?? findDeep2(doc, "article");
1873
- const a = article ? kids2(article) : [];
1874
- const front = find(a, ["front"]);
1875
- const meta = front ? find(kids2(front), ["article-meta"]) ?? front : void 0;
1876
- const titleNode = meta ? find(kids2(meta), ["title-group", "article-title"]) ?? findDeep2(kids2(meta), "article-title") : void 0;
1877
- const abstractNode = meta ? kids2(meta).find((x) => tagOf2(x) === "abstract" && !attr2(x, "abstract-type")) ?? findDeep2(kids2(meta), "abstract") : void 0;
1878
- const body = find(a, ["body"]);
1879
- const back = find(a, ["back"]);
1547
+
1548
+ // src/tiers/index.ts
1549
+ var BUILTIN = [
1550
+ { name: "inbox", make: () => inbox },
1551
+ { name: "pmc", make: () => pmc },
1552
+ { name: "europepmc", make: () => europepmc },
1553
+ { name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
1554
+ { name: "unpaywall", make: () => unpaywall },
1555
+ { name: "openalex", make: () => openalex },
1556
+ { name: "core", make: (o) => coreTier(o.api_key) }
1557
+ ];
1558
+
1559
+ // src/tiers/url.ts
1560
+ var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
1561
+ function urlTier(name, template, headers) {
1562
+ const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
1880
1563
  return {
1881
- title: titleNode ? squash2(inlineText2(kids2(titleNode))) : "",
1882
- abstract: abstractNode ? renderBlocks(kids2(abstractNode).filter((x) => tagOf2(x) !== "title"), 2) : [],
1883
- body: body ? renderBlocks(kids2(body), 1) : [],
1884
- declarations: declarations(meta, back),
1885
- back: back ? renderBlocks(kids2(back), 1) : [],
1886
- hasBody: !!body && kids2(body).length > 0
1564
+ name,
1565
+ applies(p) {
1566
+ const missing = used.filter((k) => !p[k]);
1567
+ return missing.length ? `no ${missing.join("/")}` : true;
1568
+ },
1569
+ async run(p, ctx) {
1570
+ if (!used.length) return miss(name, "template has no placeholder");
1571
+ const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
1572
+ return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
1573
+ }
1887
1574
  };
1888
1575
  }
1889
- function renderFulltextMd(doc, headLine2) {
1890
- const parts = [`# ${doc.title}`, "", headLine2, ""];
1891
- if (doc.abstract.length) parts.push("## Abstract", "", ...doc.abstract.flatMap((b) => [b, ""]));
1892
- parts.push(...doc.body.flatMap((b) => [b, ""]));
1893
- if (doc.declarations.length) parts.push("## Declarations", "", ...doc.declarations.flatMap((b) => [b, ""]));
1894
- parts.push(...doc.back.flatMap((b) => [b, ""]));
1895
- return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
1896
- }
1897
-
1898
- // src/browser.ts
1899
- import { existsSync as existsSync4, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
1900
- import { join as join4 } from "path";
1901
- import { chromium } from "playwright-core";
1902
1576
 
1903
- // src/layout.ts
1904
- import { createHash } from "crypto";
1905
- import { existsSync as existsSync3 } from "fs";
1906
- import { join as join3, resolve as resolve2 } from "path";
1907
- var AI4L_LIBRARY_DIR = (() => {
1908
- const env = process.env.AI4L_LIBRARY_DIR;
1909
- if (env && env.trim()) return resolve2(env.trim());
1910
- if (existsSync3("/library")) return "/library";
1911
- return resolve2("library");
1912
- })();
1913
- var doiSlug = (doi) => doi.toLowerCase().replace(/[^a-z0-9.-]+/g, "_");
1914
- function entryDirRel(type, num2) {
1915
- if (type === "doi") {
1916
- const h = createHash("sha1").update(num2).digest("hex");
1917
- return `${type}/${h.slice(0, 2)}/${h.slice(0, 4)}/${num2}`;
1918
- }
1919
- if (!/^\d+$/.test(num2)) return `${type}/${num2.slice(0, 2)}/${num2.slice(0, 4)}/${num2}`;
1920
- const l1 = num2.length > 6 ? num2.slice(0, -6) : "0";
1921
- const l2 = num2.length > 4 ? num2.slice(0, -4) : "0";
1922
- return `${type}/${l1}/${l2}/${num2}`;
1923
- }
1924
- function entryPaths(libraryDir, type, num2) {
1925
- if (type === "doi") num2 = doiSlug(num2);
1926
- const entry = `${type}-${num2}`;
1927
- const entryDir = join3(libraryDir, entryDirRel(type, num2));
1577
+ // src/config.ts
1578
+ var VARIABLES = [
1579
+ "AI4L_LIBRARY_DIR",
1580
+ "AI4L_LIBRARY_ENV",
1581
+ "AI4L_LIBRARY_TIERS",
1582
+ "AI4L_LIBRARY_CONTACT_EMAIL",
1583
+ "AI4L_LIBRARY_ELSEVIER_API_KEY",
1584
+ "AI4L_LIBRARY_WILEY_TDM_TOKEN",
1585
+ "AI4L_LIBRARY_CORE_API_KEY",
1586
+ "AI4L_LIBRARY_PROXY_URL",
1587
+ "AI4L_LIBRARY_PROXY_HEADERS",
1588
+ "AI4L_LIBRARY_BROWSER_1",
1589
+ "AI4L_LIBRARY_BROWSER_2",
1590
+ "AI4L_LIBRARY_BROWSER_3",
1591
+ "AI4L_LIBRARY_BROWSER_HEADLESS",
1592
+ "AI4L_LIBRARY_VERIFY_URL",
1593
+ "AI4L_LIBRARY_VERIFY_API_KEY",
1594
+ "AI4L_LIBRARY_VERIFY_MODEL",
1595
+ "AI4L_LIBRARY_VERIFY_THRESHOLD",
1596
+ "AI4L_LIBRARY_VERIFY_TIMEOUT_SECS",
1597
+ "AI4L_LIBRARY_RETRY_AFTER_DAYS",
1598
+ "AI4L_LIBRARY_RETRY_BLOCKED",
1599
+ "AI4L_LIBRARY_REFRESH_AFTER_DAYS",
1600
+ "NCBI_API_KEY",
1601
+ "BRIGHTDATA_API_TOKEN",
1602
+ "BRIGHTDATA_ZONE"
1603
+ ];
1604
+ var sources = (env = process.env) => Object.fromEntries(VARIABLES.map((v) => [v, sourceOf(v, env)]));
1605
+ function settings(env = process.env) {
1928
1606
  return {
1929
- entry,
1930
- entryDir,
1931
- xml: join3(entryDir, `${entry}-abstract.xml`),
1932
- md: join3(entryDir, `${entry}-abstract.md`),
1933
- meta: join3(entryDir, `${entry}-meta.md`),
1934
- fulltextMd: join3(entryDir, `${entry}-fulltext.md`),
1935
- fulltextRaw: (format) => join3(entryDir, `${entry}-fulltext.${format}`)
1607
+ retry_after_days: num("AI4L_LIBRARY_RETRY_AFTER_DAYS", 1, env),
1608
+ retry_blocked: bool("AI4L_LIBRARY_RETRY_BLOCKED", false, env),
1609
+ refresh_after_days: num("AI4L_LIBRARY_REFRESH_AFTER_DAYS", 30, env)
1936
1610
  };
1937
1611
  }
1938
-
1939
- // src/browser.ts
1940
- function browserConfig(env = process.env) {
1941
- const out = { browsers: [], headless: "auto" };
1942
- for (let i = 1; i <= 9; i++) {
1943
- const v = str(`AI4L_LIBRARY_BROWSER_${i}`, env);
1944
- if (v && !out.browsers.includes(v)) out.browsers.push(v);
1612
+ function proxyHeaders(spec) {
1613
+ const out = {};
1614
+ for (const part of spec.split(";")) {
1615
+ const i = part.indexOf(":");
1616
+ if (i < 0) continue;
1617
+ const name = part.slice(0, i).trim(), value = part.slice(i + 1).trim();
1618
+ if (name && value) out[name] = value;
1945
1619
  }
1946
- const h = str("AI4L_LIBRARY_BROWSER_HEADLESS", env).toLowerCase();
1947
- if (h === "true" || h === "false") out.headless = h === "true";
1948
1620
  return out;
1949
1621
  }
1950
- var INSTALL_HINT = "npx playwright-core install chromium";
1951
- var isRemote = (endpoint) => /^wss?:\/\//i.test(endpoint);
1952
- var isCdp = (endpoint) => /^cdp:\/\//i.test(endpoint);
1953
- var cdpHttp = (endpoint) => endpoint.replace(/^cdp:\/\//i, "http://");
1954
- var rungKind = (endpoint) => isCdp(endpoint) ? "chrome" : isRemote(endpoint) ? "server" : "local";
1955
- function localChromium() {
1622
+ function resolveTiers(env = process.env) {
1623
+ const problems = [];
1624
+ const available = /* @__PURE__ */ new Map();
1625
+ const tokens = { elsevier: str("AI4L_LIBRARY_ELSEVIER_API_KEY", env) || void 0, wiley: str("AI4L_LIBRARY_WILEY_TDM_TOKEN", env) || void 0 };
1626
+ for (const t of BUILTIN) available.set(t.name, t.make({ tokens, api_key: str("AI4L_LIBRARY_CORE_API_KEY", env) || void 0 }));
1627
+ const proxyUrl = str("AI4L_LIBRARY_PROXY_URL", env);
1628
+ if (proxyUrl) available.set("proxy", urlTier("proxy", proxyUrl, proxyHeaders(str("AI4L_LIBRARY_PROXY_HEADERS", env))));
1629
+ const listed = str("AI4L_LIBRARY_TIERS", env);
1630
+ const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
1631
+ const tiers = [], seen = /* @__PURE__ */ new Set();
1632
+ for (const name of order2) {
1633
+ if (seen.has(name)) continue;
1634
+ seen.add(name);
1635
+ const t = available.get(name);
1636
+ if (t) tiers.push(t);
1637
+ else if (name === "proxy") problems.push('AI4L_LIBRARY_TIERS: "proxy" listed, but AI4L_LIBRARY_PROXY_URL is unset');
1638
+ else problems.push(`AI4L_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when AI4L_LIBRARY_PROXY_URL is set)`);
1639
+ }
1640
+ const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
1641
+ return { tiers, all, source: sourceOf("AI4L_LIBRARY_TIERS", env), problems };
1642
+ }
1643
+
1644
+ // src/log.ts
1645
+ import { appendFileSync, mkdirSync as mkdirSync2, renameSync, statSync } from "fs";
1646
+ import { hostname } from "os";
1647
+ import { join as join5 } from "path";
1648
+ var HOST = hostname();
1649
+ var LOG_FILE = join5(AI4L_LIBRARY_DIR, "library.log");
1650
+ function stamp() {
1651
+ const d = /* @__PURE__ */ new Date();
1652
+ const p = (n) => String(n).padStart(2, "0");
1653
+ return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
1654
+ }
1655
+ var LOG_MAX_BYTES = 50 * 1024 * 1024;
1656
+ function rotateIfLarge(file, max = LOG_MAX_BYTES) {
1657
+ let size = 0;
1956
1658
  try {
1957
- const p = chromium.executablePath();
1958
- return existsSync4(p) ? p : null;
1659
+ size = statSync(file).size;
1959
1660
  } catch {
1960
- return null;
1661
+ return false;
1961
1662
  }
1962
- }
1963
- function browserLabel(endpoint) {
1964
- if (rungKind(endpoint) === "local") return "local";
1663
+ if (size <= max) return false;
1965
1664
  try {
1966
- const h = new URL(isCdp(endpoint) ? cdpHttp(endpoint) : endpoint).hostname;
1967
- return isCdp(endpoint) ? `chrome@${h}` : h;
1665
+ renameSync(file, `${file}.1`);
1666
+ return true;
1968
1667
  } catch {
1969
- return endpoint;
1668
+ return false;
1970
1669
  }
1971
1670
  }
1972
- var hasDisplay = () => !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
1973
- function browserHeadless(endpoint, cfg = browserConfig()) {
1974
- if (isCdp(endpoint)) return false;
1975
- return cfg.headless === "auto" ? isRemote(endpoint) ? false : !hasDisplay() : cfg.headless;
1976
- }
1977
- function browserRungs(cfg = browserConfig()) {
1978
- return cfg.browsers.map((endpoint) => ({
1979
- endpoint,
1980
- label: browserLabel(endpoint),
1981
- kind: rungKind(endpoint),
1982
- available: rungKind(endpoint) !== "local" || localChromium() !== null,
1983
- headless: browserHeadless(endpoint, cfg)
1984
- }));
1985
- }
1986
- var CHALLENGE = /just a moment|nur einen moment|attention required|verify you are human|human verification|checking your browser|access denied/i;
1987
- var CONNECT_TIMEOUT_MS = 2e4;
1988
- var NAV_TIMEOUT_MS = 2e4;
1989
- var CHALLENGE_WAIT_MS = 8e3;
1990
- var DOWNLOAD_WAIT_MS = 1e4;
1991
- var IDLE_CLOSE_MS = 12e4;
1992
- var ctxs = /* @__PURE__ */ new Map();
1993
- var idleTimer = null;
1994
- function profileDir() {
1995
- const dir = join4(AI4L_LIBRARY_DIR, ".browser");
1996
- mkdirSync2(join4(dir, "Default"), { recursive: true });
1997
- const prefs = join4(dir, "Default", "Preferences");
1998
- let p = {};
1671
+ function log(tool, id, message) {
1672
+ const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
1673
+ const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
1674
+ `;
1999
1675
  try {
2000
- p = JSON.parse(readFileSync3(prefs, "utf8"));
1676
+ mkdirSync2(AI4L_LIBRARY_DIR, { recursive: true });
1677
+ rotateIfLarge(LOG_FILE);
1678
+ appendFileSync(LOG_FILE, line);
2001
1679
  } catch {
2002
1680
  }
2003
- p.plugins = { ...p.plugins ?? {}, always_open_pdf_externally: true };
2004
- p.download = { ...p.download ?? {}, prompt_for_download: false };
2005
- writeFileSync2(prefs, JSON.stringify(p));
2006
- return dir;
2007
- }
2008
- var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
2009
- var LOCAL_WINDOW_ARGS = [...LAUNCH_ARGS, "--window-position=4000,4000"];
2010
- var TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "Europe/Berlin";
2011
- var CONTEXT_OPTS = { acceptDownloads: true, viewport: null, locale: "en-US", timezoneId: TIMEZONE };
2012
- function stateFile(endpoint) {
2013
- const dir = join4(AI4L_LIBRARY_DIR, ".browser");
2014
- mkdirSync2(dir, { recursive: true });
2015
- return join4(dir, `${browserLabel(endpoint).replace(/[^A-Za-z0-9.-]+/g, "_")}.json`);
2016
- }
2017
- async function context(endpoint, cfg) {
2018
- let p = ctxs.get(endpoint);
2019
- if (!p) {
2020
- p = (async () => {
2021
- const headless = browserHeadless(endpoint, cfg);
2022
- const drop = () => {
2023
- if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
2024
- };
2025
- if (isCdp(endpoint)) {
2026
- const browser = await chromium.connectOverCDP(cdpHttp(endpoint), { timeout: CONNECT_TIMEOUT_MS });
2027
- browser.on("disconnected", drop);
2028
- const ctx2 = browser.contexts()[0] ?? await browser.newContext(CONTEXT_OPTS);
2029
- return { ctx: ctx2, browser };
2030
- }
2031
- if (isRemote(endpoint)) {
2032
- const browser = await chromium.connect(endpoint, {
2033
- timeout: CONNECT_TIMEOUT_MS,
2034
- headers: { "x-playwright-launch-options": JSON.stringify({ headless, args: LAUNCH_ARGS, ignoreDefaultArgs: ["--enable-automation"] }) }
2035
- });
2036
- browser.on("disconnected", drop);
2037
- const state = stateFile(endpoint);
2038
- const ctx2 = await browser.newContext({ ...CONTEXT_OPTS, ...existsSync4(state) ? { storageState: state } : {} });
2039
- return { ctx: ctx2, browser };
2040
- }
2041
- if (!localChromium()) throw new Error(`no local Chromium (${INSTALL_HINT})`);
2042
- const ctx = await chromium.launchPersistentContext(profileDir(), {
2043
- channel: "chromium",
2044
- headless,
2045
- ...CONTEXT_OPTS,
2046
- args: headless ? LAUNCH_ARGS : LOCAL_WINDOW_ARGS,
2047
- ignoreDefaultArgs: ["--enable-automation"]
2048
- });
2049
- ctx.on("close", drop);
2050
- return { ctx, browser: null };
2051
- })();
2052
- ctxs.set(endpoint, p);
2053
- p.catch(() => {
2054
- if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
2055
- });
2056
- }
2057
- return (await p).ctx;
2058
- }
2059
- function touchIdle() {
2060
- if (idleTimer) clearTimeout(idleTimer);
2061
- idleTimer = setTimeout(() => {
2062
- void closeBrowsers();
2063
- }, IDLE_CLOSE_MS);
2064
- idleTimer.unref();
2065
- }
2066
- async function closeBrowsers() {
2067
- const open = [...ctxs.entries()];
2068
- ctxs.clear();
2069
- for (const [endpoint, p] of open) {
2070
- try {
2071
- const { ctx, browser } = await p;
2072
- if (isCdp(endpoint)) {
2073
- await browser?.close().catch(() => void 0);
2074
- continue;
2075
- }
2076
- if (browser) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
2077
- await ctx.close().catch(() => void 0);
2078
- await browser?.close().catch(() => void 0);
2079
- } catch {
2080
- }
2081
- }
2082
- }
2083
- async function browserFetch(url, endpoint, cfg = browserConfig()) {
2084
- const ctx = await context(endpoint, cfg);
2085
- touchIdle();
2086
- const page = await ctx.newPage();
2087
- try {
2088
- let pdfBest = null;
2089
- const pdfReads = [];
2090
- let status, contentType = "";
2091
- page.on("response", (r) => {
2092
- if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) {
2093
- status = r.status();
2094
- contentType = r.headers()["content-type"] ?? "";
2095
- }
2096
- if (r.status() !== 200 || !/application\/pdf/i.test(r.headers()["content-type"] ?? "")) return;
2097
- pdfReads.push(r.body().then((b) => {
2098
- if (b.subarray(0, 5).toString("latin1") === "%PDF-" && (!pdfBest || b.length > pdfBest.length)) pdfBest = b;
2099
- }).catch(() => void 0));
2100
- });
2101
- let download2 = null;
2102
- const dl = page.waitForEvent("download", { timeout: NAV_TIMEOUT_MS + CHALLENGE_WAIT_MS + DOWNLOAD_WAIT_MS }).then((d) => {
2103
- download2 = d;
2104
- return d;
2105
- }).catch(() => null);
2106
- let navError = null;
2107
- const nav = page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).then((resp) => {
2108
- if (status === void 0) {
2109
- status = resp?.status();
2110
- contentType = resp?.headers()["content-type"] ?? "";
2111
- }
2112
- }).catch((e) => {
2113
- navError = (e instanceof Error ? e.message : String(e)).split("\n")[0].slice(0, 80);
2114
- });
2115
- await Promise.race([nav, dl]);
2116
- const pause = (ms) => Promise.race([dl, new Promise((r) => setTimeout(r, ms))]);
2117
- if (download2) {
2118
- } else if (navError && !/download is starting|net::ERR_ABORTED/i.test(navError)) {
2119
- await pause(2e3);
2120
- if (!download2) throw new Error(`navigation: ${navError}`);
2121
- } else {
2122
- await pause(2e3);
2123
- const t0 = Date.now();
2124
- while (!download2 && Date.now() - t0 < CHALLENGE_WAIT_MS) {
2125
- const title2 = await page.title().catch(() => "");
2126
- if (!CHALLENGE.test(title2)) break;
2127
- await pause(1e3);
2128
- }
2129
- if (!download2) await pause(DOWNLOAD_WAIT_MS / 5);
2130
- }
2131
- if (download2) {
2132
- const d = download2;
2133
- return { bytes: readFileSync3(await d.path()), contentType: "application/pdf", url: d.url(), via: "download" };
2134
- }
2135
- const title = await page.title().catch(() => "");
2136
- if (CHALLENGE.test(title)) throw new Error(`challenge not cleared (${title.slice(0, 40)})`);
2137
- if (status && status >= 400) throw new Error(`HTTP ${status}`);
2138
- const docType = await page.evaluate(() => document.contentType).catch(() => "");
2139
- const inViewer = /application\/pdf/i.test(contentType) || /application\/pdf/i.test(docType);
2140
- if (inViewer) {
2141
- const t1 = Date.now();
2142
- while (Date.now() - t1 < DOWNLOAD_WAIT_MS) {
2143
- await Promise.all(pdfReads);
2144
- if (pdfBest && pdfBest.length > 4096) break;
2145
- if (!pdfReads.length && Date.now() - t1 > 3e3) break;
2146
- await new Promise((r) => setTimeout(r, 500));
2147
- }
2148
- if (pdfBest) return { bytes: pdfBest, contentType: "application/pdf", url: page.url(), via: "response" };
2149
- }
2150
- if (inViewer) {
2151
- const r = await ctx.request.get(page.url(), { timeout: NAV_TIMEOUT_MS });
2152
- if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
2153
- return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
2154
- }
2155
- const html = await page.content();
2156
- return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
2157
- } finally {
2158
- await page.close().catch(() => void 0);
2159
- if (isRemote(endpoint)) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
2160
- }
2161
1681
  }
2162
1682
 
2163
- // src/unlock.ts
2164
- var ENDPOINT = "https://api.brightdata.com/request";
2165
- var TIMEOUT_MS2 = 6e4;
2166
- var unlockerConfigured = () => !!process.env.BRIGHTDATA_API_TOKEN?.trim();
2167
- async function unlock(url, fetchImpl = fetch) {
2168
- const token = process.env.BRIGHTDATA_API_TOKEN?.trim();
2169
- if (!token) throw new Error("unlocker not configured");
2170
- const zone = process.env.BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
2171
- const res = await fetchImpl(ENDPOINT, {
2172
- method: "POST",
2173
- headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
2174
- body: JSON.stringify({ zone, url, format: "raw" }),
2175
- signal: AbortSignal.timeout(TIMEOUT_MS2)
2176
- });
2177
- if (!res.ok) throw new Error(`unlocker HTTP ${res.status}`);
2178
- return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
2179
- }
1683
+ // src/tools.ts
1684
+ import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
1685
+ import { z } from "zod";
2180
1686
 
2181
- // src/tiers/tier.ts
2182
- function oaVersion(v) {
2183
- if (v === "acceptedVersion") return "accepted_manuscript";
2184
- if (v === "submittedVersion") return "preprint";
2185
- return "published";
2186
- }
2187
- var miss = (tier, reason) => ({ ok: false, tier, reason });
2188
- var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
1687
+ // src/entry.ts
1688
+ import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readdirSync as readdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, rmSync, writeFileSync as writeFileSync2 } from "fs";
1689
+ import { join as join6 } from "path";
1690
+ import { basename, dirname } from "path";
2189
1691
 
2190
- // src/download.ts
2191
- var hostOf = (url) => {
2192
- try {
2193
- return new URL(url).hostname.replace(/^www\./, "");
2194
- } catch {
2195
- return url;
2196
- }
1692
+ // src/pubmed.ts
1693
+ import { XMLParser as XMLParser2 } from "fast-xml-parser";
1694
+ var CORRECTION_TYPES = {
1695
+ ErratumIn: "erratum_in",
1696
+ RetractionIn: "retraction_in",
1697
+ PartialRetractionIn: "partial_retraction_in",
1698
+ RetractedandRepublishedIn: "retracted_and_republished_in",
1699
+ ExpressionOfConcernIn: "expression_of_concern_in",
1700
+ UpdateIn: "update_in"
2197
1701
  };
2198
- var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
2199
- function order(candidates) {
2200
- const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
2201
- return [...candidates].sort((a, b) => rank(a) - rank(b));
2202
- }
2203
- async function download(tier, p, candidates, ctx) {
2204
- const usable = candidates.filter((c) => !NCBI.test(c.url));
2205
- if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
2206
- const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
2207
- if (!ordered.length) return miss(tier, "only copies already tried");
2208
- const reasons = [];
2209
- for (const c of ordered) {
2210
- ctx.tried.add(c.url);
2211
- let r = await tryOne(p, c, tier);
2212
- const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
2213
- if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
2214
- ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
2215
- const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
2216
- for (const b of browserRungs()) {
2217
- if (r.ok || c.headers || c.service || !b.available || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
2218
- r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
2219
- ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
2220
- }
2221
- if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
2222
- r = await tryOne(p, target, tier, "unlocker");
2223
- ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
2224
- }
2225
- if (r.ok) return r;
2226
- reasons.push(`${r.reason}: ${hostOf(c.url)}`);
2227
- }
2228
- return miss(tier, reasons.join("; "));
1702
+ var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
1703
+ var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
1704
+ var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
1705
+ function efetchUrl(pmid) {
1706
+ const u = new URL(EFETCH);
1707
+ u.searchParams.set("db", "pubmed");
1708
+ u.searchParams.set("id", pmid);
1709
+ u.searchParams.set("retmode", "xml");
1710
+ if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
1711
+ u.searchParams.set("tool", "ai4l-library");
1712
+ u.searchParams.set("email", contactEmail());
1713
+ return u.toString();
2229
1714
  }
2230
- var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
2231
- async function tryOne(p, c, tier, via = "plain") {
2232
- let got;
1715
+ async function fetchPubmedXml(pmid) {
2233
1716
  try {
2234
- got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
1717
+ return await fetchText(efetchUrl(pmid));
2235
1718
  } catch (e) {
2236
- const body = e?.body;
2237
- if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
2238
- return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
1719
+ throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
2239
1720
  }
2240
- const { bytes, contentType } = got;
2241
- if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
2242
- if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
2243
- const r = await convertBytes(p, bytes, contentType);
2244
- if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
2245
- return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
2246
1721
  }
2247
- var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
2248
- function serviceMessage(bytes, contentType = "") {
2249
- const text3 = bytes.toString("utf8").trim();
2250
- if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
2251
- try {
2252
- const j = JSON.parse(text3);
2253
- const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
2254
- if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
2255
- } catch {
2256
- }
2257
- const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/\s+/g, " ").trim();
2258
- return (visible || text3).slice(0, 160);
1722
+ var parser2 = new XMLParser2({
1723
+ ignoreAttributes: false,
1724
+ attributeNamePrefix: "@_",
1725
+ parseTagValue: false,
1726
+ processEntities: true,
1727
+ stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
1728
+ isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
1729
+ });
1730
+ function text(n) {
1731
+ if (n == null) return "";
1732
+ if (typeof n === "object") return text(n["#text"]);
1733
+ return String(n);
2259
1734
  }
2260
- function describe(reason, format, md) {
2261
- const words2 = md.split(/\s+/).filter(Boolean).length;
2262
- const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
2263
- const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
2264
- if (reason === "no methods/results") return `no article body (${seen})`;
2265
- if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
2266
- return reason;
1735
+ var plain = (n) => decodeEntities(text(n)).trim();
1736
+ function inlineText2(raw) {
1737
+ return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
2267
1738
  }
2268
- async function convertBytes(p, bytes, contentType = "") {
2269
- let format;
2270
- let md;
2271
- if (looksLikePdf(bytes)) {
2272
- format = "pdf";
2273
- const stats = {};
2274
- try {
2275
- md = await pdfToMarkdown(bytes, p.title, p.headLine, stats);
2276
- } catch (e) {
2277
- return { ok: false, reason: `pdf: ${errMsg(e)}` };
2278
- }
2279
- if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
2280
- } else {
2281
- const text3 = bytes.toString("utf8");
2282
- if (isWallPage(text3)) return { ok: false, reason: "wall page" };
2283
- if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
2284
- format = "xml";
2285
- const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
2286
- if (!doc.title) doc.title = p.title;
2287
- if (!doc.hasBody) return { ok: false, reason: "xml without body" };
2288
- md = renderFulltextMd(doc, p.headLine);
2289
- } else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
2290
- format = "html";
2291
- const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
2292
- md = `# ${p.title}
2293
-
2294
- ${p.headLine}
2295
-
2296
- ${htmlToMarkdown(body)}`;
2297
- } else {
2298
- return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
2299
- }
1739
+ function decodeEntities(s) {
1740
+ return s.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(parseInt(d, 10))).replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&amp;/g, "&");
1741
+ }
1742
+ function parsePubmedXml(xml) {
1743
+ const doc = parser2.parse(xml);
1744
+ const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
1745
+ if (!article) {
1746
+ const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
1747
+ return book ? parseBook(book) : null;
2300
1748
  }
2301
- const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
2302
- const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
2303
- const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
2304
- if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
2305
- return { ok: true, format, md };
1749
+ const cit = article.MedlineCitation ?? {};
1750
+ const art = cit.Article ?? {};
1751
+ const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
1752
+ const idOf = (t) => {
1753
+ const hit = ids.find((i) => i["@_IdType"] === t);
1754
+ return hit ? text(hit).trim() || null : null;
1755
+ };
1756
+ const authorList = art.AuthorList?.[0]?.Author ?? [];
1757
+ const authors = authorList.map((a) => {
1758
+ if (a.CollectiveName) return plain(a.CollectiveName);
1759
+ const last = plain(a.LastName);
1760
+ const ini = plain(a.Initials);
1761
+ return [last, ini].filter(Boolean).join(" ");
1762
+ }).filter(Boolean);
1763
+ const first = authorList[0];
1764
+ const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
1765
+ const pub = art.Journal?.JournalIssue?.PubDate ?? {};
1766
+ let year = text(pub.Year).trim() || null;
1767
+ if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
1768
+ const abstract = (art.Abstract?.AbstractText ?? []).map((a) => ({
1769
+ label: typeof a === "object" ? a["@_Label"] ?? null : null,
1770
+ text: inlineText2(text(a))
1771
+ })).filter((a) => a.text);
1772
+ const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
1773
+ const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
1774
+ if (!type) return [];
1775
+ return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText2(text(c.RefSource)) }];
1776
+ });
1777
+ const funding = [];
1778
+ for (const g of art.GrantList?.Grant ?? []) {
1779
+ const a = plain(g.Agency);
1780
+ if (a && !funding.includes(a)) funding.push(a);
1781
+ }
1782
+ return {
1783
+ coi: inlineText2(text(cit.CoiStatement)) || null,
1784
+ funding,
1785
+ pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
1786
+ pmcid: idOf("pmc"),
1787
+ doi: idOf("doi"),
1788
+ title: inlineText2(text(art.ArticleTitle)),
1789
+ authors,
1790
+ firstAuthor,
1791
+ journal: plain(art.Journal?.Title) || null,
1792
+ year,
1793
+ publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
1794
+ abstract,
1795
+ corrections
1796
+ };
2306
1797
  }
2307
- var ELSEVIER_TAGS = [
2308
- [/ja:article\b/g, "article"],
2309
- [/ja:body\b/g, "body"],
2310
- [/ja:head\b/g, "front"],
2311
- [/ce:sections\b/g, "sec-group"],
2312
- [/ce:section-title\b/g, "title"],
2313
- [/ce:section\b/g, "sec"],
2314
- [/ce:simple-para\b/g, "p"],
2315
- [/ce:para\b/g, "p"],
2316
- [/ce:abstract-sec\b/g, "sec"],
2317
- [/ce:abstract\b/g, "abstract"],
2318
- [/ce:title\b/g, "article-title"],
2319
- [/ce:italic\b/g, "italic"],
2320
- [/ce:bold\b/g, "bold"],
2321
- [/ce:sup\b/g, "sup"],
2322
- [/ce:inf\b/g, "sub"],
2323
- [/ce:cross-refs?\b/g, "xref"],
2324
- [/ce:list-item\b/g, "list-item"],
2325
- [/ce:list\b/g, "list"],
2326
- [/ce:label\b/g, "label"],
2327
- [/ce:caption\b/g, "caption"],
2328
- [/ce:table\b/g, "table-wrap"],
2329
- [/ce:figure\b/g, "fig"],
2330
- [/ce:bib-reference\b/g, "ref"],
2331
- [/ce:bibliography\b/g, "ref-list"],
2332
- [/sb:reference\b/g, "mixed-citation"],
2333
- [/ce:acknowledgment\b/g, "ack"],
2334
- [/ce:inter-ref\b/g, "ext-link"]
2335
- ];
2336
- function elsevierToJats(xml) {
2337
- let out = xml;
2338
- for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
2339
- return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
1798
+ function parseBook(b) {
1799
+ const d = b.BookDocument ?? {};
1800
+ const book = d.Book ?? {};
1801
+ const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
1802
+ const idOf = (t) => {
1803
+ const hit = ids.find((i) => i["@_IdType"] === t);
1804
+ return hit ? text(hit).trim() || null : null;
1805
+ };
1806
+ const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
1807
+ const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
1808
+ const people = pick?.Author ?? [];
1809
+ const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
1810
+ const first = people[0];
1811
+ const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText2(text(a)) })).filter((a) => a.text);
1812
+ const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
1813
+ return {
1814
+ pmid: text(d.PMID).trim() || idOf("pubmed") || "",
1815
+ pmcid: null,
1816
+ doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
1817
+ title: inlineText2(text(d.ArticleTitle)) || inlineText2(text(book.BookTitle)),
1818
+ authors,
1819
+ firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
1820
+ journal: plain(book.Publisher?.PublisherName) || null,
1821
+ year: text(book.PubDate?.Year).trim() || null,
1822
+ publicationTypes: [...new Set(types)],
1823
+ abstract,
1824
+ corrections: [],
1825
+ coi: null,
1826
+ funding: []
1827
+ };
2340
1828
  }
2341
1829
 
2342
- // src/tiers/core.ts
2343
- var WINDOW_MS = 6e4;
2344
- var PER_WINDOW = 10;
2345
- var MAX_WAIT_MS = 65e3;
2346
- var TIMEOUT_MS3 = 1e4;
2347
- var calls = [];
2348
- async function pace() {
2349
- const now = Date.now();
2350
- while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
2351
- if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
2352
- calls.push(Date.now());
1830
+ // src/spl.ts
1831
+ import { XMLParser as XMLParser3 } from "fast-xml-parser";
1832
+ var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
1833
+ var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
1834
+ async function fetchSpl(setid) {
1835
+ const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
1836
+ if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
1837
+ return xml;
2353
1838
  }
2354
- async function coreFetch(url, key) {
2355
- const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
2356
- for (let attempt = 0; attempt < 2; attempt++) {
2357
- await pace();
2358
- const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
2359
- if (res.ok) return await res.text();
2360
- const body = await res.text().catch(() => "");
2361
- if (res.status === 429 && attempt === 0) {
2362
- const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
2363
- const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
2364
- await new Promise((r) => setTimeout(r, wait));
1839
+ var parser3 = new XMLParser3({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
1840
+ var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
1841
+ var kids2 = (n) => n[tagOf2(n)] ?? [];
1842
+ var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
1843
+ var squash2 = (s) => s.replace(/\s+/g, " ").trim();
1844
+ var child = (n, tag) => kids2(n).find((x) => tagOf2(x) === tag);
1845
+ function findDeep2(nodes, tag) {
1846
+ for (const n of nodes) {
1847
+ if ("#text" in n) continue;
1848
+ if (tagOf2(n) === tag) return n;
1849
+ const h = findDeep2(kids2(n), tag);
1850
+ if (h) return h;
1851
+ }
1852
+ return void 0;
1853
+ }
1854
+ function inline(nodes) {
1855
+ let out = "";
1856
+ for (const n of nodes) {
1857
+ if ("#text" in n) {
1858
+ out += String(n["#text"]);
2365
1859
  continue;
2366
1860
  }
2367
- throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
1861
+ const t = tagOf2(n);
1862
+ const inner = () => inline(kids2(n));
1863
+ switch (t) {
1864
+ case "content": {
1865
+ const style = attr2(n, "styleCode").toLowerCase();
1866
+ const s = inner();
1867
+ out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
1868
+ break;
1869
+ }
1870
+ case "sup":
1871
+ out += `^${inner()}^`;
1872
+ break;
1873
+ case "sub":
1874
+ out += `~${inner()}~`;
1875
+ break;
1876
+ case "br":
1877
+ out += " ";
1878
+ break;
1879
+ case "footnote":
1880
+ out += ` (${squash2(inner())})`;
1881
+ break;
1882
+ case "footnoteRef":
1883
+ case "renderMultiMedia":
1884
+ break;
1885
+ default:
1886
+ out += inner();
1887
+ }
2368
1888
  }
2369
- throw new Error("HTTP 429");
1889
+ return out;
2370
1890
  }
2371
- function coreTier(apiKey) {
2372
- const key = apiKey?.trim();
2373
- return {
2374
- name: "core",
2375
- applies: (p) => p.doi ? true : "no doi",
2376
- async run(p, ctx) {
2377
- if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
2378
- let data;
2379
- try {
2380
- const q = encodeURIComponent(`doi:"${p.doi}"`);
2381
- data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
2382
- } catch (e) {
2383
- return miss(this.name, errMsg(e));
2384
- }
2385
- const candidates = coreCandidates(data);
2386
- if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
2387
- ctx.onTry?.(`located ${candidates.length} copies`);
2388
- return download(this.name, p, candidates, ctx);
1891
+ function table(n) {
1892
+ const row2 = (x) => kids2(x).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => squash2(inline(kids2(c))).replace(/\|/g, "\\|"));
1893
+ const collect = (nodes, into) => {
1894
+ for (const x of nodes) {
1895
+ if ("#text" in x) continue;
1896
+ if (tagOf2(x) === "tr") into.push(row2(x));
1897
+ else if (!["thead", "tfoot"].includes(tagOf2(x))) collect(kids2(x), into);
2389
1898
  }
2390
1899
  };
1900
+ const head = [], body = [], foot = [];
1901
+ for (const x of kids2(n)) {
1902
+ const t = tagOf2(x);
1903
+ if (t === "thead") collect(kids2(x), head);
1904
+ else if (t === "tfoot") collect(kids2(x), foot);
1905
+ else if (t === "tr") body.push(row2(x));
1906
+ else if (!("#text" in x) && t !== "caption") collect(kids2(x), body);
1907
+ }
1908
+ const rows = [...head, ...body];
1909
+ if (!rows.length) return "";
1910
+ const width = Math.max(...rows.map((r) => r.length));
1911
+ const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
1912
+ const cap = child(n, "caption");
1913
+ const title = cap ? squash2(inline(kids2(cap))) : "";
1914
+ const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
1915
+ return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
2391
1916
  }
2392
- function coreCandidates(d) {
1917
+ function blocks(nodes, depth) {
2393
1918
  const out = [];
2394
- const seen = /* @__PURE__ */ new Set();
2395
- const add = (url, pdf, host) => {
2396
- if (!url || seen.has(url)) return;
2397
- seen.add(url);
2398
- out.push({ url, version: "published", source: "core", host, pdf });
2399
- };
2400
- for (const w of d.results ?? []) {
2401
- add(w.downloadUrl, true, "core");
2402
- for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
2403
- for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
1919
+ for (const n of nodes) {
1920
+ if ("#text" in n) {
1921
+ const t2 = squash2(String(n["#text"]));
1922
+ if (t2) out.push(t2);
1923
+ continue;
1924
+ }
1925
+ const t = tagOf2(n);
1926
+ const c = kids2(n);
1927
+ switch (t) {
1928
+ case "paragraph": {
1929
+ const s = squash2(inline(c.filter((x) => !["table", "list"].includes(tagOf2(x)))));
1930
+ if (s) out.push(s);
1931
+ out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf2(x))), depth));
1932
+ break;
1933
+ }
1934
+ case "list": {
1935
+ const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr2(n, "listType"));
1936
+ let i = 0;
1937
+ for (const it of c.filter((x) => tagOf2(x) === "item")) {
1938
+ i++;
1939
+ const body = blocks(kids2(it).filter((x) => tagOf2(x) !== "caption"), depth).join(" ");
1940
+ if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
1941
+ }
1942
+ break;
1943
+ }
1944
+ case "table": {
1945
+ const md = table(n);
1946
+ if (md) out.push(md);
1947
+ break;
1948
+ }
1949
+ case "renderMultiMedia":
1950
+ case "observationMedia":
1951
+ case "subject":
1952
+ break;
1953
+ case "section":
1954
+ out.push(...section(n, depth));
1955
+ break;
1956
+ case "component":
1957
+ case "text":
1958
+ case "excerpt":
1959
+ case "highlight":
1960
+ default:
1961
+ out.push(...blocks(c, depth));
1962
+ break;
1963
+ }
2404
1964
  }
2405
1965
  return out;
2406
1966
  }
2407
-
2408
- // src/tiers/europepmc.ts
2409
- var europepmc = {
2410
- name: "europepmc",
2411
- applies: (p) => p.pmid ? true : "no pmid",
2412
- async run(p, ctx) {
2413
- let data;
2414
- try {
2415
- const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
2416
- data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
2417
- } catch (e) {
2418
- return miss(this.name, errMsg(e));
1967
+ var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
1968
+ var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
1969
+ var plain2 = (nodes) => {
1970
+ let out = "";
1971
+ for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf2(n) === "br" ? " " : plain2(kids2(n));
1972
+ return out;
1973
+ };
1974
+ function section(n, depth) {
1975
+ const code = child(n, "code");
1976
+ const c = code ? attr2(code, "code") : "";
1977
+ if (SKIP_CODES.has(c)) return [];
1978
+ const title = child(n, "title");
1979
+ let head = title ? squash2(plain2(kids2(title))) : "";
1980
+ if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
1981
+ if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
1982
+ const out = [];
1983
+ if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
1984
+ const rest = kids2(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf2(x)));
1985
+ out.push(...blocks(rest, head ? depth + 1 : depth));
1986
+ return out;
1987
+ }
1988
+ var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
1989
+ function parseSpl(xml) {
1990
+ const doc = parser3.parse(xml);
1991
+ const d = doc.find((x) => tagOf2(x) === "document");
1992
+ if (!d) return null;
1993
+ const k = kids2(d);
1994
+ const setid = attr2(child(d, "setId") ?? {}, "root").toLowerCase();
1995
+ const version = attr2(child(d, "versionNumber") ?? {}, "value") || null;
1996
+ const effective = isoDate(attr2(child(d, "effectiveTime") ?? {}, "value"));
1997
+ const named = (nodes) => {
1998
+ for (const n of nodes) {
1999
+ if ("#text" in n) continue;
2000
+ if (tagOf2(n) === "manufacturedProduct") {
2001
+ const inner2 = child(n, "manufacturedProduct") ?? n;
2002
+ if (child(inner2, "name")) return inner2;
2003
+ }
2004
+ const h = named(kids2(n));
2005
+ if (h) return h;
2419
2006
  }
2420
- const rec = data.resultList?.result?.[0];
2421
- if (!rec) return miss(this.name, "not in europe pmc");
2422
- const candidates = europepmcCandidates(rec, p.pmcid);
2423
- if (!candidates.length) return miss(this.name, "no free copy");
2424
- ctx.onTry?.(`located ${candidates.length} copies`);
2425
- return download(this.name, p, candidates, ctx);
2007
+ return void 0;
2008
+ };
2009
+ const inner = named(k);
2010
+ const brand = inner ? squash2(inline(kids2(child(inner, "name") ?? { name: [] }))) || null : null;
2011
+ const genericNode = inner ? findDeep2(kids2(inner), "genericMedicine") : void 0;
2012
+ const generic = genericNode ? squash2(inline(kids2(child(genericNode, "name") ?? { name: [] }))) || null : null;
2013
+ const formNode = inner ? child(inner, "formCode") : void 0;
2014
+ const form = formNode ? attr2(formNode, "displayName").toLowerCase() || null : null;
2015
+ const org = findDeep2(k, "representedOrganization");
2016
+ const labeler = org ? squash2(inline(kids2(child(org, "name") ?? { name: [] }))) || null : null;
2017
+ const body = findDeep2(k, "structuredBody");
2018
+ const sections2 = body ? blocks(kids2(body), 2) : [];
2019
+ const docTitle = child(d, "title");
2020
+ const lines = [];
2021
+ let cur = "";
2022
+ for (const x of docTitle ? kids2(docTitle) : []) {
2023
+ if (!("#text" in x) && tagOf2(x) === "br") {
2024
+ lines.push(cur);
2025
+ cur = "";
2026
+ } else cur += "#text" in x ? String(x["#text"]) : inline([x]);
2426
2027
  }
2028
+ lines.push(cur);
2029
+ const highlights = lines.map(squash2).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
2030
+ const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
2031
+ const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
2032
+ const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
2033
+ return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
2034
+ }
2035
+ function headLineSpl(s) {
2036
+ return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
2037
+ }
2038
+ function renderLabelMd(s) {
2039
+ const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
2040
+ if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
2041
+ parts.push(...s.sections.flatMap((b) => [b, ""]));
2042
+ return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
2043
+ }
2044
+
2045
+ // src/ctgov.ts
2046
+ var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
2047
+ var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
2048
+ async function fetchStudy(nctid) {
2049
+ const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
2050
+ if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
2051
+ return json2;
2052
+ }
2053
+ var STATUS = {
2054
+ NOT_YET_RECRUITING: "Not yet recruiting",
2055
+ RECRUITING: "Recruiting",
2056
+ ENROLLING_BY_INVITATION: "Enrolling by invitation",
2057
+ ACTIVE_NOT_RECRUITING: "Active, not recruiting",
2058
+ SUSPENDED: "Suspended",
2059
+ TERMINATED: "Terminated",
2060
+ COMPLETED: "Completed",
2061
+ WITHDRAWN: "Withdrawn",
2062
+ UNKNOWN: "Unknown status",
2063
+ AVAILABLE: "Available",
2064
+ NO_LONGER_AVAILABLE: "No longer available",
2065
+ TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
2066
+ APPROVED_FOR_MARKETING: "Approved for marketing",
2067
+ WITHHELD: "Withheld"
2427
2068
  };
2428
- function europepmcCandidates(r, pmcid) {
2069
+ var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
2070
+ var words = (v) => {
2071
+ const s = String(v ?? "");
2072
+ if (!s) return "";
2073
+ if (STATUS[s]) return STATUS[s];
2074
+ if (PHASE[s]) return PHASE[s];
2075
+ const w = s.toLowerCase().replace(/_/g, " ");
2076
+ return w.charAt(0).toUpperCase() + w.slice(1);
2077
+ };
2078
+ var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
2079
+ var list = (a) => Array.isArray(a) ? a : [];
2080
+ var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
2081
+ var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
2082
+ var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
2083
+ function parseStudy(json2) {
2084
+ let d;
2085
+ try {
2086
+ d = JSON.parse(json2);
2087
+ } catch {
2088
+ return null;
2089
+ }
2090
+ const ps = d?.protocolSection;
2091
+ if (!ps) return null;
2092
+ const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
2093
+ const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
2094
+ const nctid = String(idm.nctId ?? "").toUpperCase();
2429
2095
  const out = [];
2430
- const seen = /* @__PURE__ */ new Set();
2431
- const add = (url, pdf, host, version) => {
2432
- if (!url || seen.has(url)) return;
2433
- seen.add(url);
2434
- out.push({ url, version, source: "europepmc", host, pdf });
2096
+ const sec = (title, lines) => {
2097
+ const body = lines.filter((l) => !!l && l.trim() !== "");
2098
+ if (body.length) out.push(`## ${title}`, ...body);
2435
2099
  };
2436
- const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
2437
- const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
2438
- if (id && r.inEPMC === "Y") {
2439
- if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
2440
- if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
2100
+ const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
2101
+ sec("Identification", [
2102
+ idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
2103
+ idm.acronym ? `Acronym: ${idm.acronym}` : null,
2104
+ idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
2105
+ secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
2106
+ ]);
2107
+ sec("Status", [
2108
+ `Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
2109
+ st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
2110
+ st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
2111
+ st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
2112
+ st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
2113
+ st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
2114
+ st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
2115
+ `Results posted: ${d.hasResults ? "yes" : "no"}`
2116
+ ]);
2117
+ const rp = sp.responsibleParty;
2118
+ sec("Sponsor and collaborators", [
2119
+ sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
2120
+ list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
2121
+ rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
2122
+ ]);
2123
+ sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
2124
+ sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
2125
+ const di = dm.designInfo ?? {};
2126
+ const mi = di.maskingInfo ?? {};
2127
+ sec("Design", [
2128
+ dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
2129
+ list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
2130
+ di.allocation ? `Allocation: ${words(di.allocation)}` : null,
2131
+ di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
2132
+ di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
2133
+ mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
2134
+ dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
2135
+ ]);
2136
+ const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
2137
+ const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
2138
+ sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
2139
+ const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
2140
+ sec("Outcomes", [
2141
+ list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
2142
+ ...list(om.primaryOutcomes).map(outcome),
2143
+ list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
2144
+ ...list(om.secondaryOutcomes).map(outcome),
2145
+ list(om.otherOutcomes).length ? "### Other outcomes" : null,
2146
+ ...list(om.otherOutcomes).map(outcome)
2147
+ ]);
2148
+ sec("Eligibility", [
2149
+ [el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
2150
+ el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
2151
+ ]);
2152
+ const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
2153
+ sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
2154
+ const refs = list(rf.references);
2155
+ sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
2156
+ const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
2157
+ if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
2158
+ return {
2159
+ nctid,
2160
+ title: para(idm.briefTitle) || nctid,
2161
+ acronym: idm.acronym ?? null,
2162
+ sponsor: sp.leadSponsor?.name ?? null,
2163
+ firstPosted: st.studyFirstPostDateStruct?.date ?? null,
2164
+ lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
2165
+ status: st.overallStatus ? words(st.overallStatus) : null,
2166
+ hasResults: !!d.hasResults,
2167
+ publications,
2168
+ sections: out
2169
+ };
2170
+ }
2171
+ function results(rs) {
2172
+ const out = ["## Results"];
2173
+ const pf = rs.participantFlowModule;
2174
+ if (pf) {
2175
+ out.push("### Participant flow");
2176
+ const groups = list(pf.groups);
2177
+ out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
2178
+ if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
2179
+ for (const p of list(pf.periods)) {
2180
+ const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
2181
+ const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
2182
+ for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
2183
+ out.push(table2(head, rows));
2184
+ }
2441
2185
  }
2442
- for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
2443
- if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
2444
- if (u.documentStyle === "abs") continue;
2445
- if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
2446
- add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
2186
+ const bc = rs.baselineCharacteristicsModule;
2187
+ if (bc) {
2188
+ out.push("### Baseline characteristics");
2189
+ const groups = list(bc.groups);
2190
+ out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
2191
+ for (const m of list(bc.measures)) {
2192
+ const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
2193
+ const rows = [];
2194
+ for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
2195
+ const x = list(cat.measurements).find((v) => v.groupId === g.id);
2196
+ return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
2197
+ })]);
2198
+ out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
2199
+ }
2200
+ }
2201
+ const om = rs.outcomeMeasuresModule;
2202
+ if (om) {
2203
+ out.push("### Outcome measures");
2204
+ for (const m of list(om.outcomeMeasures)) {
2205
+ const groups = list(m.groups);
2206
+ const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
2207
+ out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
2208
+ if (m.description) out.push(para(m.description));
2209
+ out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
2210
+ if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
2211
+ const rows = [];
2212
+ for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
2213
+ const x = list(cat.measurements).find((v) => v.groupId === g.id);
2214
+ return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
2215
+ })]);
2216
+ const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
2217
+ if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
2218
+ out.push(table2(["", ...groups.map((g) => g.title)], rows));
2219
+ for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
2220
+ }
2221
+ }
2222
+ const ae = rs.adverseEventsModule;
2223
+ if (ae) {
2224
+ out.push("### Adverse events");
2225
+ const groups = list(ae.eventGroups);
2226
+ out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
2227
+ const summary = [
2228
+ ["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
2229
+ ["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
2230
+ ["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
2231
+ ].filter((r) => r.slice(1).some(Boolean));
2232
+ out.push(table2(["", ...groups.map((g) => g.title)], summary));
2233
+ const events = (title, evs) => {
2234
+ if (!evs.length) return;
2235
+ out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
2236
+ const s = list(e.stats).find((x) => x.groupId === g.id);
2237
+ return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
2238
+ })])));
2239
+ };
2240
+ events("Serious adverse events", list(ae.seriousEvents));
2241
+ events("Other adverse events", list(ae.otherEvents));
2447
2242
  }
2448
2243
  return out;
2449
2244
  }
2245
+ function headLineStudy(s) {
2246
+ return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
2247
+ }
2248
+ function renderStudyMd(s) {
2249
+ const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
2250
+ return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
2251
+ }
2450
2252
 
2451
- // src/tiers/inbox.ts
2452
- import { existsSync as existsSync5, readdirSync as readdirSync2, readFileSync as readFileSync4, unlinkSync } from "fs";
2453
- import { join as join5 } from "path";
2454
- var INBOX_DIR = join5(AI4L_LIBRARY_DIR, "inbox");
2455
- var EXT = { ".pdf": "application/pdf", ".xml": "text/xml", ".html": "text/html", ".htm": "text/html" };
2456
- function findInboxFile(dir, pmid) {
2457
- if (!existsSync5(dir)) return null;
2458
- const re = new RegExp(`(^|[^0-9])${pmid}([^0-9]|$)`);
2459
- for (const name of readdirSync2(dir).sort()) {
2460
- const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
2461
- if (ext in EXT && re.test(name.slice(0, -ext.length))) return join5(dir, name);
2253
+ // src/entry.ts
2254
+ function emptyStatus(requested, type, id, libraryDir) {
2255
+ return {
2256
+ requested,
2257
+ type,
2258
+ id,
2259
+ entry: null,
2260
+ entry_dir: null,
2261
+ library_dir: libraryDir,
2262
+ pmid: null,
2263
+ setid: null,
2264
+ nctid: null,
2265
+ publications: [],
2266
+ pmcid: null,
2267
+ doi: null,
2268
+ title: null,
2269
+ first_author: null,
2270
+ year: null,
2271
+ publication_types: [],
2272
+ retracted: false,
2273
+ corrections: [],
2274
+ coi: null,
2275
+ funding: [],
2276
+ citable: true,
2277
+ comment: null,
2278
+ has_abstract: null,
2279
+ level: null,
2280
+ version: null,
2281
+ retrieved_by: null,
2282
+ retrieved_date: null,
2283
+ pubmed_checked: null,
2284
+ attempts: [],
2285
+ files: [],
2286
+ glyphs_unresolved: null
2287
+ };
2288
+ }
2289
+ var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
2290
+ function headLine(r) {
2291
+ return [
2292
+ r.authors.join(", ") || null,
2293
+ r.journal,
2294
+ r.year,
2295
+ r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
2296
+ r.doi ? `DOI ${r.doi}` : null,
2297
+ r.pmcid ? `PMCID ${r.pmcid}` : null
2298
+ ].filter(Boolean).join(" \xB7 ");
2299
+ }
2300
+ function renderAbstractMd(r) {
2301
+ const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
2302
+ return `# ${r.title}
2303
+
2304
+ ${headLine(r)}
2305
+
2306
+ ## Abstract
2307
+
2308
+ ${body}
2309
+ `;
2310
+ }
2311
+ function readHeadLine(p) {
2312
+ try {
2313
+ const lines = readFileSync4(p.md, "utf8").split("\n").filter((l) => l.trim());
2314
+ return lines[1] ?? "";
2315
+ } catch {
2316
+ return "";
2462
2317
  }
2463
- return null;
2464
2318
  }
2465
- function inboxTier(dir = INBOX_DIR) {
2319
+ function metaFromRecord(r, files, converter) {
2466
2320
  return {
2467
- name: "inbox",
2468
- always: true,
2469
- applies: (p) => p.pmid ? true : "no pmid",
2470
- async run(p, ctx) {
2471
- const file = findInboxFile(dir, p.pmid);
2472
- if (!file) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
2473
- const name = file.slice(file.lastIndexOf("/") + 1);
2474
- const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
2475
- const bytes = readFileSync4(file);
2476
- const r = await convertBytes(p, bytes, EXT[ext]);
2477
- if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
2478
- ctx.onTry?.(`${name} ok, removed from the inbox`);
2479
- unlinkSync(file);
2480
- return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
2481
- }
2321
+ pmid: r.pmid,
2322
+ setid: null,
2323
+ nctid: null,
2324
+ publications: [],
2325
+ pmcid: r.pmcid,
2326
+ doi: r.doi,
2327
+ title: r.title,
2328
+ first_author: r.firstAuthor,
2329
+ year: r.year,
2330
+ publication_types: r.publicationTypes,
2331
+ retracted: isRetracted(r),
2332
+ corrections: r.corrections,
2333
+ coi: r.coi,
2334
+ funding: r.funding,
2335
+ has_abstract: r.abstract.length > 0,
2336
+ level: "abstract",
2337
+ version: null,
2338
+ files,
2339
+ retrieved_by: "pubmed",
2340
+ retrieved_date: today(),
2341
+ converted_by: converter,
2342
+ pubmed_checked: today(),
2343
+ attempts: []
2482
2344
  };
2483
2345
  }
2484
- var inbox = inboxTier();
2485
-
2486
- // src/tiers/openalex.ts
2487
- var openalex = {
2488
- name: "openalex",
2489
- applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
2490
- async run(p, ctx) {
2491
- const mail = contactEmail();
2492
- const key = p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`;
2493
- let data;
2494
- try {
2495
- data = JSON.parse(await fetchText(`https://api.openalex.org/works/${encodeURIComponent(key)}?mailto=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
2496
- } catch (e) {
2497
- return miss(this.name, errMsg(e));
2498
- }
2499
- const candidates = openalexCandidates(data);
2500
- if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
2501
- ctx.onTry?.(`located ${candidates.length} copies`);
2502
- return download(this.name, p, candidates, ctx);
2346
+ var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
2347
+ var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
2348
+ function parseCorrection(v) {
2349
+ const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
2350
+ if (!m) return null;
2351
+ let source;
2352
+ try {
2353
+ source = JSON.parse(m[3]);
2354
+ } catch {
2355
+ return null;
2503
2356
  }
2504
- };
2505
- function openalexCandidates(d) {
2506
- const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
2507
- const seen = /* @__PURE__ */ new Set();
2508
- const out = [];
2509
- for (const l of locs) {
2510
- for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
2511
- if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
2512
- seen.add(url);
2513
- out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
2514
- }
2357
+ return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
2358
+ }
2359
+ function renderMeta(m) {
2360
+ const lines = [
2361
+ ["pmid", m.pmid],
2362
+ ...m.setid ? [["setid", m.setid]] : [],
2363
+ ...m.nctid ? [["nctid", m.nctid]] : [],
2364
+ ...m.nctid ? [["publications", m.publications.join(", ")]] : [],
2365
+ ["pmcid", m.pmcid ?? ""],
2366
+ ["doi", m.doi ?? ""],
2367
+ ["title", JSON.stringify(m.title)],
2368
+ ["first_author", m.first_author ?? ""],
2369
+ ["year", m.year ?? ""],
2370
+ ["publication_types", m.publication_types.join("; ")],
2371
+ ["retracted", m.retracted ? "yes" : "no"],
2372
+ ...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
2373
+ ["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
2374
+ ["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
2375
+ // a JSON array: a statement may hold semicolons
2376
+ ["has_abstract", m.has_abstract ? "yes" : "no"],
2377
+ ["level", m.level],
2378
+ ["version", m.version ?? ""],
2379
+ ["files", m.files.join(", ")],
2380
+ ["retrieved_by", m.retrieved_by],
2381
+ ["retrieved_date", m.retrieved_date],
2382
+ ["converted_by", m.converted_by],
2383
+ ["pubmed_checked", m.pubmed_checked],
2384
+ ["attempts", m.attempts.join(", ")]
2385
+ ];
2386
+ return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
2387
+ }
2388
+ function parseFrontmatter(md) {
2389
+ const m = /^---\n([\s\S]*?)\n---/.exec(md);
2390
+ const fm = {}, all = {};
2391
+ for (const line of (m?.[1] ?? "").split("\n")) {
2392
+ const i = line.indexOf(":");
2393
+ if (i <= 0) continue;
2394
+ const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
2395
+ fm[k] = v;
2396
+ (all[k] ??= []).push(v);
2515
2397
  }
2516
- return out;
2398
+ return { fm, all };
2517
2399
  }
2518
-
2519
- // src/tiers/pmc.ts
2520
- function pmcEfetchUrl(pmcid) {
2521
- const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
2522
- u.searchParams.set("db", "pmc");
2523
- u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
2524
- u.searchParams.set("retmode", "xml");
2525
- if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
2526
- u.searchParams.set("tool", "ai4l-library");
2527
- u.searchParams.set("email", contactEmail());
2528
- return u.toString();
2400
+ var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
2401
+ function splitAttempts(v) {
2402
+ if (!v) return [];
2403
+ const out = [];
2404
+ let cur = "", depth = 0;
2405
+ for (const ch of v) {
2406
+ if (ch === "(") depth++;
2407
+ else if (ch === ")") depth = Math.max(0, depth - 1);
2408
+ if (ch === "," && depth === 0) {
2409
+ out.push(cur);
2410
+ cur = "";
2411
+ } else cur += ch;
2412
+ }
2413
+ out.push(cur);
2414
+ const merged = [];
2415
+ for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
2416
+ if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
2417
+ else merged.push(piece);
2418
+ }
2419
+ return merged;
2529
2420
  }
2530
- var pmc = {
2531
- name: "pmc",
2532
- applies: (p) => p.pmcid ? true : "no pmcid",
2533
- async run(p) {
2534
- let xml;
2535
- try {
2536
- xml = await fetchText(pmcEfetchUrl(p.pmcid));
2537
- } catch (e) {
2538
- return miss(this.name, errMsg(e));
2539
- }
2540
- const doc = parseJats(xml);
2541
- if (!doc.hasBody) return miss(this.name, "no body");
2542
- const md = renderFulltextMd(doc, p.headLine);
2543
- const check = checkFulltext(md, p.title);
2544
- if (!check.ok) return miss(this.name, check.reason);
2545
- return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
2421
+ var opt = (v) => v ? v : null;
2422
+ var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
2423
+ function splitPublicationTypes(v) {
2424
+ if (!v) return [];
2425
+ if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
2426
+ const out = [];
2427
+ for (const piece of list2(v)) {
2428
+ if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
2429
+ else out.push(piece);
2546
2430
  }
2547
- };
2548
-
2549
- // src/tiers/tdm.ts
2550
- var CROSSREF = "https://api.crossref.org/works/";
2551
- var TOKENED = [
2552
- { host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
2553
- { host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
2554
- ];
2555
- var hostOf2 = (url) => {
2431
+ return out;
2432
+ }
2433
+ function abstractMissing(p) {
2556
2434
  try {
2557
- return new URL(url).hostname;
2435
+ return readFileSync4(p.md, "utf8").includes("_No abstract available._");
2558
2436
  } catch {
2559
- return "";
2437
+ return false;
2560
2438
  }
2561
- };
2562
- function tdmCandidates(work, tokens = {}, env = process.env) {
2563
- const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
2564
- const seen = /* @__PURE__ */ new Set();
2565
- const candidates = [];
2566
- const missingTokens = /* @__PURE__ */ new Set();
2567
- for (const l of links) {
2568
- const ct = (l["content-type"] ?? "").toLowerCase();
2569
- if (seen.has(l.URL)) {
2570
- const prev = candidates.find((c) => c.url === l.URL);
2571
- if (prev && /pdf/.test(ct)) prev.pdf = true;
2572
- continue;
2439
+ }
2440
+ function correctionsFromXml(p) {
2441
+ try {
2442
+ return parsePubmedXml(readFileSync4(p.xml, "utf8"))?.corrections ?? [];
2443
+ } catch {
2444
+ return [];
2445
+ }
2446
+ }
2447
+ function declarationsFromXml(p) {
2448
+ try {
2449
+ const r = parsePubmedXml(readFileSync4(p.xml, "utf8"));
2450
+ return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
2451
+ } catch {
2452
+ return { coi: null, funding: [] };
2453
+ }
2454
+ }
2455
+ function readMeta(p) {
2456
+ const { fm, all } = parseFrontmatter(readFileSync4(p.meta, "utf8"));
2457
+ let title = fm.title ?? "";
2458
+ if (title.startsWith('"')) {
2459
+ try {
2460
+ title = JSON.parse(title);
2461
+ } catch {
2573
2462
  }
2574
- seen.add(l.URL);
2575
- const rule = TOKENED.find((r) => r.host.test(hostOf2(l.URL)));
2576
- let headers;
2577
- if (rule) {
2578
- const token = tokens[rule.key]?.trim() || env[rule.env]?.trim();
2579
- if (!token) {
2580
- missingTokens.add(rule.key);
2581
- continue;
2463
+ }
2464
+ const legacy = fm.pubmed_checked === void 0;
2465
+ const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
2466
+ let coi = null, funding = [];
2467
+ if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
2468
+ else {
2469
+ if (fm.coi.startsWith('"')) {
2470
+ try {
2471
+ coi = JSON.parse(fm.coi);
2472
+ } catch {
2473
+ coi = fm.coi;
2582
2474
  }
2583
- headers = { [rule.header]: token };
2584
- }
2585
- if (/text\/plain/.test(ct)) continue;
2586
- candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf2(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
2475
+ } else coi = fm.coi || null;
2476
+ funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
2587
2477
  }
2588
- const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
2589
- candidates.sort((a, b) => rank(a) - rank(b));
2590
- return { candidates, missingTokens: [...missingTokens] };
2478
+ return {
2479
+ pmid: fm.pmid ?? "",
2480
+ setid: opt(fm.setid),
2481
+ nctid: opt(fm.nctid),
2482
+ publications: list2(fm.publications),
2483
+ pmcid: opt(fm.pmcid),
2484
+ doi: opt(fm.doi),
2485
+ title,
2486
+ first_author: opt(fm.first_author),
2487
+ year: opt(fm.year),
2488
+ publication_types: splitPublicationTypes(fm.publication_types),
2489
+ retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
2490
+ corrections,
2491
+ coi,
2492
+ funding,
2493
+ has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
2494
+ level: fm.level === "full_text" ? "full_text" : "abstract",
2495
+ version: opt(fm.version),
2496
+ files: list2(fm.files),
2497
+ retrieved_by: fm.retrieved_by ?? "",
2498
+ retrieved_date: fm.retrieved_date ?? "",
2499
+ converted_by: fm.converted_by ?? "",
2500
+ pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
2501
+ attempts: splitAttempts(fm.attempts)
2502
+ };
2591
2503
  }
2592
- function tdmTier(tokens = {}) {
2504
+ function writeAtomic(file, data) {
2505
+ const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
2506
+ writeFileSync2(tmp, data);
2507
+ renameSync2(tmp, file);
2508
+ }
2509
+ function writeMeta(p, m) {
2510
+ writeAtomic(p.meta, renderMeta(m));
2511
+ }
2512
+ function copyEntry(from, to) {
2513
+ const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
2514
+ rmSync(tmp, { recursive: true, force: true });
2515
+ mkdirSync3(tmp, { recursive: true });
2516
+ const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
2517
+ for (const n of readdirSync2(from.entryDir)) if (join6(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join6(from.entryDir, n), join6(tmp, rename(n)));
2518
+ const m = readMeta(from);
2519
+ const copy = { ...m, files: m.files.map(rename) };
2520
+ writeFileSync2(join6(tmp, basename(to.meta)), renderMeta(copy));
2521
+ mkdirSync3(dirname(to.entryDir), { recursive: true });
2522
+ if (existsSync5(to.entryDir)) renameSync2(to.entryDir, old);
2523
+ renameSync2(tmp, to.entryDir);
2524
+ rmSync(old, { recursive: true, force: true });
2525
+ return copy;
2526
+ }
2527
+ function citability(m) {
2528
+ const of = (t) => m.corrections.filter((c) => c.type === t);
2529
+ const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
2530
+ if (m.retracted) {
2531
+ const r = m.corrections.filter(isRetractionNotice);
2532
+ return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
2533
+ }
2534
+ const notes = [
2535
+ ...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
2536
+ ...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
2537
+ ...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
2538
+ ];
2539
+ return { citable: true, comment: notes.length ? notes.join("; ") : null };
2540
+ }
2541
+ function statusFromMeta(p, m, base) {
2542
+ const full = existsSync5(p.fulltextMd) ? readFileSync4(p.fulltextMd, "utf8") : null;
2543
+ const cit = citability(m);
2544
+ if (full?.includes(`
2545
+ ${CORRECTED_HEADING}
2546
+ `) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
2593
2547
  return {
2594
- name: "tdm",
2595
- applies: (p) => p.doi ? true : "no doi",
2596
- async run(p, ctx) {
2597
- let work;
2598
- try {
2599
- work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
2600
- } catch (e) {
2601
- return miss(this.name, errMsg(e));
2602
- }
2603
- const { candidates, missingTokens } = tdmCandidates(work, tokens);
2604
- if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
2605
- ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
2606
- return download(this.name, p, candidates, ctx);
2607
- }
2548
+ ...base,
2549
+ entry: p.entry,
2550
+ entry_dir: p.entryDir,
2551
+ pmid: m.pmid || null,
2552
+ setid: m.setid,
2553
+ nctid: m.nctid,
2554
+ publications: m.publications,
2555
+ pmcid: m.pmcid,
2556
+ doi: m.doi,
2557
+ title: m.title || null,
2558
+ first_author: m.first_author,
2559
+ year: m.year,
2560
+ publication_types: m.publication_types,
2561
+ retracted: m.retracted,
2562
+ corrections: m.corrections,
2563
+ coi: m.coi,
2564
+ funding: m.funding,
2565
+ ...cit,
2566
+ // What the entry holds: full_text, abstract, or only the title (the meta's
2567
+ // level is "abstract" until a full text arrives; has_abstract says whether
2568
+ // there is abstract text).
2569
+ has_abstract: m.has_abstract,
2570
+ level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
2571
+ version: m.version,
2572
+ retrieved_by: m.retrieved_by || null,
2573
+ retrieved_date: m.retrieved_date || null,
2574
+ pubmed_checked: m.pubmed_checked || null,
2575
+ attempts: m.attempts,
2576
+ files: m.files,
2577
+ glyphs_unresolved: full === null ? null : countUnresolved(full)
2608
2578
  };
2609
2579
  }
2610
- var tdm = tdmTier();
2611
-
2612
- // src/tiers/unpaywall.ts
2613
- var unpaywall = {
2614
- name: "unpaywall",
2615
- applies: (p) => p.doi ? true : "no doi",
2616
- async run(p, ctx) {
2617
- const mail = contactEmail();
2618
- let data;
2619
- try {
2620
- data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
2621
- } catch (e) {
2622
- return miss(this.name, errMsg(e));
2580
+ function readStatus(p, base) {
2581
+ return statusFromMeta(p, readMeta(p), base);
2582
+ }
2583
+ function entryExists(p) {
2584
+ return existsSync5(p.meta);
2585
+ }
2586
+ function writeEntry(p, r, xml, converter) {
2587
+ mkdirSync3(p.entryDir, { recursive: true });
2588
+ writeAtomic(p.xml, xml);
2589
+ writeAtomic(p.md, renderAbstractMd(r));
2590
+ const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
2591
+ writeMeta(p, meta);
2592
+ return meta;
2593
+ }
2594
+ function refreshRecord(p, m, r, xml) {
2595
+ writeAtomic(p.xml, xml);
2596
+ syncCorrectedBlock(p);
2597
+ const next = {
2598
+ ...m,
2599
+ pmcid: r.pmcid ?? m.pmcid,
2600
+ doi: r.doi ?? m.doi,
2601
+ publication_types: r.publicationTypes,
2602
+ retracted: isRetracted(r),
2603
+ corrections: r.corrections,
2604
+ coi: r.coi ?? m.coi,
2605
+ funding: r.funding.length ? r.funding : m.funding,
2606
+ // the paper's own statements stay when PubMed has none
2607
+ pubmed_checked: today()
2608
+ };
2609
+ writeMeta(p, next);
2610
+ return next;
2611
+ }
2612
+ function daysSince(date, now = /* @__PURE__ */ new Date()) {
2613
+ const t = new Date(date).getTime();
2614
+ return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
2615
+ }
2616
+ var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
2617
+ var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
2618
+ function declarationsFromMd(md) {
2619
+ let coi = null;
2620
+ const funding = [];
2621
+ const lines = md.split("\n");
2622
+ let section2 = "";
2623
+ for (let i = 0; i < lines.length; i++) {
2624
+ const l = lines[i].trim();
2625
+ const h = /^#{2,6}\s+(.*)$/.exec(l);
2626
+ if (h) {
2627
+ section2 = h[1];
2628
+ const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
2629
+ if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
2630
+ if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
2631
+ continue;
2623
2632
  }
2624
- const candidates = unpaywallCandidates(data);
2625
- if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
2626
- ctx.onTry?.(`located ${candidates.length} copies`);
2627
- return download(this.name, p, candidates, ctx);
2628
- }
2629
- };
2630
- function unpaywallCandidates(d) {
2631
- const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
2632
- const seen = /* @__PURE__ */ new Set();
2633
+ if (section2 !== "Declarations" || !l) continue;
2634
+ const f = FUNDING_RE.exec(l);
2635
+ const c = COI_RE.exec(l);
2636
+ if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
2637
+ else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
2638
+ }
2639
+ return { coi, funding };
2640
+ }
2641
+ var CORRECTED_HEADING = "## Corrected in PubMed abstract";
2642
+ function correctedSentences(r) {
2633
2643
  const out = [];
2634
- for (const l of locs) {
2635
- for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
2636
- if (!url || seen.has(url)) continue;
2637
- seen.add(url);
2638
- out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
2644
+ for (const a of r.abstract) {
2645
+ for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
2646
+ const i = s.indexOf("[corrected]");
2647
+ if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
2639
2648
  }
2640
2649
  }
2641
- return out;
2650
+ return out.filter(Boolean);
2642
2651
  }
2652
+ function withCorrectedBlock(md, sentences) {
2653
+ const base = md.split(`
2654
+ ${CORRECTED_HEADING}
2655
+ `)[0].trimEnd();
2656
+ if (!sentences.length) return base + "\n";
2657
+ return `${base}
2643
2658
 
2644
- // src/tiers/index.ts
2645
- var BUILTIN = [
2646
- { name: "inbox", make: () => inbox },
2647
- { name: "pmc", make: () => pmc },
2648
- { name: "europepmc", make: () => europepmc },
2649
- { name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
2650
- { name: "unpaywall", make: () => unpaywall },
2651
- { name: "openalex", make: () => openalex },
2652
- { name: "core", make: (o) => coreTier(o.api_key) }
2653
- ];
2659
+ ${CORRECTED_HEADING}
2654
2660
 
2655
- // src/tiers/url.ts
2656
- var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
2657
- function urlTier(name, template, headers) {
2658
- const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
2659
- return {
2660
- name,
2661
- applies(p) {
2662
- const missing = used.filter((k) => !p[k]);
2663
- return missing.length ? `no ${missing.join("/")}` : true;
2664
- },
2665
- async run(p, ctx) {
2666
- if (!used.length) return miss(name, "template has no placeholder");
2667
- const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
2668
- return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
2669
- }
2661
+ NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
2662
+
2663
+ ${sentences.join("\n\n")}
2664
+ `;
2665
+ }
2666
+ function syncCorrectedBlock(p) {
2667
+ if (!existsSync5(p.fulltextMd) || !existsSync5(p.xml)) return;
2668
+ let r;
2669
+ try {
2670
+ r = parsePubmedXml(readFileSync4(p.xml, "utf8"));
2671
+ } catch {
2672
+ return;
2673
+ }
2674
+ if (!r) return;
2675
+ const md = readFileSync4(p.fulltextMd, "utf8");
2676
+ const next = withCorrectedBlock(md, correctedSentences(r));
2677
+ if (next !== md) writeAtomic(p.fulltextMd, next);
2678
+ }
2679
+ function applyFulltext(p, m, hit, converter) {
2680
+ const raw = p.fulltextRaw(hit.format);
2681
+ writeAtomic(raw, hit.raw);
2682
+ writeAtomic(p.fulltextMd, hit.md);
2683
+ syncCorrectedBlock(p);
2684
+ const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
2685
+ const own = declarationsFromMd(hit.md);
2686
+ const next = {
2687
+ ...m,
2688
+ level: "full_text",
2689
+ version: hit.version,
2690
+ files,
2691
+ retrieved_by: hit.tier,
2692
+ retrieved_date: today(),
2693
+ converted_by: converter,
2694
+ coi: m.coi ?? own.coi,
2695
+ funding: m.funding.length ? m.funding : own.funding
2670
2696
  };
2697
+ writeMeta(p, next);
2698
+ return next;
2671
2699
  }
2672
-
2673
- // src/config.ts
2674
- var VARIABLES = [
2675
- "AI4L_LIBRARY_DIR",
2676
- "AI4L_LIBRARY_ENV",
2677
- "AI4L_LIBRARY_TIERS",
2678
- "AI4L_LIBRARY_CONTACT_EMAIL",
2679
- "AI4L_LIBRARY_ELSEVIER_API_KEY",
2680
- "AI4L_LIBRARY_WILEY_TDM_TOKEN",
2681
- "AI4L_LIBRARY_CORE_API_KEY",
2682
- "AI4L_LIBRARY_PROXY_URL",
2683
- "AI4L_LIBRARY_PROXY_HEADERS",
2684
- "AI4L_LIBRARY_BROWSER_1",
2685
- "AI4L_LIBRARY_BROWSER_2",
2686
- "AI4L_LIBRARY_BROWSER_3",
2687
- "AI4L_LIBRARY_BROWSER_HEADLESS",
2688
- "AI4L_LIBRARY_VERIFY_URL",
2689
- "AI4L_LIBRARY_VERIFY_API_KEY",
2690
- "AI4L_LIBRARY_VERIFY_MODEL",
2691
- "AI4L_LIBRARY_VERIFY_THRESHOLD",
2692
- "AI4L_LIBRARY_VERIFY_TIMEOUT_SECS",
2693
- "AI4L_LIBRARY_RETRY_AFTER_DAYS",
2694
- "AI4L_LIBRARY_RETRY_BLOCKED",
2695
- "AI4L_LIBRARY_REFRESH_AFTER_DAYS",
2696
- "NCBI_API_KEY",
2697
- "BRIGHTDATA_API_TOKEN",
2698
- "BRIGHTDATA_ZONE"
2699
- ];
2700
- var sources = (env = process.env) => Object.fromEntries(VARIABLES.map((v) => [v, sourceOf(v, env)]));
2701
- function settings(env = process.env) {
2702
- return {
2703
- retry_after_days: num("AI4L_LIBRARY_RETRY_AFTER_DAYS", 1, env),
2704
- retry_blocked: bool("AI4L_LIBRARY_RETRY_BLOCKED", false, env),
2705
- refresh_after_days: num("AI4L_LIBRARY_REFRESH_AFTER_DAYS", 30, env)
2700
+ function recordAttempts(p, m, labels) {
2701
+ if (!labels.length) return m;
2702
+ const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
2703
+ const replaced = new Set(labels.map(tierOf));
2704
+ const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
2705
+ writeMeta(p, next);
2706
+ return next;
2707
+ }
2708
+ function writeLabelEntry(p, s, xml, converter) {
2709
+ mkdirSync3(p.entryDir, { recursive: true });
2710
+ const raw = p.fulltextRaw("xml");
2711
+ writeAtomic(raw, xml);
2712
+ writeAtomic(p.fulltextMd, renderLabelMd(s));
2713
+ const meta = {
2714
+ pmid: "",
2715
+ setid: s.setid,
2716
+ nctid: null,
2717
+ publications: [],
2718
+ pmcid: null,
2719
+ doi: null,
2720
+ title: s.title,
2721
+ first_author: s.labeler,
2722
+ year: s.effective?.slice(0, 4) ?? null,
2723
+ publication_types: ["Drug Label"],
2724
+ retracted: false,
2725
+ corrections: [],
2726
+ coi: null,
2727
+ funding: [],
2728
+ has_abstract: false,
2729
+ level: "full_text",
2730
+ version: s.version ? `spl ${s.version}` : null,
2731
+ files: [basename(raw), basename(p.fulltextMd)],
2732
+ retrieved_by: "dailymed",
2733
+ retrieved_date: today(),
2734
+ converted_by: converter,
2735
+ pubmed_checked: today(),
2736
+ attempts: []
2706
2737
  };
2738
+ writeMeta(p, meta);
2739
+ return meta;
2707
2740
  }
2708
- function proxyHeaders(spec) {
2709
- const out = {};
2710
- for (const part of spec.split(";")) {
2711
- const i = part.indexOf(":");
2712
- if (i < 0) continue;
2713
- const name = part.slice(0, i).trim(), value = part.slice(i + 1).trim();
2714
- if (name && value) out[name] = value;
2715
- }
2716
- return out;
2741
+ function writeStudyEntry(p, s, json2, converter) {
2742
+ mkdirSync3(p.entryDir, { recursive: true });
2743
+ const raw = p.fulltextRaw("json");
2744
+ writeAtomic(raw, json2);
2745
+ writeAtomic(p.fulltextMd, renderStudyMd(s));
2746
+ const meta = {
2747
+ pmid: "",
2748
+ setid: null,
2749
+ nctid: s.nctid,
2750
+ publications: s.publications,
2751
+ pmcid: null,
2752
+ doi: null,
2753
+ title: s.title,
2754
+ first_author: s.sponsor,
2755
+ year: s.firstPosted?.slice(0, 4) ?? null,
2756
+ publication_types: ["Trial Registration"],
2757
+ retracted: false,
2758
+ corrections: [],
2759
+ coi: null,
2760
+ funding: [],
2761
+ has_abstract: false,
2762
+ level: "full_text",
2763
+ version: s.lastUpdate,
2764
+ files: [basename(raw), basename(p.fulltextMd)],
2765
+ retrieved_by: "ctgov",
2766
+ retrieved_date: today(),
2767
+ converted_by: converter,
2768
+ pubmed_checked: today(),
2769
+ attempts: []
2770
+ };
2771
+ writeMeta(p, meta);
2772
+ return meta;
2717
2773
  }
2718
- function resolveTiers(env = process.env) {
2719
- const problems = [];
2720
- const available = /* @__PURE__ */ new Map();
2721
- const tokens = { elsevier: str("AI4L_LIBRARY_ELSEVIER_API_KEY", env) || void 0, wiley: str("AI4L_LIBRARY_WILEY_TDM_TOKEN", env) || void 0 };
2722
- for (const t of BUILTIN) available.set(t.name, t.make({ tokens, api_key: str("AI4L_LIBRARY_CORE_API_KEY", env) || void 0 }));
2723
- const proxyUrl = str("AI4L_LIBRARY_PROXY_URL", env);
2724
- if (proxyUrl) available.set("proxy", urlTier("proxy", proxyUrl, proxyHeaders(str("AI4L_LIBRARY_PROXY_HEADERS", env))));
2725
- const listed = str("AI4L_LIBRARY_TIERS", env);
2726
- const order2 = listed ? listed.split(",").map((s) => s.trim()).filter(Boolean) : [...available.keys()];
2727
- const tiers = [], seen = /* @__PURE__ */ new Set();
2728
- for (const name of order2) {
2729
- if (seen.has(name)) continue;
2730
- seen.add(name);
2731
- const t = available.get(name);
2732
- if (t) tiers.push(t);
2733
- else if (name === "proxy") problems.push('AI4L_LIBRARY_TIERS: "proxy" listed, but AI4L_LIBRARY_PROXY_URL is unset');
2734
- else problems.push(`AI4L_LIBRARY_TIERS: unknown tier "${name}" (built-in: ${BUILTIN.map((t2) => t2.name).join(", ")}; proxy when AI4L_LIBRARY_PROXY_URL is set)`);
2735
- }
2736
- const all = [...tiers.map((t) => ({ name: t.name, enabled: true })), ...[...available.keys()].filter((n) => !seen.has(n)).map((name) => ({ name, enabled: false }))];
2737
- return { tiers, all, source: sourceOf("AI4L_LIBRARY_TIERS", env), problems };
2774
+ function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
2775
+ mkdirSync3(p.entryDir, { recursive: true });
2776
+ const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
2777
+ writeAtomic(raw, json2);
2778
+ writeAtomic(p.md, renderAbstractMd(r));
2779
+ const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
2780
+ writeMeta(p, meta);
2781
+ return meta;
2738
2782
  }
2739
2783
 
2740
2784
  // src/cascade.ts
@@ -2769,7 +2813,7 @@ async function fulltextCascade(p, attempts, onEvent, tiers = resolveTiers().tier
2769
2813
  for (const tier of tiers) {
2770
2814
  if (!tier.always && shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked)) continue;
2771
2815
  const applies = tier.applies(p);
2772
- const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }) };
2816
+ const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }), onAlert: (message) => onEvent?.({ ok: "alert", tier: tier.name, message }) };
2773
2817
  const r = applies === true ? await tier.run(p, ctx) : { ...miss(tier.name, applies), skipped: true };
2774
2818
  onEvent?.(r);
2775
2819
  if (r.ok) return { hit: r, misses };
@@ -2809,8 +2853,8 @@ function parseId(raw) {
2809
2853
  }
2810
2854
 
2811
2855
  // src/info.ts
2812
- import { existsSync as existsSync7, readdirSync as readdirSync3, readFileSync as readFileSync6, statSync } from "fs";
2813
- import { dirname as dirname2, join as join6 } from "path";
2856
+ import { existsSync as existsSync7, readdirSync as readdirSync3, readFileSync as readFileSync6, statSync as statSync2 } from "fs";
2857
+ import { dirname as dirname2, join as join7 } from "path";
2814
2858
 
2815
2859
  // src/verify.ts
2816
2860
  import { existsSync as existsSync6, readFileSync as readFileSync5 } from "fs";
@@ -2960,8 +3004,8 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
2960
3004
  function* metaFiles(dir) {
2961
3005
  if (!existsSync7(dir)) return;
2962
3006
  for (const name of readdirSync3(dir)) {
2963
- const p = join6(dir, name);
2964
- if (statSync(p).isDirectory()) yield* metaFiles(p);
3007
+ const p = join7(dir, name);
3008
+ if (statSync2(p).isDirectory()) yield* metaFiles(p);
2965
3009
  else if (name.endsWith("-meta.md")) yield p;
2966
3010
  }
2967
3011
  }
@@ -3000,8 +3044,8 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
3000
3044
  };
3001
3045
  const ids = [];
3002
3046
  for (const type of existsSync7(libraryDir) ? readdirSync3(libraryDir) : []) {
3003
- const typeDir = join6(libraryDir, type);
3004
- if (!statSync(typeDir).isDirectory()) continue;
3047
+ const typeDir = join7(libraryDir, type);
3048
+ if (!statSync2(typeDir).isDirectory()) continue;
3005
3049
  for (const meta of metaFiles(typeDir)) {
3006
3050
  const md = readFileSync6(meta, "utf8");
3007
3051
  const raw = field(md, "level") || "unknown";
@@ -3015,7 +3059,7 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
3015
3059
  const dir = dirname2(meta);
3016
3060
  for (const f of readdirSync3(dir)) {
3017
3061
  try {
3018
- const st = statSync(join6(dir, f));
3062
+ const st = statSync2(join7(dir, f));
3019
3063
  if (st.isFile()) info.size_bytes += st.size;
3020
3064
  } catch {
3021
3065
  }
@@ -3143,45 +3187,6 @@ function search(md, query, maxHits) {
3143
3187
  }
3144
3188
  var escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
3145
3189
 
3146
- // src/log.ts
3147
- import { appendFileSync, mkdirSync as mkdirSync3, renameSync as renameSync2, statSync as statSync2 } from "fs";
3148
- import { hostname } from "os";
3149
- import { join as join7 } from "path";
3150
- var HOST = hostname();
3151
- var LOG_FILE = join7(AI4L_LIBRARY_DIR, "library.log");
3152
- function stamp() {
3153
- const d = /* @__PURE__ */ new Date();
3154
- const p = (n) => String(n).padStart(2, "0");
3155
- return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
3156
- }
3157
- var LOG_MAX_BYTES = 50 * 1024 * 1024;
3158
- function rotateIfLarge(file, max = LOG_MAX_BYTES) {
3159
- let size = 0;
3160
- try {
3161
- size = statSync2(file).size;
3162
- } catch {
3163
- return false;
3164
- }
3165
- if (size <= max) return false;
3166
- try {
3167
- renameSync2(file, `${file}.1`);
3168
- return true;
3169
- } catch {
3170
- return false;
3171
- }
3172
- }
3173
- function log(tool, id, message) {
3174
- const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
3175
- const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
3176
- `;
3177
- try {
3178
- mkdirSync3(AI4L_LIBRARY_DIR, { recursive: true });
3179
- rotateIfLarge(LOG_FILE);
3180
- appendFileSync(LOG_FILE, line);
3181
- } catch {
3182
- }
3183
- }
3184
-
3185
3190
  // src/doi.ts
3186
3191
  var NCBI_ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi";
3187
3192
  async function pmidForDoi(doi) {
@@ -3371,12 +3376,17 @@ function noticeTag(m) {
3371
3376
  ].filter(Boolean);
3372
3377
  return parts.length ? " " + parts.join(" ") : "";
3373
3378
  }
3379
+ var modelFailure = (error, cfg) => error.startsWith("no verify model configured") || !!cfg && error.startsWith(`${cfg.model}:`);
3374
3380
  function registerTools(server2) {
3375
3381
  async function completeFulltext(paths, meta, requested) {
3376
3382
  const paper = { title: meta.title, pmid: meta.pmid, pmcid: meta.pmcid, doi: meta.doi, headLine: readHeadLine(paths) };
3377
3383
  const tried = /* @__PURE__ */ new Set();
3378
3384
  let ran = false;
3379
3385
  const { hit } = await fulltextCascade(paper, meta.attempts, (e) => {
3386
+ if (e.ok === "alert") {
3387
+ log("library_add", requested, `!! ${e.message}`);
3388
+ return;
3389
+ }
3380
3390
  if (e.ok === "try") tried.add(e.tier);
3381
3391
  if (e.ok === false && e.ephemeral) return;
3382
3392
  ran = true;
@@ -3688,7 +3698,7 @@ function registerTools(server2) {
3688
3698
  const cfg = verifyConfig();
3689
3699
  const r = await verifyClaim(claim, id, cfg, cfg ? decider(cfg) : null);
3690
3700
  const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
3691
- log("library_verify", r.id, r.error ? `${secs} verdict=${"error".padEnd(5)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(5)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
3701
+ log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(5)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(5)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
3692
3702
  return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
3693
3703
  }
3694
3704
  );
@@ -3724,11 +3734,11 @@ function registerTools(server2) {
3724
3734
  }
3725
3735
 
3726
3736
  // src/index.ts
3727
- loadDotenv();
3728
3737
  var server = new McpServer(
3729
3738
  { name: pkg.name, version: pkg.version },
3730
3739
  { instructions: INSTRUCTIONS }
3731
3740
  );
3732
3741
  registerTools(server);
3742
+ for (const problem of resolveTiers().problems) log("startup", null, `!! ${problem}`);
3733
3743
  var transport = new StdioServerTransport();
3734
3744
  await server.connect(transport);