fh-library 0.2.5 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -11,10 +11,10 @@ function dotenvFile(env = process.env) {
11
11
  return join(resolve(lib), ".env");
12
12
  }
13
13
  var loadedFromDotenv = /* @__PURE__ */ new Set();
14
- function loadDotenv(file = dotenvFile()) {
15
- if (!existsSync(file)) return [];
14
+ function loadDotenv(file2 = dotenvFile()) {
15
+ if (!existsSync(file2)) return [];
16
16
  const loaded = [];
17
- for (const raw of readFileSync(file, "utf8").split(/\r?\n/)) {
17
+ for (const raw of readFileSync(file2, "utf8").split(/\r?\n/)) {
18
18
  const line = raw.trim();
19
19
  if (!line || line.startsWith("#")) continue;
20
20
  const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
@@ -60,6 +60,11 @@ var contactEmail = (env = process.env) => str("FH_LIBRARY_CONTACT_EMAIL", env) |
60
60
  // src/http.ts
61
61
  var TIMEOUT_MS = 3e4;
62
62
  var CHROME_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
63
+ var BROWSER_HEADERS = {
64
+ "User-Agent": CHROME_UA,
65
+ "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
66
+ "Accept-Language": "en-US,en;q=0.9"
67
+ };
63
68
  var LIBRARY_UA = `fh-library (+https://github.com/forever-healthy/fh-library; ${contactEmail()})`;
64
69
  function httpReason(status, body) {
65
70
  const m = body.length < 4e3 && /<statusCode>([A-Z_]+)<\/statusCode>|"statusCode"\s*:\s*"([A-Z_]+)"/.exec(body);
@@ -655,20 +660,20 @@ function headingOf(line) {
655
660
  if (bare2.length > 60 || !/^[A-Za-z][A-Za-z &'\u2019]*$/.test(bare2)) return null;
656
661
  return BY_SQUEEZED.get(squeeze(bare2)) ?? null;
657
662
  }
658
- async function pdfToMarkdown(bytes, title, headLine2, stats) {
663
+ async function pdfToMarkdown(bytes, title, headLine2, stats2) {
659
664
  const pdf = await getDocumentProxy(new Uint8Array(bytes), { fontExtraProperties: true });
660
665
  const text3 = await extractPages(pdf);
661
- if (stats) {
662
- stats.pages = pdf.numPages;
663
- stats.chars = text3.replace(/\s+/g, " ").trim().length;
666
+ if (stats2) {
667
+ stats2.pages = pdf.numPages;
668
+ stats2.chars = text3.replace(/\s+/g, " ").trim().length;
664
669
  }
665
670
  const lines = text3.split(/\r?\n/).map((l) => l.replace(/\s+/g, " ").trim());
666
671
  const out = [`# ${title}`, "", headLine2, ""];
667
- let para2 = [];
672
+ let para3 = [];
668
673
  const flush = () => {
669
- if (para2.length) {
670
- out.push(para2.join(" "), "");
671
- para2 = [];
674
+ if (para3.length) {
675
+ out.push(para3.join(" "), "");
676
+ para3 = [];
672
677
  }
673
678
  };
674
679
  for (const line of lines) {
@@ -682,7 +687,7 @@ async function pdfToMarkdown(bytes, title, headLine2, stats) {
682
687
  out.push(`## ${h}`, "");
683
688
  continue;
684
689
  }
685
- para2.push(line);
690
+ para3.push(line);
686
691
  }
687
692
  flush();
688
693
  return fixGlyphs(out.join("\n")).replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
@@ -875,9 +880,9 @@ function dehyphenate(text3) {
875
880
  return text3.replace(/([A-Za-z]+)\u0000\n([A-Za-z]+)/g, (m, x, y) => {
876
881
  if (!/^[a-z]/.test(y)) return `${x}-${y}`;
877
882
  if (SUSPENDED.has(y.toLowerCase())) return `${x}- ${y}`;
878
- const hyph = `${x}-${y}`.toLowerCase(), joined = `${x}${y}`.toLowerCase();
883
+ const hyph = `${x}-${y}`.toLowerCase(), joined3 = `${x}${y}`.toLowerCase();
879
884
  if (seen(hyph)) return `${x}-${y}`;
880
- if (seen(joined)) return `${x}${y}`;
885
+ if (seen(joined3)) return `${x}${y}`;
881
886
  return COMPOUND_PREFIX.has(x.toLowerCase()) ? `${x}-${y}` : `${x}${y}`;
882
887
  }).replace(/\u0000\n/g, "-\n").replace(/\u0000/g, "-");
883
888
  }
@@ -1000,6 +1005,8 @@ var NAV_TIMEOUT_MS = 2e4;
1000
1005
  var CHALLENGE_WAIT_MS = 8e3;
1001
1006
  var DOWNLOAD_WAIT_MS = 1e4;
1002
1007
  var IDLE_CLOSE_MS = 12e4;
1008
+ var SETTLE_MS = 8e3;
1009
+ var SETTLE_WORDS = 100;
1003
1010
  var ctxs = /* @__PURE__ */ new Map();
1004
1011
  var idleTimer = null;
1005
1012
  var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
@@ -1141,6 +1148,12 @@ async function browserFetch(url, endpoint, cfg = browserConfig()) {
1141
1148
  if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
1142
1149
  return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
1143
1150
  }
1151
+ const words2 = () => page.evaluate(() => (document.body?.innerText ?? "").split(/\s+/).filter(Boolean).length).catch(() => 0);
1152
+ if (await words2() < SETTLE_WORDS) {
1153
+ await page.waitForLoadState("networkidle", { timeout: SETTLE_MS }).catch(() => void 0);
1154
+ const t2 = Date.now();
1155
+ while (Date.now() - t2 < SETTLE_MS && await words2() < SETTLE_WORDS) await new Promise((r) => setTimeout(r, 1e3));
1156
+ }
1144
1157
  const html = await page.content();
1145
1158
  return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
1146
1159
  } finally {
@@ -1150,14 +1163,39 @@ async function browserFetch(url, endpoint, cfg = browserConfig()) {
1150
1163
  }
1151
1164
 
1152
1165
  // src/unlock.ts
1166
+ import { request as httpsRequest } from "https";
1153
1167
  var ENDPOINT = "https://api.brightdata.com/request";
1154
1168
  var TIMEOUT_MS2 = 6e4;
1169
+ function post(url, init) {
1170
+ return new Promise((resolve3, reject) => {
1171
+ const req = httpsRequest(url, { method: "POST", headers: { ...init.headers, "Content-Length": String(Buffer.byteLength(init.body)) }, signal: init.signal }, (res) => {
1172
+ const chunks = [];
1173
+ res.on("data", (c) => chunks.push(c));
1174
+ res.on("error", reject);
1175
+ res.on("end", () => {
1176
+ const body = Buffer.concat(chunks);
1177
+ const status = res.statusCode ?? 0;
1178
+ resolve3({
1179
+ ok: status >= 200 && status < 300,
1180
+ status,
1181
+ headers: { get: (name) => {
1182
+ const v = res.headers[name.toLowerCase()];
1183
+ return v == null ? null : Array.isArray(v) ? v.join(", ") : v;
1184
+ } },
1185
+ arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength)
1186
+ });
1187
+ });
1188
+ });
1189
+ req.on("error", (e) => reject(e.name === "AbortError" || e.name === "TimeoutError" ? new Error("timeout") : e));
1190
+ req.end(init.body);
1191
+ });
1192
+ }
1155
1193
  var unlockerConfigured = () => !!process.env.FH_LIBRARY_BRIGHTDATA_API_TOKEN?.trim();
1156
- async function unlock(url, fetchImpl = fetch) {
1194
+ async function unlock(url, postImpl = post) {
1157
1195
  const token = process.env.FH_LIBRARY_BRIGHTDATA_API_TOKEN?.trim();
1158
1196
  if (!token) throw new Error("unlocker not configured");
1159
1197
  const zone = process.env.FH_LIBRARY_BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
1160
- const res = await fetchImpl(ENDPOINT, {
1198
+ const res = await postImpl(ENDPOINT, {
1161
1199
  method: "POST",
1162
1200
  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
1163
1201
  body: JSON.stringify({ zone, url, format: "raw" }),
@@ -1264,13 +1302,13 @@ async function convertBytes(p, bytes, contentType = "") {
1264
1302
  let md;
1265
1303
  if (looksLikePdf(bytes)) {
1266
1304
  format = "pdf";
1267
- const stats = {};
1305
+ const stats2 = {};
1268
1306
  try {
1269
- md = await pdfToMarkdown(bytes, p.title, p.headLine, stats);
1307
+ md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
1270
1308
  } catch (e) {
1271
1309
  return { ok: false, reason: `pdf: ${errMsg(e)}` };
1272
1310
  }
1273
- if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
1311
+ if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
1274
1312
  } else {
1275
1313
  const text3 = bytes.toString("utf8");
1276
1314
  if (isWallPage(text3)) return { ok: false, reason: "wall page" };
@@ -1462,15 +1500,15 @@ function inboxTier(dir = INBOX_DIR) {
1462
1500
  always: true,
1463
1501
  applies: (p) => p.pmid ? true : "no pmid",
1464
1502
  async run(p, ctx) {
1465
- const file = findInboxFile(dir, p.pmid);
1466
- if (!file) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
1467
- const name = file.slice(file.lastIndexOf("/") + 1);
1503
+ const file2 = findInboxFile(dir, p.pmid);
1504
+ if (!file2) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
1505
+ const name = file2.slice(file2.lastIndexOf("/") + 1);
1468
1506
  const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
1469
- const bytes = readFileSync3(file);
1507
+ const bytes = readFileSync3(file2);
1470
1508
  const r = await convertBytes(p, bytes, EXT[ext]);
1471
1509
  if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
1472
1510
  ctx.onTry?.(`${name} ok, removed from the inbox`);
1473
- unlinkSync(file);
1511
+ unlinkSync(file2);
1474
1512
  return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
1475
1513
  }
1476
1514
  };
@@ -1743,24 +1781,24 @@ function stamp() {
1743
1781
  return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
1744
1782
  }
1745
1783
  var LOG_MAX_BYTES = 50 * 1024 * 1024;
1746
- function rotateIfLarge(file, max = LOG_MAX_BYTES) {
1784
+ function rotateIfLarge(file2, max = LOG_MAX_BYTES) {
1747
1785
  let size = 0;
1748
1786
  try {
1749
- size = statSync(file).size;
1787
+ size = statSync(file2).size;
1750
1788
  } catch {
1751
1789
  return false;
1752
1790
  }
1753
1791
  if (size <= max) return false;
1754
1792
  try {
1755
- renameSync(file, `${file}.1`);
1793
+ renameSync(file2, `${file2}.1`);
1756
1794
  return true;
1757
1795
  } catch {
1758
1796
  return false;
1759
1797
  }
1760
1798
  }
1761
1799
  function log(tool, id, message) {
1762
- const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
1763
- const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
1800
+ const clean2 = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
1801
+ const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean2}
1764
1802
  `;
1765
1803
  try {
1766
1804
  mkdirSync2(FH_LIBRARY_DIR, { recursive: true });
@@ -1770,9 +1808,9 @@ function log(tool, id, message) {
1770
1808
  }
1771
1809
  }
1772
1810
 
1773
- // src/tools.ts
1774
- import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
1775
- import { z } from "zod";
1811
+ // src/stats.ts
1812
+ import { mkdirSync as mkdirSync4, readdirSync as readdirSync3, readFileSync as readFileSync5, statSync as statSync2, unlinkSync as unlinkSync2 } from "fs";
1813
+ import { join as join7 } from "path";
1776
1814
 
1777
1815
  // src/entry.ts
1778
1816
  import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readdirSync as readdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, rmSync, writeFileSync } from "fs";
@@ -2591,10 +2629,10 @@ function readMeta(p) {
2591
2629
  attempts: splitAttempts(fm.attempts)
2592
2630
  };
2593
2631
  }
2594
- function writeAtomic(file, data) {
2595
- const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
2632
+ function writeAtomic(file2, data) {
2633
+ const tmp = `${file2}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
2596
2634
  writeFileSync(tmp, data);
2597
- renameSync2(tmp, file);
2635
+ renameSync2(tmp, file2);
2598
2636
  }
2599
2637
  function writeMeta(p, m) {
2600
2638
  writeAtomic(p.meta, renderMeta(m));
@@ -2873,6 +2911,67 @@ function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", record
2873
2911
  return meta;
2874
2912
  }
2875
2913
 
2914
+ // src/stats.ts
2915
+ var HEARTBEAT_MS = 5 * 6e4;
2916
+ var STALE_MS = 20 * 6e4;
2917
+ var stats = null;
2918
+ var file = null;
2919
+ function startStats(pkg2, libraryDir = FH_LIBRARY_DIR) {
2920
+ stats = { host: HOST, pid: process.pid, package: pkg2.name, version: pkg2.version, started: (/* @__PURE__ */ new Date()).toISOString(), verify: { hits: 0, misses: 0 }, fetch: { hits: 0, misses: 0 }, search: { hits: 0, misses: 0 } };
2921
+ file = join7(libraryDir, ".servers", `${HOST}-${process.pid}.json`);
2922
+ sweep(join7(libraryDir, ".servers"));
2923
+ publish();
2924
+ setInterval(publish, HEARTBEAT_MS).unref();
2925
+ const stop = () => {
2926
+ try {
2927
+ if (file) unlinkSync2(file);
2928
+ } catch {
2929
+ }
2930
+ };
2931
+ process.on("exit", stop);
2932
+ for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) process.on(sig, () => process.exit(0));
2933
+ return stats;
2934
+ }
2935
+ function sweep(dir, staleMs = STALE_MS) {
2936
+ let n = 0;
2937
+ let names;
2938
+ try {
2939
+ names = readdirSync3(dir);
2940
+ } catch {
2941
+ return 0;
2942
+ }
2943
+ for (const name of names) {
2944
+ if (!name.endsWith(".json")) continue;
2945
+ const p = join7(dir, name);
2946
+ try {
2947
+ if (Date.now() - statSync2(p).mtimeMs > staleMs) {
2948
+ unlinkSync2(p);
2949
+ n++;
2950
+ }
2951
+ } catch {
2952
+ }
2953
+ }
2954
+ return n;
2955
+ }
2956
+ function bump(kind, hit) {
2957
+ if (!stats) return;
2958
+ stats[kind][hit ? "hits" : "misses"]++;
2959
+ publish();
2960
+ }
2961
+ var currentStats = () => stats;
2962
+ function publish() {
2963
+ if (!stats || !file) return;
2964
+ try {
2965
+ mkdirSync4(join7(file, ".."), { recursive: true });
2966
+ writeAtomic(file, JSON.stringify(stats, null, 2) + "\n");
2967
+ } catch {
2968
+ }
2969
+ }
2970
+
2971
+ // src/tools.ts
2972
+ import { existsSync as existsSync11, readFileSync as readFileSync11 } from "fs";
2973
+ import { z } from "zod";
2974
+
2876
2975
  // src/cascade.ts
2877
2976
  var PERMANENT = /^(no pmcid|no doi|no pmid|no pmid or doi|no body|closed|no oa copy|no free copy|not in core|no full text|xml without body|template has no placeholder|title missing|title not in text|no methods\/results|no article body|scanned pdf|not a pdf|only copies already tried|only pmc copies|nothing located|unknown content|no text-mining links|needs [a-z/]+ token)/i;
2878
2977
  var BLOCKED = /^(challenge not cleared|empty response)/i;
@@ -2945,11 +3044,11 @@ function parseId(raw) {
2945
3044
  }
2946
3045
 
2947
3046
  // src/info.ts
2948
- import { existsSync as existsSync7, readdirSync as readdirSync3, readFileSync as readFileSync6, statSync as statSync2 } from "fs";
2949
- import { dirname as dirname2, join as join7 } from "path";
3047
+ import { existsSync as existsSync7, readdirSync as readdirSync4, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
3048
+ import { dirname as dirname2, join as join8 } from "path";
2950
3049
 
2951
3050
  // src/verify.ts
2952
- import { existsSync as existsSync6, readFileSync as readFileSync5 } from "fs";
3051
+ import { existsSync as existsSync6, readFileSync as readFileSync6 } from "fs";
2953
3052
  var DEFAULT_MODEL = "pplx-decider-flash-v1.2";
2954
3053
  var QUESTION = "Does the paper state this, in substance? Ignore section labels and definitions of terms in parentheses. Every fact and number of the statement must be stated in the paper or follow from its numbers; a rounded number counts. If any part of the statement is absent from the paper or contradicted by it, answer false.";
2955
3054
  function verifyConfig(env = process.env) {
@@ -3013,7 +3112,8 @@ function citationGroups(claim) {
3013
3112
  function fragment(claim) {
3014
3113
  const body = claim.trim().replace(/^[-*+]\s+/, "").replace(/^\*\*[^*]+:\*\*\s*/, "").replace(/^[("'\u201c]+/, "");
3015
3114
  const first = /^(\S+)/.exec(body)?.[1] ?? "";
3016
- if (/^(and|or|but|nor|which|whereas|while)$/i.test(first.replace(/[,;:]$/, ""))) return `claim is a fragment: starts with "${first}" \u2014 send a complete statement with its subject`;
3115
+ const oddsRatio = first === "OR" && /^OR\s+(\d|=|\(|of\b|for\b)/.test(body);
3116
+ if (!oddsRatio && /^(and|or|but|nor|which|whereas|while)$/i.test(first.replace(/[,;:]$/, ""))) return `claim is a fragment: starts with "${first}" \u2014 send a complete statement with its subject`;
3017
3117
  if (/^[a-z]/.test(body)) return `claim is a fragment: starts with "${first}" in lowercase \u2014 send a complete statement with its subject`;
3018
3118
  return null;
3019
3119
  }
@@ -3051,9 +3151,9 @@ async function readPaper(raw, libraryDir = FH_LIBRARY_DIR) {
3051
3151
  const paths = entryPaths(libraryDir, p.type, p.id);
3052
3152
  if (!entryExists(paths)) return { id: p.requested, error: "not in library" };
3053
3153
  const full = existsSync6(paths.fulltextMd);
3054
- if (full) return { id: p.requested, basis: "full_text", text: readFileSync5(paths.fulltextMd, "utf8") };
3154
+ if (full) return { id: p.requested, basis: "full_text", text: readFileSync6(paths.fulltextMd, "utf8") };
3055
3155
  const meta = readMeta(paths);
3056
- return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync5(paths.md, "utf8") };
3156
+ return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync6(paths.md, "utf8") };
3057
3157
  }
3058
3158
  async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
3059
3159
  const out = { claim, id, verdict: "error", reason: null, score: null, basis: null, windows: 0, glyphs_unresolved: 0, superseded: null, model: cfg?.model ?? null };
@@ -3095,9 +3195,9 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
3095
3195
  // src/info.ts
3096
3196
  function* metaFiles(dir) {
3097
3197
  if (!existsSync7(dir)) return;
3098
- for (const name of readdirSync3(dir)) {
3099
- const p = join7(dir, name);
3100
- if (statSync2(p).isDirectory()) yield* metaFiles(p);
3198
+ for (const name of readdirSync4(dir)) {
3199
+ const p = join8(dir, name);
3200
+ if (statSync3(p).isDirectory()) yield* metaFiles(p);
3101
3201
  else if (name.endsWith("-meta.md")) yield p;
3102
3202
  }
3103
3203
  }
@@ -3135,11 +3235,11 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
3135
3235
  updates: 0
3136
3236
  };
3137
3237
  const ids = [];
3138
- for (const type of existsSync7(libraryDir) ? readdirSync3(libraryDir) : []) {
3139
- const typeDir = join7(libraryDir, type);
3140
- if (!statSync2(typeDir).isDirectory()) continue;
3238
+ for (const type of existsSync7(libraryDir) ? readdirSync4(libraryDir) : []) {
3239
+ const typeDir = join8(libraryDir, type);
3240
+ if (!statSync3(typeDir).isDirectory()) continue;
3141
3241
  for (const meta of metaFiles(typeDir)) {
3142
- const md = readFileSync6(meta, "utf8");
3242
+ const md = readFileSync7(meta, "utf8");
3143
3243
  const raw = field(md, "level") || "unknown";
3144
3244
  const level = raw === "abstract" && field(md, "has_abstract") === "no" ? "title" : raw;
3145
3245
  info.entries++;
@@ -3149,9 +3249,9 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
3149
3249
  t.entries++;
3150
3250
  if (level === "full_text" || level === "abstract" || level === "title") t[level]++;
3151
3251
  const dir = dirname2(meta);
3152
- for (const f of readdirSync3(dir)) {
3252
+ for (const f of readdirSync4(dir)) {
3153
3253
  try {
3154
- const st = statSync2(join7(dir, f));
3254
+ const st = statSync3(join8(dir, f));
3155
3255
  if (st.isFile()) info.size_bytes += st.size;
3156
3256
  } catch {
3157
3257
  }
@@ -3180,7 +3280,7 @@ function noticeTypes(md, metaPath) {
3180
3280
  return out;
3181
3281
  }
3182
3282
  try {
3183
- const xml = readFileSync6(metaPath.replace(/-meta\.md$/, "-abstract.xml"), "utf8");
3283
+ const xml = readFileSync7(metaPath.replace(/-meta\.md$/, "-abstract.xml"), "utf8");
3184
3284
  for (const m of xml.matchAll(/<CommentsCorrections RefType="([A-Za-z]+)"/g)) if (XML_TYPES[m[1]]) out.add(XML_TYPES[m[1]]);
3185
3285
  } catch {
3186
3286
  }
@@ -3329,10 +3429,10 @@ function paragraphs(md, scope) {
3329
3429
  return h;
3330
3430
  };
3331
3431
  const out = [];
3332
- let para2 = [], paraStart = 0, header = null;
3432
+ let para3 = [], paraStart = 0, header = null;
3333
3433
  const flush = () => {
3334
- const text3 = para2.join("\n").trim();
3335
- para2 = [];
3434
+ const text3 = para3.join("\n").trim();
3435
+ para3 = [];
3336
3436
  if (text3) out.push({ section: sectionAt(paraStart), text: text3 });
3337
3437
  };
3338
3438
  for (let i = from; i < to; i++) {
@@ -3353,8 +3453,8 @@ function paragraphs(md, scope) {
3353
3453
  continue;
3354
3454
  }
3355
3455
  header = null;
3356
- if (!para2.length) paraStart = i;
3357
- para2.push(l);
3456
+ if (!para3.length) paraStart = i;
3457
+ para3.push(l);
3358
3458
  }
3359
3459
  flush();
3360
3460
  return out;
@@ -3557,15 +3657,567 @@ function parseCsl(json2) {
3557
3657
  };
3558
3658
  }
3559
3659
 
3660
+ // src/urlfetch.ts
3661
+ import { createHash as createHash2 } from "crypto";
3662
+ import { existsSync as existsSync8, mkdirSync as mkdirSync5, readFileSync as readFileSync8 } from "fs";
3663
+ import { join as join9 } from "path";
3664
+ var TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src)$/i;
3665
+ function normalizeUrl(raw) {
3666
+ let u;
3667
+ try {
3668
+ u = new URL(raw.trim());
3669
+ } catch {
3670
+ return null;
3671
+ }
3672
+ if (u.protocol !== "http:" && u.protocol !== "https:") return null;
3673
+ u.hash = "";
3674
+ u.hostname = u.hostname.toLowerCase();
3675
+ if ([...u.searchParams.keys()].some((k) => TRACKING.test(k))) {
3676
+ for (const k of [...u.searchParams.keys()]) if (TRACKING.test(k)) u.searchParams.delete(k);
3677
+ if (![...u.searchParams.keys()].length) u.search = "";
3678
+ }
3679
+ return u.toString();
3680
+ }
3681
+ var urlHash = (normalized2) => createHash2("sha1").update(normalized2).digest("hex").slice(0, 16);
3682
+ function urlPaths(libraryDir, normalized2) {
3683
+ const hash = urlHash(normalized2);
3684
+ const entry = `url-${hash}`;
3685
+ const entryDir = join9(libraryDir, "url", hash.slice(0, 2), hash.slice(0, 4), hash);
3686
+ return { hash, entry, entryDir, meta: join9(entryDir, `${entry}-meta.md`), md: join9(entryDir, `${entry}-page.md`), raw: (ext) => join9(entryDir, `${entry}-page.${ext}`) };
3687
+ }
3688
+ function renderUrlMeta(m) {
3689
+ const lines = [
3690
+ ["url", m.url],
3691
+ ["final_url", m.final_url],
3692
+ ["title", m.title === null ? "" : JSON.stringify(m.title)],
3693
+ ["format", m.format],
3694
+ ["content_type", m.content_type],
3695
+ ["level", m.level],
3696
+ ["retrieved_by", m.retrieved_by],
3697
+ ["fetched", m.fetched],
3698
+ ["error", m.error ?? ""],
3699
+ ["files", m.files.join(", ")],
3700
+ ["converted_by", m.converted_by]
3701
+ ];
3702
+ return lines.map(([k, v]) => `${k}: ${v}`).join("\n") + "\n";
3703
+ }
3704
+ function readUrlMeta(p) {
3705
+ if (!existsSync8(p.meta)) return null;
3706
+ const md = readFileSync8(p.meta, "utf8");
3707
+ const f = (k) => {
3708
+ const m = new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md);
3709
+ return m ? m[1].trim() : "";
3710
+ };
3711
+ let title = null;
3712
+ try {
3713
+ title = f("title") ? JSON.parse(f("title")) : null;
3714
+ } catch {
3715
+ title = f("title") || null;
3716
+ }
3717
+ return {
3718
+ url: f("url"),
3719
+ final_url: f("final_url"),
3720
+ title,
3721
+ format: f("format") || "failed",
3722
+ content_type: f("content_type"),
3723
+ level: f("level") === "full_text" ? "full_text" : "title",
3724
+ retrieved_by: f("retrieved_by"),
3725
+ fetched: f("fetched"),
3726
+ error: f("error") || null,
3727
+ files: f("files") ? f("files").split(",").map((s) => s.trim()).filter(Boolean) : [],
3728
+ converted_by: f("converted_by")
3729
+ };
3730
+ }
3731
+ function writeUrlMeta(p, m) {
3732
+ mkdirSync5(p.entryDir, { recursive: true });
3733
+ writeAtomic(p.meta, renderUrlMeta(m));
3734
+ }
3735
+ var SHELL_WORDS = 100;
3736
+ var EMPTY_WORDS = 10;
3737
+ var SHELL_PHRASE = /\bloading\b|javascript|enable it|connecting to|please wait|redirecting/i;
3738
+ function isScriptShell(markdownOrHtml) {
3739
+ const text3 = markdownOrHtml.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>|<noscript[\s\S]*?<\/noscript>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&[a-z#0-9]+;/gi, " ").replace(/\]\([^)]*\)/g, ")");
3740
+ const words2 = text3.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
3741
+ return { shell: words2 < EMPTY_WORDS || words2 < SHELL_WORDS && SHELL_PHRASE.test(text3), words: words2 };
3742
+ }
3743
+ var viaName = (via) => typeof via === "object" ? `browser ${via.label}` : via;
3744
+ async function rung(url, via) {
3745
+ const name = viaName(via);
3746
+ let got;
3747
+ try {
3748
+ got = via === "brightdata" ? await unlock(url) : typeof via === "object" ? await browserFetch(url, via.browser) : await fetchBytes(url, BROWSER_HEADERS);
3749
+ } catch (e) {
3750
+ return { ok: false, reason: errMsg(e), url: e?.url, via: name };
3751
+ }
3752
+ if (via === "brightdata" && !got.bytes.length) return { ok: false, reason: "empty response", via: name };
3753
+ const r = { ok: true, bytes: got.bytes, contentType: got.contentType, url: got.url ?? url, via: name };
3754
+ if (!looksLikePdf(got.bytes)) {
3755
+ const text3 = got.bytes.toString("utf8");
3756
+ if (isHtml(got.contentType, text3)) {
3757
+ if (isWallPage(text3)) return { ok: false, reason: "wall page", url: r.url, via: name };
3758
+ let md = "";
3759
+ try {
3760
+ md = htmlToMarkdown(/<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3);
3761
+ } catch {
3762
+ }
3763
+ const s = isScriptShell(md || text3);
3764
+ if (s.shell) return { ok: false, reason: `script shell (${s.words} words)`, url: r.url, via: name, shell: r };
3765
+ }
3766
+ }
3767
+ return r;
3768
+ }
3769
+ var climb = async (url, onTry, onAlert) => {
3770
+ let r = await rung(url, "fetch");
3771
+ const firstReason = r.ok ? void 0 : r.reason;
3772
+ const target = !r.ok && r.url && r.url !== url ? r.url : url;
3773
+ if (target !== url) onTry?.(`\u2192 ${target}`);
3774
+ onTry?.(r.ok ? "ok fetch" : `failed fetch: ${r.reason}`);
3775
+ const escalates = (g) => !g.ok && (g.via !== "fetch" || /^(HTTP 403|HTTP 429|wall page|script shell)/.test(g.reason));
3776
+ let shell = !r.ok ? r.shell : void 0;
3777
+ for (const b of browserRungs()) {
3778
+ if (!escalates(r)) break;
3779
+ r = await rung(target, { browser: b.endpoint, label: b.label });
3780
+ if (!r.ok && r.shell && !shell) shell = r.shell;
3781
+ onTry?.(r.ok ? `ok browser ${b.label}` : `failed browser ${b.label}: ${r.reason}`);
3782
+ if (!r.ok && r.reason.startsWith(MISMATCH)) onAlert?.(`browser ${b.label}: ${r.reason}`);
3783
+ }
3784
+ if (!r.ok && unlockerConfigured() && (r.via !== "fetch" || /^(HTTP 403|wall page|script shell)/.test(r.reason))) {
3785
+ r = await rung(target, "brightdata");
3786
+ onTry?.(r.ok ? "ok brightdata" : `failed brightdata: ${r.reason}`);
3787
+ if (!r.ok && /^unlocker HTTP/.test(r.reason)) onAlert?.(`brightdata: ${r.reason}`);
3788
+ }
3789
+ if (!r.ok && shell) {
3790
+ onTry?.(`script shell kept: no rung rendered the page`);
3791
+ return shell;
3792
+ }
3793
+ return r.ok || r.via === "fetch" ? r : { ...r, first: firstReason };
3794
+ };
3795
+ var mediaType = (contentType) => contentType.split(";")[0].trim().toLowerCase();
3796
+ function isHtml(contentType, text3) {
3797
+ const ct = contentType.toLowerCase();
3798
+ if (/<html/i.test(text3.slice(0, 100))) return true;
3799
+ if (ct.includes("text/html") || ct.includes("application/xhtml+xml")) return true;
3800
+ return !ct && /^(<!doctype html|<html)/.test(text3.replace(/^\uFEFF/, "").trimStart().slice(0, 20).toLowerCase());
3801
+ }
3802
+ var ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "\u2013", mdash: "\u2014", hellip: "\u2026", copy: "\xA9", reg: "\xAE", trade: "\u2122" };
3803
+ var decode2 = (s) => s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
3804
+ if (e[0] === "#") {
3805
+ const n = e[1].toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
3806
+ return Number.isFinite(n) ? String.fromCodePoint(n) : m;
3807
+ }
3808
+ return ENTITIES[e.toLowerCase()] ?? m;
3809
+ });
3810
+ var visibleText = (html) => decode2(html.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").replace(/¶\s*$/, "").trim();
3811
+ var normalized = (text3) => {
3812
+ const words2 = text3.toLowerCase().match(/[\p{L}\p{N}]+/gu);
3813
+ return words2 ? ` ${words2.join(" ")} ` : "";
3814
+ };
3815
+ function pageTitle(html) {
3816
+ const head = html.slice(0, 2e5);
3817
+ const og = /<meta\b[^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(head)?.[0] ?? /<meta\b[^>]*\bcontent\s*=\s*["'][^"']*["'][^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(head)?.[0];
3818
+ const ogTitle = og ? visibleText(/\bcontent\s*=\s*("([^"]*)"|'([^']*)')/i.exec(og)?.[2] ?? /\bcontent\s*=\s*'([^']*)'/i.exec(og)?.[1] ?? "") : "";
3819
+ const title = visibleText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1] ?? "");
3820
+ const h1 = visibleText(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1] ?? "");
3821
+ const headingInTitle = !!normalized(h1) && normalized(title).includes(normalized(h1));
3822
+ if (ogTitle) return ogTitle;
3823
+ if (headingInTitle) return h1;
3824
+ return title || h1 || null;
3825
+ }
3826
+ var visibleLine = (line) => line.replace(/\]\([^)]*\)/g, " ").replace(/<[a-zA-Z/!?][^>]*>/g, " ");
3827
+ function opensWithTitle(content, title) {
3828
+ const first = content.split("\n").find((l) => l.trim()) ?? "";
3829
+ const line = normalized(visibleLine(first)), wanted = normalized(title);
3830
+ if (!line || !wanted) return false;
3831
+ if (line.includes(wanted)) return true;
3832
+ return first.trimStart().startsWith("#") && wanted.includes(line);
3833
+ }
3834
+ function withPageTitle(html, content) {
3835
+ if (content.startsWith("<error>")) return content;
3836
+ const title = pageTitle(html);
3837
+ if (title === null || opensWithTitle(content, title)) return content;
3838
+ return `# ${title.replace(/([\\`*_\[\]<>#&|~])/g, "\\$1")}
3839
+
3840
+ ${content.trimStart()}`;
3841
+ }
3842
+ var SIMPLIFY_FAILED = "<error>Page failed to be simplified from HTML</error>";
3843
+ async function convert(bytes, contentType) {
3844
+ if (looksLikePdf(bytes)) {
3845
+ const md = (await pdfToMarkdown(bytes, "", "")).replace(/^# \n\n/, "");
3846
+ return { format: "pdf", md, title: null, ext: "pdf" };
3847
+ }
3848
+ const text3 = bytes.toString("utf8");
3849
+ if (mediaType(contentType) === "text/markdown") return { format: "markdown", md: text3, title: null, ext: "md" };
3850
+ if (isHtml(contentType, text3)) {
3851
+ const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
3852
+ let md;
3853
+ try {
3854
+ md = htmlToMarkdown(body);
3855
+ } catch {
3856
+ md = "";
3857
+ }
3858
+ md = md.trim() ? withPageTitle(text3, md) : SIMPLIFY_FAILED;
3859
+ return { format: "html", md, title: pageTitle(text3), ext: "html" };
3860
+ }
3861
+ return { format: "text", md: null, title: null, ext: "txt" };
3862
+ }
3863
+ function fetchAnswer(url, body, prefix, maxLength, startIndex) {
3864
+ const original = body.length;
3865
+ let content;
3866
+ if (startIndex >= original) content = "<error>No more content available.</error>";
3867
+ else {
3868
+ const cut = body.slice(startIndex, startIndex + maxLength);
3869
+ content = cut;
3870
+ const remaining = original - (startIndex + cut.length);
3871
+ if (cut.length === maxLength && remaining > 0) content += `
3872
+
3873
+ <error>Content truncated. Call the fetch tool with a start_index of ${startIndex + cut.length} to get more content.</error>`;
3874
+ }
3875
+ return `${prefix}Contents of ${url}:
3876
+ ${content}`;
3877
+ }
3878
+ var ID_WIDTH = 13;
3879
+ function logId(url) {
3880
+ const bare2 = url.replace(/^https?:\/\//i, "");
3881
+ return bare2.length > ID_WIDTH ? bare2.slice(0, ID_WIDTH - 1) + "\u2026" : bare2;
3882
+ }
3883
+ var failureText = (url, reason) => {
3884
+ const m = /^HTTP (\d{3})\b/.exec(reason);
3885
+ return m ? `Failed to fetch ${url} - status code ${m[1]}` : `Failed to fetch ${url}: ${reason}`;
3886
+ };
3887
+ function titleLine(m, md) {
3888
+ if (m.title) return `# ${m.title}`;
3889
+ const first = md.split("\n").find((l) => l.trim());
3890
+ if (first) return first.trim();
3891
+ return `(${m.content_type || "no content type"}, no title)`;
3892
+ }
3893
+ function serve(requested, p, m, args, cached) {
3894
+ const maxLength = args.max_length ?? 5e3, startIndex = args.start_index ?? 0;
3895
+ const read = (file2) => existsSync8(file2) ? readFileSync8(file2, "utf8") : "";
3896
+ if (args.title_only) return { ok: true, text: `Contents of ${requested}:
3897
+ ${titleLine(m, read(p.md))}`, cached, meta: m };
3898
+ const rawFile = m.files.find((f) => !f.endsWith("-meta.md") && !f.endsWith("-page.md"));
3899
+ let body, prefix = "";
3900
+ if (m.format === "markdown") body = read(p.md);
3901
+ else if (m.format === "text" || args.raw) {
3902
+ body = rawFile ? read(join9(p.entryDir, rawFile)) : read(p.md);
3903
+ prefix = `Content type ${m.content_type || "(not declared)"} cannot be simplified to markdown, but here is the raw content:
3904
+ `;
3905
+ } else body = read(p.md);
3906
+ return { ok: true, text: fetchAnswer(requested, body, prefix, maxLength, startIndex), cached, meta: m };
3907
+ }
3908
+ var inflight = /* @__PURE__ */ new Map();
3909
+ async function libraryFetch(libraryDir, requested, args, converter, logs = {}, fetchImpl = climb) {
3910
+ const url = normalizeUrl(requested);
3911
+ if (!url) return { ok: false, error: `Failed to fetch ${requested}: invalid URL`, cached: false, meta: null };
3912
+ const p = urlPaths(libraryDir, url);
3913
+ const s = settings();
3914
+ const fresh = (m2) => daysSince(m2.fetched) < (m2.format === "failed" ? s.retry_after_days : s.refresh_after_days);
3915
+ const answer = (m2, cached) => m2.format === "failed" ? { ok: false, error: failureText(requested, m2.error ?? "unknown error"), cached, meta: m2 } : serve(requested, p, m2, args, cached);
3916
+ const stored = readUrlMeta(p);
3917
+ if (stored && fresh(stored)) return answer(stored, true);
3918
+ let run = inflight.get(p.hash);
3919
+ if (!run) {
3920
+ run = (async () => {
3921
+ const r = await fetchImpl(url, logs.onTry, logs.onAlert);
3922
+ const base = { url, fetched: today(), converted_by: converter };
3923
+ if (!r.ok) {
3924
+ const error = r.first && /^HTTP \d{3}/.test(r.first) ? r.first : r.reason;
3925
+ writeUrlMeta(p, { ...base, final_url: r.url ?? url, title: null, format: "failed", content_type: "", level: "title", retrieved_by: r.via, error, files: [`${p.entry}-meta.md`] });
3926
+ return;
3927
+ }
3928
+ const page = await convert(r.bytes, r.contentType);
3929
+ mkdirSync5(p.entryDir, { recursive: true });
3930
+ const files = [`${p.entry}-meta.md`];
3931
+ if (page.md !== null) {
3932
+ writeAtomic(p.md, page.md);
3933
+ files.push(`${p.entry}-page.md`);
3934
+ }
3935
+ if (page.ext !== "md") {
3936
+ writeAtomic(p.raw(page.ext), r.bytes);
3937
+ files.push(`${p.entry}-page.${page.ext}`);
3938
+ }
3939
+ writeUrlMeta(p, { ...base, final_url: r.url, title: page.title, format: page.format, content_type: mediaType(r.contentType), level: "full_text", retrieved_by: r.via, error: null, files });
3940
+ })().finally(() => inflight.delete(p.hash));
3941
+ inflight.set(p.hash, run);
3942
+ }
3943
+ await run;
3944
+ const m = readUrlMeta(p);
3945
+ if (!m) return { ok: false, error: `Failed to fetch ${requested}: nothing stored`, cached: false, meta: null };
3946
+ return answer(m, false);
3947
+ }
3948
+
3949
+ // src/verifycache.ts
3950
+ import { createHash as createHash3 } from "crypto";
3951
+ import { existsSync as existsSync9, mkdirSync as mkdirSync6, readFileSync as readFileSync9 } from "fs";
3952
+ import { dirname as dirname3, join as join10 } from "path";
3953
+ var sha = (s) => createHash3("sha1").update(s).digest("hex");
3954
+ var normalizeClaim = (claim) => claim.replace(/\s+/g, " ").trim();
3955
+ function foldClaim(claim) {
3956
+ return claim.normalize("NFKC").toLowerCase().replace(/[\u2018\u2019\u201a\u201b\u2032]/g, "'").replace(/[\u201c\u201d\u201e\u201f\u2033]/g, '"').replace(/[\u2010-\u2015\u2212]/g, "-").replace(/\+\/-|\+-/g, "\xB1").replace(/\s+/g, " ").trim().replace(/[.;:,]+$/, "").trim();
3957
+ }
3958
+ var textHash = (text3) => sha(text3).slice(0, 16);
3959
+ var cacheKey = (claim, id, model, threshold, text3) => sha(`${foldClaim(claim)}
3960
+ ${id}
3961
+ ${model}
3962
+ ${threshold}
3963
+ ${text3}`).slice(0, 16);
3964
+ var verdictFile = (libraryDir, key) => join10(libraryDir, "verify", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
3965
+ function readVerdict(file2) {
3966
+ if (!existsSync9(file2)) return null;
3967
+ try {
3968
+ return JSON.parse(readFileSync9(file2, "utf8"));
3969
+ } catch {
3970
+ return null;
3971
+ }
3972
+ }
3973
+ function lookup(file2, maxDays) {
3974
+ const c = readVerdict(file2);
3975
+ return c && daysSince(c.date) < maxDays ? c : null;
3976
+ }
3977
+ function remember(file2, c) {
3978
+ mkdirSync6(dirname3(file2), { recursive: true });
3979
+ writeAtomic(file2, JSON.stringify(c, null, 2) + "\n");
3980
+ }
3981
+ async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR, read = readPaper) {
3982
+ const paper = await read(id, libraryDir);
3983
+ const once = () => paper;
3984
+ if ("error" in paper || !cfg) return { ...await verifyClaim(claim, id, cfg, decide, once), cached: false };
3985
+ const text3 = textHash(paper.text);
3986
+ const file2 = verdictFile(libraryDir, cacheKey(claim, paper.id, cfg.model, cfg.threshold, text3));
3987
+ const hit = lookup(file2, settings().refresh_after_days);
3988
+ if (hit) {
3989
+ return { claim, id: paper.id, verdict: hit.verdict, reason: hit.reason, score: hit.score, basis: hit.basis, windows: hit.windows, glyphs_unresolved: hit.glyphs_unresolved, superseded: hit.superseded, model: hit.model, cached: true };
3990
+ }
3991
+ const r = await verifyClaim(claim, id, cfg, decide, once);
3992
+ if (!r.error && r.score !== null) {
3993
+ remember(file2, { key: file2.slice(-21, -5), id: paper.id, claim: normalizeClaim(claim), model: cfg.model, threshold: cfg.threshold, text: text3, date: today(), verdict: r.verdict, reason: r.reason, score: r.score, basis: r.basis, windows: r.windows, glyphs_unresolved: r.glyphs_unresolved, superseded: r.superseded });
3994
+ }
3995
+ return { ...r, cached: false };
3996
+ }
3997
+
3998
+ // src/json.ts
3999
+ var DROP = /* @__PURE__ */ new Set(["entry_dir", "library_dir", "files"]);
4000
+ var compact2 = (o) => JSON.stringify(o);
4001
+ function lean(o) {
4002
+ if (Array.isArray(o)) return o.map((row2) => row2 && typeof row2 === "object" && !Array.isArray(row2) ? leanObject(row2) : row2);
4003
+ if (o && typeof o === "object") return leanObject(o);
4004
+ return o;
4005
+ }
4006
+ function leanObject(o) {
4007
+ const out = {};
4008
+ for (const [k, v] of Object.entries(o)) {
4009
+ if (DROP.has(k) || v === null || v === void 0 || Array.isArray(v) && v.length === 0) continue;
4010
+ out[k] = Array.isArray(v) ? lean(v) : v;
4011
+ }
4012
+ return out;
4013
+ }
4014
+ var leanJson = (o) => compact2(lean(o));
4015
+
4016
+ // src/trials.ts
4017
+ import { createHash as createHash4 } from "crypto";
4018
+ import { existsSync as existsSync10, mkdirSync as mkdirSync7, readFileSync as readFileSync10 } from "fs";
4019
+ import { dirname as dirname4, join as join11 } from "path";
4020
+ var SEARCH_AFTER_DAYS = 7;
4021
+ var API2 = "https://clinicaltrials.gov/api/v2/studies";
4022
+ var FIELDS = ["NCTId", "BriefTitle", "OverallStatus", "WhyStopped", "Phase", "StudyType", "EnrollmentCount", "EnrollmentType", "StartDate", "PrimaryCompletionDate", "CompletionDate", "LastUpdatePostDate", "PrimaryOutcomeMeasure", "PrimaryOutcomeTimeFrame", "Condition", "InterventionName", "LeadSponsorName", "HasResults"];
4023
+ var ONGOING = ["RECRUITING", "NOT_YET_RECRUITING", "ACTIVE_NOT_RECRUITING", "ENROLLING_BY_INVITATION"];
4024
+ var MAX_RESULTS = 100;
4025
+ var DEFAULT_RESULTS = 20;
4026
+ var SORTS = { relevance: null, newest: "LastUpdatePostDate:desc", largest: "EnrollmentCount:desc", completion: "CompletionDate:desc" };
4027
+ var COLUMNS = [
4028
+ "nct",
4029
+ "title",
4030
+ "status",
4031
+ // "Terminated — slow accrual": a trial past its completion date is not "ongoing" (ER.md 28.7)
4032
+ "phase",
4033
+ "enrollment",
4034
+ // "100 (estimated)" / "48 (actual)" — participant count (28.5)
4035
+ "start",
4036
+ "primary_completion",
4037
+ "primary_outcome",
4038
+ // the first primary outcome with its time frame, cut at 80 characters — the primary endpoint (28.5)
4039
+ "conditions",
4040
+ "interventions",
4041
+ // "IVF; Infertility, Female" — one string each
4042
+ "has_results"
4043
+ // posted results: get_study_results (28.8)
4044
+ ];
4045
+ var clean = (s) => (s ?? "").replace(/\s+/g, " ").trim();
4046
+ function normalizeQuery(q) {
4047
+ const out = { status: q.status ?? "all", sort: q.sort ?? "relevance", max_results: Math.min(MAX_RESULTS, Math.max(1, Math.floor(q.max_results ?? DEFAULT_RESULTS))) };
4048
+ if (clean(q.query)) out.query = clean(q.query);
4049
+ if (clean(q.intervention)) out.intervention = clean(q.intervention);
4050
+ if (clean(q.condition)) out.condition = clean(q.condition);
4051
+ if (!out.query && !out.intervention && !out.condition) return { error: "empty search: give query, intervention or condition" };
4052
+ if (!["ongoing", "completed", "all"].includes(out.status)) return { error: `status must be ongoing, completed or all, not "${String(q.status)}"` };
4053
+ if (!(out.sort in SORTS)) return { error: `sort must be relevance, newest, largest or completion, not "${String(q.sort)}"` };
4054
+ return out;
4055
+ }
4056
+ var searchKey = (q) => createHash4("sha1").update(JSON.stringify(["clinicaltrials", q.query ?? "", q.intervention ?? "", q.condition ?? "", q.status, q.sort, q.max_results])).digest("hex").slice(0, 16);
4057
+ var searchFile = (libraryDir, key) => join11(libraryDir, "search", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
4058
+ function searchUrl(q) {
4059
+ const p = new URLSearchParams();
4060
+ if (q.query) p.set("query.term", q.query);
4061
+ if (q.intervention) p.set("query.intr", q.intervention);
4062
+ if (q.condition) p.set("query.cond", q.condition);
4063
+ if (q.status === "ongoing") p.set("filter.overallStatus", ONGOING.join(","));
4064
+ if (q.status === "completed") p.set("filter.overallStatus", "COMPLETED");
4065
+ if (SORTS[q.sort]) p.set("sort", SORTS[q.sort]);
4066
+ p.set("fields", FIELDS.join(","));
4067
+ p.set("pageSize", String(q.max_results));
4068
+ p.set("countTotal", "true");
4069
+ return `${API2}?${p.toString()}`;
4070
+ }
4071
+ var STATUS2 = { NOT_YET_RECRUITING: "Not yet recruiting", RECRUITING: "Recruiting", ENROLLING_BY_INVITATION: "Enrolling by invitation", ACTIVE_NOT_RECRUITING: "Active, not recruiting", SUSPENDED: "Suspended", TERMINATED: "Terminated", COMPLETED: "Completed", WITHDRAWN: "Withdrawn", UNKNOWN: "Unknown status", AVAILABLE: "Available", NO_LONGER_AVAILABLE: "No longer available", TEMPORARILY_NOT_AVAILABLE: "Temporarily not available", APPROVED_FOR_MARKETING: "Approved for marketing", WITHHELD: "Withheld" };
4072
+ var PHASE2 = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
4073
+ var joined = (xs) => xs.length ? xs.map((x) => x.replace(/\s+/g, " ").trim()).filter(Boolean).join("; ") || null : null;
4074
+ function parseRows(json2) {
4075
+ const d = JSON.parse(json2);
4076
+ const rows = [];
4077
+ for (const s of Array.isArray(d.studies) ? d.studies : []) {
4078
+ const p = s.protocolSection ?? {};
4079
+ const id = p.identificationModule ?? {}, st = p.statusModule ?? {}, des = p.designModule ?? {};
4080
+ const phases = Array.isArray(des.phases) ? des.phases : [];
4081
+ const nct = String(id.nctId ?? "");
4082
+ const en = des.enrollmentInfo ?? {};
4083
+ const outcomes = Array.isArray(p.outcomesModule?.primaryOutcomes) ? p.outcomesModule.primaryOutcomes : [];
4084
+ const first = outcomes[0];
4085
+ const cut = (t, n) => t.length > n ? t.slice(0, n - 1).trimEnd() + "\u2026" : t;
4086
+ const primary = first?.measure ? cut(`${String(first.measure).replace(/\s+/g, " ").trim()}${first.timeFrame ? ` (${String(first.timeFrame).replace(/\s+/g, " ").trim()})` : ""}`, 80) + (outcomes.length > 1 ? ` +${outcomes.length - 1}` : "") : null;
4087
+ const status = (STATUS2[st.overallStatus] ?? String(st.overallStatus ?? "")) + (st.whyStopped ? ` \u2014 ${String(st.whyStopped).replace(/\s+/g, " ").trim()}` : "");
4088
+ rows.push([
4089
+ nct,
4090
+ String(id.briefTitle ?? ""),
4091
+ status,
4092
+ phases.length ? phases.map((x) => PHASE2[x] ?? x).join(" / ") : null,
4093
+ typeof en.count === "number" ? `${en.count}${en.type ? ` (${String(en.type).toLowerCase()})` : ""}` : null,
4094
+ st.startDateStruct?.date ?? null,
4095
+ st.primaryCompletionDateStruct?.date ?? null,
4096
+ primary,
4097
+ joined(Array.isArray(p.conditionsModule?.conditions) ? p.conditionsModule.conditions.map(String) : []),
4098
+ joined(Array.isArray(p.armsInterventionsModule?.interventions) ? p.armsInterventionsModule.interventions.map((i) => String(i?.name ?? "")).filter(Boolean) : []),
4099
+ s.hasResults === true
4100
+ ]);
4101
+ }
4102
+ return { total: typeof d.totalCount === "number" ? d.totalCount : rows.length, rows };
4103
+ }
4104
+ var liveFetch = (url) => fetchText(url, { "User-Agent": LIBRARY_UA, Accept: "application/json" });
4105
+ async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFetch) {
4106
+ const q = normalizeQuery(raw);
4107
+ if ("error" in q) return { ok: false, error: q.error };
4108
+ const file2 = searchFile(libraryDir, searchKey(q));
4109
+ if (existsSync10(file2)) {
4110
+ try {
4111
+ const r = JSON.parse(readFileSync10(file2, "utf8"));
4112
+ if (daysSince(r.searched) < SEARCH_AFTER_DAYS) return { ok: true, result: r, cached: true };
4113
+ } catch {
4114
+ }
4115
+ }
4116
+ let json2;
4117
+ try {
4118
+ json2 = await fetchImpl(searchUrl(q));
4119
+ } catch (e) {
4120
+ return { ok: false, error: `clinicaltrials.gov: ${e instanceof Error ? e.message : String(e)}` };
4121
+ }
4122
+ let parsed;
4123
+ try {
4124
+ parsed = parseRows(json2);
4125
+ } catch {
4126
+ return { ok: false, error: "clinicaltrials.gov: not a studies answer" };
4127
+ }
4128
+ const result = { source: "clinicaltrials", query: q, total: parsed.total, columns: COLUMNS, rows: parsed.rows, searched: today() };
4129
+ try {
4130
+ mkdirSync7(dirname4(file2), { recursive: true });
4131
+ writeAtomic(file2, JSON.stringify(result, null, 2) + "\n");
4132
+ } catch {
4133
+ }
4134
+ return { ok: true, result, cached: false };
4135
+ }
4136
+ function trialsLogId(q) {
4137
+ const s = (q.intervention ?? q.condition ?? q.query ?? "").replace(/\s+/g, " ");
4138
+ return s.length > 10 ? s.slice(0, 9) + "\u2026" : s || "-";
4139
+ }
4140
+
4141
+ // src/trialrecord.ts
4142
+ var list3 = (a) => Array.isArray(a) ? a : [];
4143
+ var para2 = (s, max = Infinity) => {
4144
+ const t = String(s ?? "").replace(/\s+/g, " ").trim();
4145
+ if (!t) return null;
4146
+ return t.length > max ? t.slice(0, max - 1).trimEnd() + "\u2026" : t;
4147
+ };
4148
+ var joined2 = (xs) => {
4149
+ const ys = xs.filter((x) => !!x);
4150
+ return ys.length ? ys.join("; ") : null;
4151
+ };
4152
+ var dated2 = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : null;
4153
+ function recordSummary(json2) {
4154
+ let d;
4155
+ try {
4156
+ d = JSON.parse(json2);
4157
+ } catch {
4158
+ return null;
4159
+ }
4160
+ const ps = d?.protocolSection;
4161
+ if (!ps) return null;
4162
+ const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
4163
+ const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
4164
+ const nct = String(idm.nctId ?? "").toUpperCase();
4165
+ const title = para2(idm.briefTitle) ?? nct;
4166
+ const official = para2(idm.officialTitle);
4167
+ const di = dm.designInfo ?? {}, mi = di.maskingInfo ?? {};
4168
+ const design = joined2([di.allocation ? words(di.allocation) : null, di.interventionModel ? words(di.interventionModel) : null, mi.masking ? `masking ${words(mi.masking).toLowerCase()}` : null, di.primaryPurpose ? `purpose ${words(di.primaryPurpose).toLowerCase()}` : null]);
4169
+ const outcome = (o) => `${para2(o.measure) ?? ""}${o.timeFrame ? ` (${para2(o.timeFrame)})` : ""}`.trim();
4170
+ const countries = [...new Set(list3(cl.locations).map((l) => l.country).filter(Boolean))];
4171
+ const refs = list3(rf.references);
4172
+ const age = el.minimumAge || el.maximumAge ? `${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : null;
4173
+ return {
4174
+ nct,
4175
+ title,
4176
+ official_title: official && official !== title ? official : null,
4177
+ acronym: para2(idm.acronym),
4178
+ status: st.overallStatus ? words(st.overallStatus) : null,
4179
+ why_stopped: para2(st.whyStopped),
4180
+ phase: list3(dm.phases).length ? list3(dm.phases).map(words).join(" / ") : null,
4181
+ study_type: dm.studyType ? words(dm.studyType) : null,
4182
+ design,
4183
+ enrollment: dm.enrollmentInfo?.count !== void 0 ? `${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null,
4184
+ start: dated2(st.startDateStruct),
4185
+ primary_completion: dated2(st.primaryCompletionDateStruct),
4186
+ completion: dated2(st.completionDateStruct),
4187
+ first_posted: st.studyFirstPostDateStruct?.date ?? null,
4188
+ last_update: st.lastUpdatePostDateStruct?.date ?? null,
4189
+ results_first_posted: st.resultsFirstPostDateStruct?.date ?? null,
4190
+ sponsor: para2(sp.leadSponsor?.name),
4191
+ collaborators: joined2(list3(sp.collaborators).map((c) => para2(c.name))),
4192
+ conditions: joined2(list3(ps.conditionsModule?.conditions).map((c) => para2(c))),
4193
+ interventions: joined2(list3(ai.interventions).map((i) => para2(`${i.type ? `${words(i.type)}: ` : ""}${i.name ?? ""}`))),
4194
+ primary_outcomes: list3(om.primaryOutcomes).map(outcome).filter(Boolean),
4195
+ secondary_outcomes: list3(om.secondaryOutcomes).length,
4196
+ eligibility: joined2([el.sex ? `sex ${words(el.sex).toLowerCase()}` : null, age ? `age ${age}` : null, el.healthyVolunteers !== void 0 ? `healthy volunteers ${el.healthyVolunteers ? "yes" : "no"}` : null]),
4197
+ locations: countries.length ? `${list3(cl.locations).length} site${list3(cl.locations).length === 1 ? "" : "s"} in ${countries.join(", ")}` : null,
4198
+ publications: refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`),
4199
+ has_results: d.hasResults === true,
4200
+ summary: para2(de.briefSummary, 400)
4201
+ };
4202
+ }
4203
+ function resultsText(nct, md, maxChars) {
4204
+ const i = md.search(/^## Results\s*$/m);
4205
+ if (i < 0) return { nct, has_results: false, text: null, truncated: false, outcomes: [], note: "no posted results in the registry record" };
4206
+ const part = md.slice(i).replace(/^## Results\s*\n/, "").trim();
4207
+ const outcomes = [...part.matchAll(/^#### (.+)$/gm)].map((m) => m[1].trim());
4208
+ const truncated = part.length > maxChars;
4209
+ return { nct, has_results: true, text: truncated ? part.slice(0, maxChars).trimEnd() + "\n\n[truncated]" : part, truncated, outcomes, note: truncated ? "cut at max_chars; library_read(id, section: <an outcome title>) returns one measure whole" : null };
4210
+ }
4211
+
3560
4212
  // src/tools.ts
3561
4213
  var pkg = (() => {
3562
4214
  try {
3563
- return JSON.parse(readFileSync7(new URL("../package.json", import.meta.url), "utf8"));
4215
+ return JSON.parse(readFileSync11(new URL("../package.json", import.meta.url), "utf8"));
3564
4216
  } catch {
3565
4217
  return { name: "fh-library", version: "0.0.0" };
3566
4218
  }
3567
4219
  })();
3568
- var INSTRUCTIONS = `fh-library keeps a local library of scientific papers, FDA drug labels and clinical trial records, so you can check what a source actually says. The server downloads each record, converts it to Markdown and stores it on disk. Every tool returns a small JSON answer; the document itself never enters your context. One ID per call.
4220
+ var INSTRUCTIONS = `fh-library keeps a local library of scientific papers, FDA drug labels, clinical trial records and web pages, so you can check what a source actually says, and searches ClinicalTrials.gov. The server downloads each record, converts it to Markdown and stores it on disk. Every tool returns a small JSON answer; the document itself never enters your context \u2014 except library_fetch, which is the standard fetch tool with a cache. One ID per call.
3569
4221
 
3570
4222
  IDs \u2014 always "type:value"
3571
4223
  - pmid:30955454 \u2014 a paper by PubMed ID. Bare digits are taken as a PMID.
@@ -3575,11 +4227,15 @@ IDs \u2014 always "type:value"
3575
4227
 
3576
4228
  Tools
3577
4229
  - library_version() \u2014 package name, version, library directory.
3578
- - library_info(ids?) \u2014 the whole library: entry counts by type and level, the retrieval sources, browsers, the decision model, the configuration. ids=true lists every entry.
4230
+ - library_info(ids?) \u2014 the whole library: entry counts by type and level, the retrieval sources, browsers, the decision model, the configuration, this server's cache hits and misses since it started (cache). ids=true lists every entry.
3579
4231
  - library_add(id) \u2014 download and store one record; returns its status. A stored full text is returned without any download; an abstract-level record retries only the sources that failed transiently; a record older than 30 days gets its correction notices re-read.
3580
4232
  - library_status(id) \u2014 the status of one record without downloading; level "missing" when it is not stored.
3581
4233
  - library_read(id, section?, query?, max_chars?, max_hits?) \u2014 one small piece of a stored document: its outline, one section, or the passages matching a query.
3582
4234
  - library_verify(claim, id) \u2014 does this one source state this one finding? A decision model judges the stored text and returns a verdict.
4235
+ - library_clinicaltrials_search_studies(query | intervention | condition, status?, sort?, max_results?) \u2014 search ClinicalTrials.gov: the total and rows in columns order (NCT ID, title, status, phase, enrollment, dates, primary outcome, conditions, interventions, has_results). Cached a week.
4236
+ - library_clinicaltrials_get_study_record(nct) \u2014 one trial as a review needs it (status, design, enrollment, dates, outcomes, eligibility, result publications, summary), from the stored record; added first when missing.
4237
+ - library_clinicaltrials_get_study_results(nct, max_chars?) \u2014 a trial's posted results as Markdown, capped; the outcome titles listed for library_read.
4238
+ - library_fetch(url, max_length?, start_index?, raw?, title_only?) \u2014 a web page as Markdown, same call and same answer as the standard fetch tool ("Contents of <url>:" and the page, cut at max_length with a note on how to continue). Prefer it for every URL: a cached page answers from disk, a walled or script-rendered page climbs the browser and unlocker ladder in one call. PDFs as Markdown. title_only returns the title line alone \u2014 enough to verify a link, at a few tokens.
3583
4239
 
3584
4240
  Hints
3585
4241
  - library_read and library_verify store a missing record themselves; the library_read answer then carries added: true. No library_add necessary first.
@@ -3587,6 +4243,8 @@ Hints
3587
4243
  - level \u2014 what is stored: "full_text", "abstract" (no full text could be found; attempts names each source tried and why it failed) or "title" (not even an abstract; citable false).
3588
4244
  - error "pmid not found" or "invalid id" \u2014 the ID itself is wrong; the same call gives the same answer.
3589
4245
  - Calls may run in parallel: library_add, library_read and library_verify. Two calls on the same missing record share one add.
4246
+ - library_fetch takes any URL that is not a paper, label or trial record; those have IDs, and library_add, library_read and library_verify work on the stored document. A second pass over the same review reads every page from disk; a failed fetch is remembered for a day.
4247
+ - To verify that a web page exists and what it is: library_fetch(url, title_only: true). The answer is "Contents of <url>:" and the page's title line \u2014 a few tokens \u2014 or the fetch error; the page itself is stored in full, so a later library_fetch of the same URL reads from disk.
3590
4248
 
3591
4249
  Verifying a claim
3592
4250
  - library_verify judges one claim against one stored document. A finding backed by three papers takes three calls with the same claim.
@@ -3598,6 +4256,7 @@ Verifying a claim
3598
4256
  - verdict "unverifiable" \u2014 only the abstract is stored and it does not state the claim; the full text may (reason says so). A retry gives the same answer until a full text is stored.
3599
4257
  - glyphs_unresolved above 0 \u2014 the PDF font lost symbols (minus, \xB1, \u2264); a fail on a number may be the font. library_read shows the passage.
3600
4258
  - superseded true \u2014 a published correction replaced the figure the claim uses; the status's corrections holds the notice.
4259
+ - Verdicts are kept for 30 days: the same claim against the same stored text answers from disk, no model call. A paper that gains its full text is judged afresh.
3601
4260
 
3602
4261
  Reading a document
3603
4262
  - library_read with only the ID returns the outline: every heading with its word count, and whether the full text or only the abstract is stored.
@@ -3608,12 +4267,12 @@ Reading a document
3608
4267
  - section and query together search inside that section only.
3609
4268
  - section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name.
3610
4269
 
3611
- Status fields: requested, type, id, entry, entry_dir, library_dir, pmid, pmcid, doi, setid, nctid, publications, title, first_author, year, publication_types, retracted, corrections, coi (conflict-of-interest statement), funding, citable, comment, has_abstract, level, version, retrieved_by, retrieved_date, pubmed_checked, attempts, files, glyphs_unresolved, error.
4270
+ Status fields: requested, type, id, entry, pmid, pmcid, doi, setid, nctid, publications, title, first_author, year, publication_types, retracted, corrections, coi (conflict-of-interest statement), funding, citable, comment, has_abstract, level, version, retrieved_by, retrieved_date, pubmed_checked, attempts, glyphs_unresolved, error. Answers are compact JSON; a field that is null or an empty list is left out (library_verify keeps its nulls: score and reason null mean something).
3612
4271
 
3613
4272
  Where records come from: PubMed for the record and abstract; the full text from a file you drop into <library>/inbox/ (PMID in the file name), PubMed Central, Europe PMC, publishers' text-mining links, Unpaywall, OpenAlex, CORE, and a proxy URL template from the configuration. Every action is appended as one line to <library>/library.log.`;
3614
4273
  var idSchema = z.string().describe('ID as "type:value": a paper "pmid:30955454" (bare digits count as pmid) or "doi:10.5812/asjsm.110753" (a paper PubMed does not index; a DOI PubMed indexes is taken as its PMID), a drug label "dailymed:<set ID>" (the set ID from its DailyMed page URL), or a trial record "nct:05173896" (the eight digits; nct:NCT05173896 and a bare NCT05173896 are accepted).');
3615
4274
  var levelWord = (m) => m.level === "abstract" && !m.has_abstract ? "title" : m.level;
3616
- var json = (s) => ({ content: [{ type: "text", text: JSON.stringify(s, null, 2) }] });
4275
+ var json = (s) => ({ content: [{ type: "text", text: leanJson(s) }] });
3617
4276
  function noticeTag(m) {
3618
4277
  const n = (t) => m.corrections.filter((c) => c.type === t).length;
3619
4278
  const parts = [
@@ -3737,13 +4396,13 @@ function registerTools(server2) {
3737
4396
  { id: idSchema },
3738
4397
  ({ id }) => addEntry(id)
3739
4398
  );
3740
- const inflight = /* @__PURE__ */ new Map();
4399
+ const inflight2 = /* @__PURE__ */ new Map();
3741
4400
  function addEntry(id) {
3742
4401
  const key = parseId(id).requested;
3743
- let p = inflight.get(key);
4402
+ let p = inflight2.get(key);
3744
4403
  if (!p) {
3745
- p = addEntryNow(id).finally(() => inflight.delete(key));
3746
- inflight.set(key, p);
4404
+ p = addEntryNow(id).finally(() => inflight2.delete(key));
4405
+ inflight2.set(key, p);
3747
4406
  }
3748
4407
  return p;
3749
4408
  }
@@ -3914,7 +4573,7 @@ function registerTools(server2) {
3914
4573
  },
3915
4574
  async ({ id, section: section2, query, max_chars, max_hits }) => {
3916
4575
  const parsed = parseId(id);
3917
- const out = (o2) => ({ content: [{ type: "text", text: JSON.stringify(o2, null, 2) }] });
4576
+ const out = (o2) => ({ content: [{ type: "text", text: leanJson(o2) }] });
3918
4577
  if (parsed.error) {
3919
4578
  log("library_read", parsed.requested, `error ${parsed.error}`);
3920
4579
  return out({ requested: parsed.requested, error: parsed.error });
@@ -3930,8 +4589,8 @@ function registerTools(server2) {
3930
4589
  }
3931
4590
  added = true;
3932
4591
  }
3933
- const full = existsSync8(paths.fulltextMd);
3934
- const md = readFileSync7(full ? paths.fulltextMd : paths.md, "utf8");
4592
+ const full = existsSync11(paths.fulltextMd);
4593
+ const md = readFileSync11(full ? paths.fulltextMd : paths.md, "utf8");
3935
4594
  const text3 = full ? "full_text" : "abstract";
3936
4595
  const base = added ? { requested: parsed.requested, entry: paths.entry, text: text3, added: true } : { requested: parsed.requested, entry: paths.entry, text: text3 };
3937
4596
  const headings2 = () => outline(md, text3).sections;
@@ -3951,8 +4610,8 @@ function registerTools(server2) {
3951
4610
  const cap = max_chars ?? 6e3;
3952
4611
  let sec = readSection(md, section2, cap);
3953
4612
  let via = "";
3954
- if (!sec && /abstract/i.test(section2) && full && existsSync8(paths.md)) {
3955
- sec = readSection(readFileSync7(paths.md, "utf8"), "abstract", cap);
4613
+ if (!sec && /abstract/i.test(section2) && full && existsSync11(paths.md)) {
4614
+ sec = readSection(readFileSync11(paths.md, "utf8"), "abstract", cap);
3956
4615
  via = " (from the abstract file)";
3957
4616
  }
3958
4617
  if (!sec && /declaration/i.test(section2)) {
@@ -3996,15 +4655,16 @@ function registerTools(server2) {
3996
4655
  log("library_verify", pid.requested, "missing adding first");
3997
4656
  await addEntry(id);
3998
4657
  }
3999
- const r = await verifyClaim(claim, id, cfg, cfg ? decider(cfg) : null);
4658
+ const { cached, ...r } = await verifyCached(claim, id, cfg, cfg ? decider(cfg) : null, FH_LIBRARY_DIR);
4659
+ if (!r.error && r.score !== null) bump("verify", cached);
4000
4660
  const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
4001
- log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
4002
- return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
4661
+ log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""}${cached ? " cached" : ""} | ${claim}`);
4662
+ return { content: [{ type: "text", text: compact2(r) }] };
4003
4663
  }
4004
4664
  );
4005
4665
  server2.tool(
4006
4666
  "library_info",
4007
- "The whole library as JSON: library directory and host; entry count and size on disk; per type (pmid, doi, dailymed, nct) the entries by level (full_text, abstract, title) and retractions; counts of retracted and not-citable entries and of errata, expressions of concern and updates; the retrieval sources in run order and which are off (tiers); the download ladder (download): the Playwright servers with the version they must run, the Chrome rungs, the Bright Data unlocker; the configuration and where each value came from \u2014 environment, .env or default, never the value (config); the decision model behind library_verify and whether it answers right now (verify). ids=true also lists every entry ID.",
4667
+ "The whole library as JSON: library directory and host; entry count and size on disk; per type (pmid, doi, dailymed, nct) the entries by level (full_text, abstract, title) and retractions; counts of retracted and not-citable entries and of errata, expressions of concern and updates; the retrieval sources in run order and which are off (tiers); the download ladder (download): the Playwright servers with the version they must run, the Chrome rungs, the Bright Data unlocker; the configuration and where each value came from \u2014 environment, .env or default, never the value (config); the decision model behind library_verify and whether it answers right now (verify); this server's cache hits and misses for library_verify and library_fetch since it started (cache). ids=true also lists every entry ID.",
4008
4668
  { ids: z.boolean().optional().describe("Also list every entry ID (type:value).") },
4009
4669
  async ({ ids }) => {
4010
4670
  const info = libraryInfo(FH_LIBRARY_DIR, HOST, ids === true);
@@ -4019,7 +4679,110 @@ function registerTools(server2) {
4019
4679
  }
4020
4680
  }
4021
4681
  log("library_info", null, `entries=${info.entries}${info.verify ? ` verify ${info.verify.model} ${info.verify.ok ? `ok ${info.verify.ms} ms` : `down: ${info.verify.error}`}` : " verify not configured"}`);
4022
- return { content: [{ type: "text", text: JSON.stringify(info, null, 2) }] };
4682
+ const s = currentStats();
4683
+ const cache = s ? { since: s.started, verify: s.verify, fetch: s.fetch, search: s.search } : null;
4684
+ return { content: [{ type: "text", text: compact2({ ...info, cache }) }] };
4685
+ }
4686
+ );
4687
+ server2.tool(
4688
+ "library_fetch",
4689
+ `Fetch a web page as Markdown: the standard fetch tool, cached and hardened. Use it for any URL instead of a plain fetch or a browser. A page fetched in the last ${settings().refresh_after_days} days is answered from disk in milliseconds; a page a plain fetch cannot get (403, rate limit, bot wall, script-rendered) is fetched through the configured browsers and unlocker in this one call. PDFs come back as Markdown. To verify a link, title_only returns just its title line.`,
4690
+ {
4691
+ url: z.string().describe("URL to fetch"),
4692
+ max_length: z.number().int().min(1).max(999999).default(5e3).describe("Maximum number of characters to return."),
4693
+ start_index: z.number().int().min(0).default(0).describe("On return output starting at this character index, useful if a previous fetch was truncated and more context is required."),
4694
+ raw: z.boolean().default(false).describe("Get the actual HTML content of the requested page, without simplification."),
4695
+ title_only: z.boolean().default(false).describe("Return only the page's title line. The whole page is still fetched and stored; use this to verify that a link resolves and what it points to.")
4696
+ },
4697
+ async ({ url, max_length, start_index, raw, title_only }) => {
4698
+ const t0 = Date.now();
4699
+ const id = logId(url), put = (line) => log("library_fetch", id, `${line} ${url}`);
4700
+ const r = await libraryFetch(FH_LIBRARY_DIR, url, { max_length, start_index, raw, title_only }, `${pkg.name} ${pkg.version}`, {
4701
+ onTry: put,
4702
+ onAlert: (line) => put(`!! ${line}`)
4703
+ });
4704
+ const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
4705
+ if (r.meta) bump("fetch", r.cached);
4706
+ if (r.ok) {
4707
+ put(`${r.cached ? "cached" : `fetched ${secs}`} ${r.meta.format} retrieved_by=${r.meta.retrieved_by} fetched=${r.meta.fetched} chars=${r.text.length}${raw ? " raw" : ""}${title_only ? " title" : ""}${start_index ? ` start=${start_index}` : ""}`);
4708
+ return { content: [{ type: "text", text: r.text }] };
4709
+ }
4710
+ put(`${r.cached ? "cached failure" : `failed ${secs}`}: ${r.error.replace(url, "<url>")}`);
4711
+ return { content: [{ type: "text", text: r.error }], isError: true };
4712
+ }
4713
+ );
4714
+ server2.tool(
4715
+ "library_clinicaltrials_search_studies",
4716
+ `Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment, start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${SEARCH_AFTER_DAYS} days. One trial in detail: library_clinicaltrials_get_study_record.`,
4717
+ {
4718
+ query: z.string().optional().describe("Free-text search across all fields (title, conditions, interventions, summary, \u2026); AND / OR / NOT allowed."),
4719
+ intervention: z.string().optional().describe('Intervention or treatment, e.g. "tadalafil", "cognitive behavioral therapy".'),
4720
+ condition: z.string().optional().describe('Condition or disease, e.g. "hypertension", "type 2 diabetes".'),
4721
+ status: z.enum(["ongoing", "completed", "all"]).default("all").describe("ongoing: recruiting, not yet recruiting, active not recruiting, enrolling by invitation. completed: completed. all: no filter."),
4722
+ sort: z.enum(["relevance", "newest", "largest", "completion"]).default("relevance").describe("relevance (the registry's ranking), newest (last update first), largest (enrollment first), completion (completion date, latest first)."),
4723
+ max_results: z.number().int().min(1).max(MAX_RESULTS).default(20).describe("Rows to return (1\u2013100).")
4724
+ },
4725
+ async (args) => {
4726
+ const t0 = Date.now();
4727
+ const r = await searchTrials(args, FH_LIBRARY_DIR);
4728
+ const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
4729
+ if (!r.ok) {
4730
+ log("library_search", "ct:-", `error: ${r.error}`);
4731
+ return { content: [{ type: "text", text: compact2({ error: r.error }) }], isError: true };
4732
+ }
4733
+ bump("search", r.cached);
4734
+ const q = r.result.query, parts = [q.intervention && `intervention=${q.intervention}`, q.condition && `condition=${q.condition}`, q.query && `query=${q.query}`].filter(Boolean).join(" ");
4735
+ log("library_search", `ct:${trialsLogId(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total} status=${q.status}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""} | ${parts}`);
4736
+ return { content: [{ type: "text", text: leanJson(r.result) }] };
4737
+ }
4738
+ );
4739
+ const nctEntry = async (id) => {
4740
+ const p = parseId(/^\d{8}$/.test(id.trim()) ? `nct:${id.trim()}` : id);
4741
+ if (p.error) return { error: p.error };
4742
+ if (p.type !== "nct") return { error: `not a trial ID: ${p.requested}` };
4743
+ const paths = entryPaths(FH_LIBRARY_DIR, p.type, p.id);
4744
+ if (!entryExists(paths)) {
4745
+ const r = await addEntry(p.requested);
4746
+ const s = JSON.parse(r.content[0].text);
4747
+ if (s.error) return { error: s.error };
4748
+ }
4749
+ return { paths, requested: p.requested };
4750
+ };
4751
+ server2.tool(
4752
+ "library_clinicaltrials_get_study_record",
4753
+ 'One ClinicalTrials.gov trial as a review needs it: titles, status and why stopped, phase, design, enrollment, dates, sponsor, conditions, interventions, primary outcomes, eligibility, sites, result publications as pmid: IDs, has_results, brief summary. From the stored record; a missing trial is added first. The whole record: library_read("nct:\u2026").',
4754
+ { nct: z.string().describe('The NCT ID: "nct:05173896", "NCT05173896" or the eight digits.') },
4755
+ async ({ nct }) => {
4756
+ const t0 = Date.now();
4757
+ const e = await nctEntry(nct);
4758
+ if ("error" in e) {
4759
+ log("library_trials", nct, `record error: ${e.error}`);
4760
+ return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
4761
+ }
4762
+ const raw = e.paths.fulltextRaw("json");
4763
+ const r = existsSync11(raw) ? recordSummary(readFileSync11(raw, "utf8")) : null;
4764
+ if (!r) {
4765
+ log("library_trials", e.requested, "record error: no stored record");
4766
+ return { content: [{ type: "text", text: compact2({ error: "no stored record" }) }], isError: true };
4767
+ }
4768
+ log("library_trials", e.requested, `record ${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s ${r.status ?? ""}${r.has_results ? " results" : ""} | ${r.title}`);
4769
+ return { content: [{ type: "text", text: leanJson(r) }] };
4770
+ }
4771
+ );
4772
+ server2.tool(
4773
+ "library_clinicaltrials_get_study_results",
4774
+ 'The posted results of one ClinicalTrials.gov trial \u2014 participant flow, baseline, every outcome measure as a table, adverse events \u2014 as Markdown from the stored record, capped at max_chars, outcome titles listed; library_read("nct:\u2026", section: <title>) returns one measure whole. has_results false when the registry holds none. A missing trial is added first.',
4775
+ { nct: z.string().describe('The NCT ID: "nct:05173896", "NCT05173896" or the eight digits.'), max_chars: z.number().int().min(200).max(6e4).default(6e3).describe("Cap for the results text (default 6000).") },
4776
+ async ({ nct, max_chars }) => {
4777
+ const e = await nctEntry(nct);
4778
+ if ("error" in e) {
4779
+ log("library_trials", nct, `results error: ${e.error}`);
4780
+ return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
4781
+ }
4782
+ const md = existsSync11(e.paths.fulltextMd) ? readFileSync11(e.paths.fulltextMd, "utf8") : "";
4783
+ const r = resultsText(e.requested.replace(/^nct:/, "NCT"), md, max_chars);
4784
+ log("library_trials", e.requested, `results ${r.has_results ? `${r.outcomes.length} outcomes${r.truncated ? " truncated" : ""}` : "none"}`);
4785
+ return { content: [{ type: "text", text: leanJson(r) }] };
4023
4786
  }
4024
4787
  );
4025
4788
  server2.tool(
@@ -4039,6 +4802,8 @@ var server = new McpServer(
4039
4802
  { instructions: INSTRUCTIONS }
4040
4803
  );
4041
4804
  registerTools(server);
4805
+ startStats(pkg);
4806
+ log("startup", null, `${pkg.name} ${pkg.version}`);
4042
4807
  for (const problem of resolveTiers().problems) log("startup", null, `!! ${problem}`);
4043
4808
  var transport = new StdioServerTransport();
4044
4809
  await server.connect(transport);