fh-library 0.2.5 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +15 -3
- package/dist/index.js +840 -75
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -11,10 +11,10 @@ function dotenvFile(env = process.env) {
|
|
|
11
11
|
return join(resolve(lib), ".env");
|
|
12
12
|
}
|
|
13
13
|
var loadedFromDotenv = /* @__PURE__ */ new Set();
|
|
14
|
-
function loadDotenv(
|
|
15
|
-
if (!existsSync(
|
|
14
|
+
function loadDotenv(file2 = dotenvFile()) {
|
|
15
|
+
if (!existsSync(file2)) return [];
|
|
16
16
|
const loaded = [];
|
|
17
|
-
for (const raw of readFileSync(
|
|
17
|
+
for (const raw of readFileSync(file2, "utf8").split(/\r?\n/)) {
|
|
18
18
|
const line = raw.trim();
|
|
19
19
|
if (!line || line.startsWith("#")) continue;
|
|
20
20
|
const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
@@ -60,6 +60,11 @@ var contactEmail = (env = process.env) => str("FH_LIBRARY_CONTACT_EMAIL", env) |
|
|
|
60
60
|
// src/http.ts
|
|
61
61
|
var TIMEOUT_MS = 3e4;
|
|
62
62
|
var CHROME_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
|
63
|
+
var BROWSER_HEADERS = {
|
|
64
|
+
"User-Agent": CHROME_UA,
|
|
65
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
66
|
+
"Accept-Language": "en-US,en;q=0.9"
|
|
67
|
+
};
|
|
63
68
|
var LIBRARY_UA = `fh-library (+https://github.com/forever-healthy/fh-library; ${contactEmail()})`;
|
|
64
69
|
function httpReason(status, body) {
|
|
65
70
|
const m = body.length < 4e3 && /<statusCode>([A-Z_]+)<\/statusCode>|"statusCode"\s*:\s*"([A-Z_]+)"/.exec(body);
|
|
@@ -655,20 +660,20 @@ function headingOf(line) {
|
|
|
655
660
|
if (bare2.length > 60 || !/^[A-Za-z][A-Za-z &'\u2019]*$/.test(bare2)) return null;
|
|
656
661
|
return BY_SQUEEZED.get(squeeze(bare2)) ?? null;
|
|
657
662
|
}
|
|
658
|
-
async function pdfToMarkdown(bytes, title, headLine2,
|
|
663
|
+
async function pdfToMarkdown(bytes, title, headLine2, stats2) {
|
|
659
664
|
const pdf = await getDocumentProxy(new Uint8Array(bytes), { fontExtraProperties: true });
|
|
660
665
|
const text3 = await extractPages(pdf);
|
|
661
|
-
if (
|
|
662
|
-
|
|
663
|
-
|
|
666
|
+
if (stats2) {
|
|
667
|
+
stats2.pages = pdf.numPages;
|
|
668
|
+
stats2.chars = text3.replace(/\s+/g, " ").trim().length;
|
|
664
669
|
}
|
|
665
670
|
const lines = text3.split(/\r?\n/).map((l) => l.replace(/\s+/g, " ").trim());
|
|
666
671
|
const out = [`# ${title}`, "", headLine2, ""];
|
|
667
|
-
let
|
|
672
|
+
let para3 = [];
|
|
668
673
|
const flush = () => {
|
|
669
|
-
if (
|
|
670
|
-
out.push(
|
|
671
|
-
|
|
674
|
+
if (para3.length) {
|
|
675
|
+
out.push(para3.join(" "), "");
|
|
676
|
+
para3 = [];
|
|
672
677
|
}
|
|
673
678
|
};
|
|
674
679
|
for (const line of lines) {
|
|
@@ -682,7 +687,7 @@ async function pdfToMarkdown(bytes, title, headLine2, stats) {
|
|
|
682
687
|
out.push(`## ${h}`, "");
|
|
683
688
|
continue;
|
|
684
689
|
}
|
|
685
|
-
|
|
690
|
+
para3.push(line);
|
|
686
691
|
}
|
|
687
692
|
flush();
|
|
688
693
|
return fixGlyphs(out.join("\n")).replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
@@ -875,9 +880,9 @@ function dehyphenate(text3) {
|
|
|
875
880
|
return text3.replace(/([A-Za-z]+)\u0000\n([A-Za-z]+)/g, (m, x, y) => {
|
|
876
881
|
if (!/^[a-z]/.test(y)) return `${x}-${y}`;
|
|
877
882
|
if (SUSPENDED.has(y.toLowerCase())) return `${x}- ${y}`;
|
|
878
|
-
const hyph = `${x}-${y}`.toLowerCase(),
|
|
883
|
+
const hyph = `${x}-${y}`.toLowerCase(), joined3 = `${x}${y}`.toLowerCase();
|
|
879
884
|
if (seen(hyph)) return `${x}-${y}`;
|
|
880
|
-
if (seen(
|
|
885
|
+
if (seen(joined3)) return `${x}${y}`;
|
|
881
886
|
return COMPOUND_PREFIX.has(x.toLowerCase()) ? `${x}-${y}` : `${x}${y}`;
|
|
882
887
|
}).replace(/\u0000\n/g, "-\n").replace(/\u0000/g, "-");
|
|
883
888
|
}
|
|
@@ -1000,6 +1005,8 @@ var NAV_TIMEOUT_MS = 2e4;
|
|
|
1000
1005
|
var CHALLENGE_WAIT_MS = 8e3;
|
|
1001
1006
|
var DOWNLOAD_WAIT_MS = 1e4;
|
|
1002
1007
|
var IDLE_CLOSE_MS = 12e4;
|
|
1008
|
+
var SETTLE_MS = 8e3;
|
|
1009
|
+
var SETTLE_WORDS = 100;
|
|
1003
1010
|
var ctxs = /* @__PURE__ */ new Map();
|
|
1004
1011
|
var idleTimer = null;
|
|
1005
1012
|
var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
|
|
@@ -1141,6 +1148,12 @@ async function browserFetch(url, endpoint, cfg = browserConfig()) {
|
|
|
1141
1148
|
if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
|
|
1142
1149
|
return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
|
|
1143
1150
|
}
|
|
1151
|
+
const words2 = () => page.evaluate(() => (document.body?.innerText ?? "").split(/\s+/).filter(Boolean).length).catch(() => 0);
|
|
1152
|
+
if (await words2() < SETTLE_WORDS) {
|
|
1153
|
+
await page.waitForLoadState("networkidle", { timeout: SETTLE_MS }).catch(() => void 0);
|
|
1154
|
+
const t2 = Date.now();
|
|
1155
|
+
while (Date.now() - t2 < SETTLE_MS && await words2() < SETTLE_WORDS) await new Promise((r) => setTimeout(r, 1e3));
|
|
1156
|
+
}
|
|
1144
1157
|
const html = await page.content();
|
|
1145
1158
|
return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
|
|
1146
1159
|
} finally {
|
|
@@ -1150,14 +1163,39 @@ async function browserFetch(url, endpoint, cfg = browserConfig()) {
|
|
|
1150
1163
|
}
|
|
1151
1164
|
|
|
1152
1165
|
// src/unlock.ts
|
|
1166
|
+
import { request as httpsRequest } from "https";
|
|
1153
1167
|
var ENDPOINT = "https://api.brightdata.com/request";
|
|
1154
1168
|
var TIMEOUT_MS2 = 6e4;
|
|
1169
|
+
function post(url, init) {
|
|
1170
|
+
return new Promise((resolve3, reject) => {
|
|
1171
|
+
const req = httpsRequest(url, { method: "POST", headers: { ...init.headers, "Content-Length": String(Buffer.byteLength(init.body)) }, signal: init.signal }, (res) => {
|
|
1172
|
+
const chunks = [];
|
|
1173
|
+
res.on("data", (c) => chunks.push(c));
|
|
1174
|
+
res.on("error", reject);
|
|
1175
|
+
res.on("end", () => {
|
|
1176
|
+
const body = Buffer.concat(chunks);
|
|
1177
|
+
const status = res.statusCode ?? 0;
|
|
1178
|
+
resolve3({
|
|
1179
|
+
ok: status >= 200 && status < 300,
|
|
1180
|
+
status,
|
|
1181
|
+
headers: { get: (name) => {
|
|
1182
|
+
const v = res.headers[name.toLowerCase()];
|
|
1183
|
+
return v == null ? null : Array.isArray(v) ? v.join(", ") : v;
|
|
1184
|
+
} },
|
|
1185
|
+
arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength)
|
|
1186
|
+
});
|
|
1187
|
+
});
|
|
1188
|
+
});
|
|
1189
|
+
req.on("error", (e) => reject(e.name === "AbortError" || e.name === "TimeoutError" ? new Error("timeout") : e));
|
|
1190
|
+
req.end(init.body);
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1155
1193
|
var unlockerConfigured = () => !!process.env.FH_LIBRARY_BRIGHTDATA_API_TOKEN?.trim();
|
|
1156
|
-
async function unlock(url,
|
|
1194
|
+
async function unlock(url, postImpl = post) {
|
|
1157
1195
|
const token = process.env.FH_LIBRARY_BRIGHTDATA_API_TOKEN?.trim();
|
|
1158
1196
|
if (!token) throw new Error("unlocker not configured");
|
|
1159
1197
|
const zone = process.env.FH_LIBRARY_BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
|
|
1160
|
-
const res = await
|
|
1198
|
+
const res = await postImpl(ENDPOINT, {
|
|
1161
1199
|
method: "POST",
|
|
1162
1200
|
headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
|
|
1163
1201
|
body: JSON.stringify({ zone, url, format: "raw" }),
|
|
@@ -1264,13 +1302,13 @@ async function convertBytes(p, bytes, contentType = "") {
|
|
|
1264
1302
|
let md;
|
|
1265
1303
|
if (looksLikePdf(bytes)) {
|
|
1266
1304
|
format = "pdf";
|
|
1267
|
-
const
|
|
1305
|
+
const stats2 = {};
|
|
1268
1306
|
try {
|
|
1269
|
-
md = await pdfToMarkdown(bytes, p.title, p.headLine,
|
|
1307
|
+
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats2);
|
|
1270
1308
|
} catch (e) {
|
|
1271
1309
|
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
1272
1310
|
}
|
|
1273
|
-
if ((
|
|
1311
|
+
if ((stats2.pages ?? 0) >= 1 && (stats2.chars ?? 0) < 500 * (stats2.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
1274
1312
|
} else {
|
|
1275
1313
|
const text3 = bytes.toString("utf8");
|
|
1276
1314
|
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
@@ -1462,15 +1500,15 @@ function inboxTier(dir = INBOX_DIR) {
|
|
|
1462
1500
|
always: true,
|
|
1463
1501
|
applies: (p) => p.pmid ? true : "no pmid",
|
|
1464
1502
|
async run(p, ctx) {
|
|
1465
|
-
const
|
|
1466
|
-
if (!
|
|
1467
|
-
const name =
|
|
1503
|
+
const file2 = findInboxFile(dir, p.pmid);
|
|
1504
|
+
if (!file2) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
|
|
1505
|
+
const name = file2.slice(file2.lastIndexOf("/") + 1);
|
|
1468
1506
|
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
1469
|
-
const bytes = readFileSync3(
|
|
1507
|
+
const bytes = readFileSync3(file2);
|
|
1470
1508
|
const r = await convertBytes(p, bytes, EXT[ext]);
|
|
1471
1509
|
if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
|
|
1472
1510
|
ctx.onTry?.(`${name} ok, removed from the inbox`);
|
|
1473
|
-
unlinkSync(
|
|
1511
|
+
unlinkSync(file2);
|
|
1474
1512
|
return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
|
|
1475
1513
|
}
|
|
1476
1514
|
};
|
|
@@ -1743,24 +1781,24 @@ function stamp() {
|
|
|
1743
1781
|
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
1744
1782
|
}
|
|
1745
1783
|
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
1746
|
-
function rotateIfLarge(
|
|
1784
|
+
function rotateIfLarge(file2, max = LOG_MAX_BYTES) {
|
|
1747
1785
|
let size = 0;
|
|
1748
1786
|
try {
|
|
1749
|
-
size = statSync(
|
|
1787
|
+
size = statSync(file2).size;
|
|
1750
1788
|
} catch {
|
|
1751
1789
|
return false;
|
|
1752
1790
|
}
|
|
1753
1791
|
if (size <= max) return false;
|
|
1754
1792
|
try {
|
|
1755
|
-
renameSync(
|
|
1793
|
+
renameSync(file2, `${file2}.1`);
|
|
1756
1794
|
return true;
|
|
1757
1795
|
} catch {
|
|
1758
1796
|
return false;
|
|
1759
1797
|
}
|
|
1760
1798
|
}
|
|
1761
1799
|
function log(tool, id, message) {
|
|
1762
|
-
const
|
|
1763
|
-
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${
|
|
1800
|
+
const clean2 = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
1801
|
+
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean2}
|
|
1764
1802
|
`;
|
|
1765
1803
|
try {
|
|
1766
1804
|
mkdirSync2(FH_LIBRARY_DIR, { recursive: true });
|
|
@@ -1770,9 +1808,9 @@ function log(tool, id, message) {
|
|
|
1770
1808
|
}
|
|
1771
1809
|
}
|
|
1772
1810
|
|
|
1773
|
-
// src/
|
|
1774
|
-
import {
|
|
1775
|
-
import {
|
|
1811
|
+
// src/stats.ts
|
|
1812
|
+
import { mkdirSync as mkdirSync4, readdirSync as readdirSync3, readFileSync as readFileSync5, statSync as statSync2, unlinkSync as unlinkSync2 } from "fs";
|
|
1813
|
+
import { join as join7 } from "path";
|
|
1776
1814
|
|
|
1777
1815
|
// src/entry.ts
|
|
1778
1816
|
import { copyFileSync, existsSync as existsSync5, mkdirSync as mkdirSync3, readdirSync as readdirSync2, readFileSync as readFileSync4, renameSync as renameSync2, rmSync, writeFileSync } from "fs";
|
|
@@ -2591,10 +2629,10 @@ function readMeta(p) {
|
|
|
2591
2629
|
attempts: splitAttempts(fm.attempts)
|
|
2592
2630
|
};
|
|
2593
2631
|
}
|
|
2594
|
-
function writeAtomic(
|
|
2595
|
-
const tmp = `${
|
|
2632
|
+
function writeAtomic(file2, data) {
|
|
2633
|
+
const tmp = `${file2}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
2596
2634
|
writeFileSync(tmp, data);
|
|
2597
|
-
renameSync2(tmp,
|
|
2635
|
+
renameSync2(tmp, file2);
|
|
2598
2636
|
}
|
|
2599
2637
|
function writeMeta(p, m) {
|
|
2600
2638
|
writeAtomic(p.meta, renderMeta(m));
|
|
@@ -2873,6 +2911,67 @@ function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", record
|
|
|
2873
2911
|
return meta;
|
|
2874
2912
|
}
|
|
2875
2913
|
|
|
2914
|
+
// src/stats.ts
|
|
2915
|
+
var HEARTBEAT_MS = 5 * 6e4;
|
|
2916
|
+
var STALE_MS = 20 * 6e4;
|
|
2917
|
+
var stats = null;
|
|
2918
|
+
var file = null;
|
|
2919
|
+
function startStats(pkg2, libraryDir = FH_LIBRARY_DIR) {
|
|
2920
|
+
stats = { host: HOST, pid: process.pid, package: pkg2.name, version: pkg2.version, started: (/* @__PURE__ */ new Date()).toISOString(), verify: { hits: 0, misses: 0 }, fetch: { hits: 0, misses: 0 }, search: { hits: 0, misses: 0 } };
|
|
2921
|
+
file = join7(libraryDir, ".servers", `${HOST}-${process.pid}.json`);
|
|
2922
|
+
sweep(join7(libraryDir, ".servers"));
|
|
2923
|
+
publish();
|
|
2924
|
+
setInterval(publish, HEARTBEAT_MS).unref();
|
|
2925
|
+
const stop = () => {
|
|
2926
|
+
try {
|
|
2927
|
+
if (file) unlinkSync2(file);
|
|
2928
|
+
} catch {
|
|
2929
|
+
}
|
|
2930
|
+
};
|
|
2931
|
+
process.on("exit", stop);
|
|
2932
|
+
for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) process.on(sig, () => process.exit(0));
|
|
2933
|
+
return stats;
|
|
2934
|
+
}
|
|
2935
|
+
function sweep(dir, staleMs = STALE_MS) {
|
|
2936
|
+
let n = 0;
|
|
2937
|
+
let names;
|
|
2938
|
+
try {
|
|
2939
|
+
names = readdirSync3(dir);
|
|
2940
|
+
} catch {
|
|
2941
|
+
return 0;
|
|
2942
|
+
}
|
|
2943
|
+
for (const name of names) {
|
|
2944
|
+
if (!name.endsWith(".json")) continue;
|
|
2945
|
+
const p = join7(dir, name);
|
|
2946
|
+
try {
|
|
2947
|
+
if (Date.now() - statSync2(p).mtimeMs > staleMs) {
|
|
2948
|
+
unlinkSync2(p);
|
|
2949
|
+
n++;
|
|
2950
|
+
}
|
|
2951
|
+
} catch {
|
|
2952
|
+
}
|
|
2953
|
+
}
|
|
2954
|
+
return n;
|
|
2955
|
+
}
|
|
2956
|
+
function bump(kind, hit) {
|
|
2957
|
+
if (!stats) return;
|
|
2958
|
+
stats[kind][hit ? "hits" : "misses"]++;
|
|
2959
|
+
publish();
|
|
2960
|
+
}
|
|
2961
|
+
var currentStats = () => stats;
|
|
2962
|
+
function publish() {
|
|
2963
|
+
if (!stats || !file) return;
|
|
2964
|
+
try {
|
|
2965
|
+
mkdirSync4(join7(file, ".."), { recursive: true });
|
|
2966
|
+
writeAtomic(file, JSON.stringify(stats, null, 2) + "\n");
|
|
2967
|
+
} catch {
|
|
2968
|
+
}
|
|
2969
|
+
}
|
|
2970
|
+
|
|
2971
|
+
// src/tools.ts
|
|
2972
|
+
import { existsSync as existsSync11, readFileSync as readFileSync11 } from "fs";
|
|
2973
|
+
import { z } from "zod";
|
|
2974
|
+
|
|
2876
2975
|
// src/cascade.ts
|
|
2877
2976
|
var PERMANENT = /^(no pmcid|no doi|no pmid|no pmid or doi|no body|closed|no oa copy|no free copy|not in core|no full text|xml without body|template has no placeholder|title missing|title not in text|no methods\/results|no article body|scanned pdf|not a pdf|only copies already tried|only pmc copies|nothing located|unknown content|no text-mining links|needs [a-z/]+ token)/i;
|
|
2878
2977
|
var BLOCKED = /^(challenge not cleared|empty response)/i;
|
|
@@ -2945,11 +3044,11 @@ function parseId(raw) {
|
|
|
2945
3044
|
}
|
|
2946
3045
|
|
|
2947
3046
|
// src/info.ts
|
|
2948
|
-
import { existsSync as existsSync7, readdirSync as
|
|
2949
|
-
import { dirname as dirname2, join as
|
|
3047
|
+
import { existsSync as existsSync7, readdirSync as readdirSync4, readFileSync as readFileSync7, statSync as statSync3 } from "fs";
|
|
3048
|
+
import { dirname as dirname2, join as join8 } from "path";
|
|
2950
3049
|
|
|
2951
3050
|
// src/verify.ts
|
|
2952
|
-
import { existsSync as existsSync6, readFileSync as
|
|
3051
|
+
import { existsSync as existsSync6, readFileSync as readFileSync6 } from "fs";
|
|
2953
3052
|
var DEFAULT_MODEL = "pplx-decider-flash-v1.2";
|
|
2954
3053
|
var QUESTION = "Does the paper state this, in substance? Ignore section labels and definitions of terms in parentheses. Every fact and number of the statement must be stated in the paper or follow from its numbers; a rounded number counts. If any part of the statement is absent from the paper or contradicted by it, answer false.";
|
|
2955
3054
|
function verifyConfig(env = process.env) {
|
|
@@ -3013,7 +3112,8 @@ function citationGroups(claim) {
|
|
|
3013
3112
|
function fragment(claim) {
|
|
3014
3113
|
const body = claim.trim().replace(/^[-*+]\s+/, "").replace(/^\*\*[^*]+:\*\*\s*/, "").replace(/^[("'\u201c]+/, "");
|
|
3015
3114
|
const first = /^(\S+)/.exec(body)?.[1] ?? "";
|
|
3016
|
-
|
|
3115
|
+
const oddsRatio = first === "OR" && /^OR\s+(\d|=|\(|of\b|for\b)/.test(body);
|
|
3116
|
+
if (!oddsRatio && /^(and|or|but|nor|which|whereas|while)$/i.test(first.replace(/[,;:]$/, ""))) return `claim is a fragment: starts with "${first}" \u2014 send a complete statement with its subject`;
|
|
3017
3117
|
if (/^[a-z]/.test(body)) return `claim is a fragment: starts with "${first}" in lowercase \u2014 send a complete statement with its subject`;
|
|
3018
3118
|
return null;
|
|
3019
3119
|
}
|
|
@@ -3051,9 +3151,9 @@ async function readPaper(raw, libraryDir = FH_LIBRARY_DIR) {
|
|
|
3051
3151
|
const paths = entryPaths(libraryDir, p.type, p.id);
|
|
3052
3152
|
if (!entryExists(paths)) return { id: p.requested, error: "not in library" };
|
|
3053
3153
|
const full = existsSync6(paths.fulltextMd);
|
|
3054
|
-
if (full) return { id: p.requested, basis: "full_text", text:
|
|
3154
|
+
if (full) return { id: p.requested, basis: "full_text", text: readFileSync6(paths.fulltextMd, "utf8") };
|
|
3055
3155
|
const meta = readMeta(paths);
|
|
3056
|
-
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text:
|
|
3156
|
+
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync6(paths.md, "utf8") };
|
|
3057
3157
|
}
|
|
3058
3158
|
async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
3059
3159
|
const out = { claim, id, verdict: "error", reason: null, score: null, basis: null, windows: 0, glyphs_unresolved: 0, superseded: null, model: cfg?.model ?? null };
|
|
@@ -3095,9 +3195,9 @@ async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
|
3095
3195
|
// src/info.ts
|
|
3096
3196
|
function* metaFiles(dir) {
|
|
3097
3197
|
if (!existsSync7(dir)) return;
|
|
3098
|
-
for (const name of
|
|
3099
|
-
const p =
|
|
3100
|
-
if (
|
|
3198
|
+
for (const name of readdirSync4(dir)) {
|
|
3199
|
+
const p = join8(dir, name);
|
|
3200
|
+
if (statSync3(p).isDirectory()) yield* metaFiles(p);
|
|
3101
3201
|
else if (name.endsWith("-meta.md")) yield p;
|
|
3102
3202
|
}
|
|
3103
3203
|
}
|
|
@@ -3135,11 +3235,11 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3135
3235
|
updates: 0
|
|
3136
3236
|
};
|
|
3137
3237
|
const ids = [];
|
|
3138
|
-
for (const type of existsSync7(libraryDir) ?
|
|
3139
|
-
const typeDir =
|
|
3140
|
-
if (!
|
|
3238
|
+
for (const type of existsSync7(libraryDir) ? readdirSync4(libraryDir) : []) {
|
|
3239
|
+
const typeDir = join8(libraryDir, type);
|
|
3240
|
+
if (!statSync3(typeDir).isDirectory()) continue;
|
|
3141
3241
|
for (const meta of metaFiles(typeDir)) {
|
|
3142
|
-
const md =
|
|
3242
|
+
const md = readFileSync7(meta, "utf8");
|
|
3143
3243
|
const raw = field(md, "level") || "unknown";
|
|
3144
3244
|
const level = raw === "abstract" && field(md, "has_abstract") === "no" ? "title" : raw;
|
|
3145
3245
|
info.entries++;
|
|
@@ -3149,9 +3249,9 @@ function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
|
3149
3249
|
t.entries++;
|
|
3150
3250
|
if (level === "full_text" || level === "abstract" || level === "title") t[level]++;
|
|
3151
3251
|
const dir = dirname2(meta);
|
|
3152
|
-
for (const f of
|
|
3252
|
+
for (const f of readdirSync4(dir)) {
|
|
3153
3253
|
try {
|
|
3154
|
-
const st =
|
|
3254
|
+
const st = statSync3(join8(dir, f));
|
|
3155
3255
|
if (st.isFile()) info.size_bytes += st.size;
|
|
3156
3256
|
} catch {
|
|
3157
3257
|
}
|
|
@@ -3180,7 +3280,7 @@ function noticeTypes(md, metaPath) {
|
|
|
3180
3280
|
return out;
|
|
3181
3281
|
}
|
|
3182
3282
|
try {
|
|
3183
|
-
const xml =
|
|
3283
|
+
const xml = readFileSync7(metaPath.replace(/-meta\.md$/, "-abstract.xml"), "utf8");
|
|
3184
3284
|
for (const m of xml.matchAll(/<CommentsCorrections RefType="([A-Za-z]+)"/g)) if (XML_TYPES[m[1]]) out.add(XML_TYPES[m[1]]);
|
|
3185
3285
|
} catch {
|
|
3186
3286
|
}
|
|
@@ -3329,10 +3429,10 @@ function paragraphs(md, scope) {
|
|
|
3329
3429
|
return h;
|
|
3330
3430
|
};
|
|
3331
3431
|
const out = [];
|
|
3332
|
-
let
|
|
3432
|
+
let para3 = [], paraStart = 0, header = null;
|
|
3333
3433
|
const flush = () => {
|
|
3334
|
-
const text3 =
|
|
3335
|
-
|
|
3434
|
+
const text3 = para3.join("\n").trim();
|
|
3435
|
+
para3 = [];
|
|
3336
3436
|
if (text3) out.push({ section: sectionAt(paraStart), text: text3 });
|
|
3337
3437
|
};
|
|
3338
3438
|
for (let i = from; i < to; i++) {
|
|
@@ -3353,8 +3453,8 @@ function paragraphs(md, scope) {
|
|
|
3353
3453
|
continue;
|
|
3354
3454
|
}
|
|
3355
3455
|
header = null;
|
|
3356
|
-
if (!
|
|
3357
|
-
|
|
3456
|
+
if (!para3.length) paraStart = i;
|
|
3457
|
+
para3.push(l);
|
|
3358
3458
|
}
|
|
3359
3459
|
flush();
|
|
3360
3460
|
return out;
|
|
@@ -3557,15 +3657,567 @@ function parseCsl(json2) {
|
|
|
3557
3657
|
};
|
|
3558
3658
|
}
|
|
3559
3659
|
|
|
3660
|
+
// src/urlfetch.ts
|
|
3661
|
+
import { createHash as createHash2 } from "crypto";
|
|
3662
|
+
import { existsSync as existsSync8, mkdirSync as mkdirSync5, readFileSync as readFileSync8 } from "fs";
|
|
3663
|
+
import { join as join9 } from "path";
|
|
3664
|
+
var TRACKING = /^(utm_[a-z]+|fbclid|gclid|dclid|msclkid|mc_cid|mc_eid|igshid|ref_src)$/i;
|
|
3665
|
+
function normalizeUrl(raw) {
|
|
3666
|
+
let u;
|
|
3667
|
+
try {
|
|
3668
|
+
u = new URL(raw.trim());
|
|
3669
|
+
} catch {
|
|
3670
|
+
return null;
|
|
3671
|
+
}
|
|
3672
|
+
if (u.protocol !== "http:" && u.protocol !== "https:") return null;
|
|
3673
|
+
u.hash = "";
|
|
3674
|
+
u.hostname = u.hostname.toLowerCase();
|
|
3675
|
+
if ([...u.searchParams.keys()].some((k) => TRACKING.test(k))) {
|
|
3676
|
+
for (const k of [...u.searchParams.keys()]) if (TRACKING.test(k)) u.searchParams.delete(k);
|
|
3677
|
+
if (![...u.searchParams.keys()].length) u.search = "";
|
|
3678
|
+
}
|
|
3679
|
+
return u.toString();
|
|
3680
|
+
}
|
|
3681
|
+
var urlHash = (normalized2) => createHash2("sha1").update(normalized2).digest("hex").slice(0, 16);
|
|
3682
|
+
function urlPaths(libraryDir, normalized2) {
|
|
3683
|
+
const hash = urlHash(normalized2);
|
|
3684
|
+
const entry = `url-${hash}`;
|
|
3685
|
+
const entryDir = join9(libraryDir, "url", hash.slice(0, 2), hash.slice(0, 4), hash);
|
|
3686
|
+
return { hash, entry, entryDir, meta: join9(entryDir, `${entry}-meta.md`), md: join9(entryDir, `${entry}-page.md`), raw: (ext) => join9(entryDir, `${entry}-page.${ext}`) };
|
|
3687
|
+
}
|
|
3688
|
+
function renderUrlMeta(m) {
|
|
3689
|
+
const lines = [
|
|
3690
|
+
["url", m.url],
|
|
3691
|
+
["final_url", m.final_url],
|
|
3692
|
+
["title", m.title === null ? "" : JSON.stringify(m.title)],
|
|
3693
|
+
["format", m.format],
|
|
3694
|
+
["content_type", m.content_type],
|
|
3695
|
+
["level", m.level],
|
|
3696
|
+
["retrieved_by", m.retrieved_by],
|
|
3697
|
+
["fetched", m.fetched],
|
|
3698
|
+
["error", m.error ?? ""],
|
|
3699
|
+
["files", m.files.join(", ")],
|
|
3700
|
+
["converted_by", m.converted_by]
|
|
3701
|
+
];
|
|
3702
|
+
return lines.map(([k, v]) => `${k}: ${v}`).join("\n") + "\n";
|
|
3703
|
+
}
|
|
3704
|
+
function readUrlMeta(p) {
|
|
3705
|
+
if (!existsSync8(p.meta)) return null;
|
|
3706
|
+
const md = readFileSync8(p.meta, "utf8");
|
|
3707
|
+
const f = (k) => {
|
|
3708
|
+
const m = new RegExp(`^${k}:[ \\t]*(.*)$`, "m").exec(md);
|
|
3709
|
+
return m ? m[1].trim() : "";
|
|
3710
|
+
};
|
|
3711
|
+
let title = null;
|
|
3712
|
+
try {
|
|
3713
|
+
title = f("title") ? JSON.parse(f("title")) : null;
|
|
3714
|
+
} catch {
|
|
3715
|
+
title = f("title") || null;
|
|
3716
|
+
}
|
|
3717
|
+
return {
|
|
3718
|
+
url: f("url"),
|
|
3719
|
+
final_url: f("final_url"),
|
|
3720
|
+
title,
|
|
3721
|
+
format: f("format") || "failed",
|
|
3722
|
+
content_type: f("content_type"),
|
|
3723
|
+
level: f("level") === "full_text" ? "full_text" : "title",
|
|
3724
|
+
retrieved_by: f("retrieved_by"),
|
|
3725
|
+
fetched: f("fetched"),
|
|
3726
|
+
error: f("error") || null,
|
|
3727
|
+
files: f("files") ? f("files").split(",").map((s) => s.trim()).filter(Boolean) : [],
|
|
3728
|
+
converted_by: f("converted_by")
|
|
3729
|
+
};
|
|
3730
|
+
}
|
|
3731
|
+
function writeUrlMeta(p, m) {
|
|
3732
|
+
mkdirSync5(p.entryDir, { recursive: true });
|
|
3733
|
+
writeAtomic(p.meta, renderUrlMeta(m));
|
|
3734
|
+
}
|
|
3735
|
+
var SHELL_WORDS = 100;
|
|
3736
|
+
var EMPTY_WORDS = 10;
|
|
3737
|
+
var SHELL_PHRASE = /\bloading\b|javascript|enable it|connecting to|please wait|redirecting/i;
|
|
3738
|
+
function isScriptShell(markdownOrHtml) {
|
|
3739
|
+
const text3 = markdownOrHtml.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>|<noscript[\s\S]*?<\/noscript>/gi, " ").replace(/<[^>]+>/g, " ").replace(/&[a-z#0-9]+;/gi, " ").replace(/\]\([^)]*\)/g, ")");
|
|
3740
|
+
const words2 = text3.split(/\s+/).filter((w) => /[\p{L}\p{N}]/u.test(w)).length;
|
|
3741
|
+
return { shell: words2 < EMPTY_WORDS || words2 < SHELL_WORDS && SHELL_PHRASE.test(text3), words: words2 };
|
|
3742
|
+
}
|
|
3743
|
+
var viaName = (via) => typeof via === "object" ? `browser ${via.label}` : via;
|
|
3744
|
+
async function rung(url, via) {
|
|
3745
|
+
const name = viaName(via);
|
|
3746
|
+
let got;
|
|
3747
|
+
try {
|
|
3748
|
+
got = via === "brightdata" ? await unlock(url) : typeof via === "object" ? await browserFetch(url, via.browser) : await fetchBytes(url, BROWSER_HEADERS);
|
|
3749
|
+
} catch (e) {
|
|
3750
|
+
return { ok: false, reason: errMsg(e), url: e?.url, via: name };
|
|
3751
|
+
}
|
|
3752
|
+
if (via === "brightdata" && !got.bytes.length) return { ok: false, reason: "empty response", via: name };
|
|
3753
|
+
const r = { ok: true, bytes: got.bytes, contentType: got.contentType, url: got.url ?? url, via: name };
|
|
3754
|
+
if (!looksLikePdf(got.bytes)) {
|
|
3755
|
+
const text3 = got.bytes.toString("utf8");
|
|
3756
|
+
if (isHtml(got.contentType, text3)) {
|
|
3757
|
+
if (isWallPage(text3)) return { ok: false, reason: "wall page", url: r.url, via: name };
|
|
3758
|
+
let md = "";
|
|
3759
|
+
try {
|
|
3760
|
+
md = htmlToMarkdown(/<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3);
|
|
3761
|
+
} catch {
|
|
3762
|
+
}
|
|
3763
|
+
const s = isScriptShell(md || text3);
|
|
3764
|
+
if (s.shell) return { ok: false, reason: `script shell (${s.words} words)`, url: r.url, via: name, shell: r };
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
return r;
|
|
3768
|
+
}
|
|
3769
|
+
var climb = async (url, onTry, onAlert) => {
|
|
3770
|
+
let r = await rung(url, "fetch");
|
|
3771
|
+
const firstReason = r.ok ? void 0 : r.reason;
|
|
3772
|
+
const target = !r.ok && r.url && r.url !== url ? r.url : url;
|
|
3773
|
+
if (target !== url) onTry?.(`\u2192 ${target}`);
|
|
3774
|
+
onTry?.(r.ok ? "ok fetch" : `failed fetch: ${r.reason}`);
|
|
3775
|
+
const escalates = (g) => !g.ok && (g.via !== "fetch" || /^(HTTP 403|HTTP 429|wall page|script shell)/.test(g.reason));
|
|
3776
|
+
let shell = !r.ok ? r.shell : void 0;
|
|
3777
|
+
for (const b of browserRungs()) {
|
|
3778
|
+
if (!escalates(r)) break;
|
|
3779
|
+
r = await rung(target, { browser: b.endpoint, label: b.label });
|
|
3780
|
+
if (!r.ok && r.shell && !shell) shell = r.shell;
|
|
3781
|
+
onTry?.(r.ok ? `ok browser ${b.label}` : `failed browser ${b.label}: ${r.reason}`);
|
|
3782
|
+
if (!r.ok && r.reason.startsWith(MISMATCH)) onAlert?.(`browser ${b.label}: ${r.reason}`);
|
|
3783
|
+
}
|
|
3784
|
+
if (!r.ok && unlockerConfigured() && (r.via !== "fetch" || /^(HTTP 403|wall page|script shell)/.test(r.reason))) {
|
|
3785
|
+
r = await rung(target, "brightdata");
|
|
3786
|
+
onTry?.(r.ok ? "ok brightdata" : `failed brightdata: ${r.reason}`);
|
|
3787
|
+
if (!r.ok && /^unlocker HTTP/.test(r.reason)) onAlert?.(`brightdata: ${r.reason}`);
|
|
3788
|
+
}
|
|
3789
|
+
if (!r.ok && shell) {
|
|
3790
|
+
onTry?.(`script shell kept: no rung rendered the page`);
|
|
3791
|
+
return shell;
|
|
3792
|
+
}
|
|
3793
|
+
return r.ok || r.via === "fetch" ? r : { ...r, first: firstReason };
|
|
3794
|
+
};
|
|
3795
|
+
var mediaType = (contentType) => contentType.split(";")[0].trim().toLowerCase();
|
|
3796
|
+
function isHtml(contentType, text3) {
|
|
3797
|
+
const ct = contentType.toLowerCase();
|
|
3798
|
+
if (/<html/i.test(text3.slice(0, 100))) return true;
|
|
3799
|
+
if (ct.includes("text/html") || ct.includes("application/xhtml+xml")) return true;
|
|
3800
|
+
return !ct && /^(<!doctype html|<html)/.test(text3.replace(/^\uFEFF/, "").trimStart().slice(0, 20).toLowerCase());
|
|
3801
|
+
}
|
|
3802
|
+
var ENTITIES = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " ", ndash: "\u2013", mdash: "\u2014", hellip: "\u2026", copy: "\xA9", reg: "\xAE", trade: "\u2122" };
|
|
3803
|
+
var decode2 = (s) => s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
|
|
3804
|
+
if (e[0] === "#") {
|
|
3805
|
+
const n = e[1].toLowerCase() === "x" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
|
|
3806
|
+
return Number.isFinite(n) ? String.fromCodePoint(n) : m;
|
|
3807
|
+
}
|
|
3808
|
+
return ENTITIES[e.toLowerCase()] ?? m;
|
|
3809
|
+
});
|
|
3810
|
+
var visibleText = (html) => decode2(html.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").replace(/¶\s*$/, "").trim();
|
|
3811
|
+
var normalized = (text3) => {
|
|
3812
|
+
const words2 = text3.toLowerCase().match(/[\p{L}\p{N}]+/gu);
|
|
3813
|
+
return words2 ? ` ${words2.join(" ")} ` : "";
|
|
3814
|
+
};
|
|
3815
|
+
function pageTitle(html) {
|
|
3816
|
+
const head = html.slice(0, 2e5);
|
|
3817
|
+
const og = /<meta\b[^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(head)?.[0] ?? /<meta\b[^>]*\bcontent\s*=\s*["'][^"']*["'][^>]*\bproperty\s*=\s*["']og:title["'][^>]*>/i.exec(head)?.[0];
|
|
3818
|
+
const ogTitle = og ? visibleText(/\bcontent\s*=\s*("([^"]*)"|'([^']*)')/i.exec(og)?.[2] ?? /\bcontent\s*=\s*'([^']*)'/i.exec(og)?.[1] ?? "") : "";
|
|
3819
|
+
const title = visibleText(/<title[^>]*>([\s\S]*?)<\/title>/i.exec(head)?.[1] ?? "");
|
|
3820
|
+
const h1 = visibleText(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1] ?? "");
|
|
3821
|
+
const headingInTitle = !!normalized(h1) && normalized(title).includes(normalized(h1));
|
|
3822
|
+
if (ogTitle) return ogTitle;
|
|
3823
|
+
if (headingInTitle) return h1;
|
|
3824
|
+
return title || h1 || null;
|
|
3825
|
+
}
|
|
3826
|
+
var visibleLine = (line) => line.replace(/\]\([^)]*\)/g, " ").replace(/<[a-zA-Z/!?][^>]*>/g, " ");
|
|
3827
|
+
function opensWithTitle(content, title) {
|
|
3828
|
+
const first = content.split("\n").find((l) => l.trim()) ?? "";
|
|
3829
|
+
const line = normalized(visibleLine(first)), wanted = normalized(title);
|
|
3830
|
+
if (!line || !wanted) return false;
|
|
3831
|
+
if (line.includes(wanted)) return true;
|
|
3832
|
+
return first.trimStart().startsWith("#") && wanted.includes(line);
|
|
3833
|
+
}
|
|
3834
|
+
function withPageTitle(html, content) {
|
|
3835
|
+
if (content.startsWith("<error>")) return content;
|
|
3836
|
+
const title = pageTitle(html);
|
|
3837
|
+
if (title === null || opensWithTitle(content, title)) return content;
|
|
3838
|
+
return `# ${title.replace(/([\\`*_\[\]<>#&|~])/g, "\\$1")}
|
|
3839
|
+
|
|
3840
|
+
${content.trimStart()}`;
|
|
3841
|
+
}
|
|
3842
|
+
var SIMPLIFY_FAILED = "<error>Page failed to be simplified from HTML</error>";
|
|
3843
|
+
async function convert(bytes, contentType) {
|
|
3844
|
+
if (looksLikePdf(bytes)) {
|
|
3845
|
+
const md = (await pdfToMarkdown(bytes, "", "")).replace(/^# \n\n/, "");
|
|
3846
|
+
return { format: "pdf", md, title: null, ext: "pdf" };
|
|
3847
|
+
}
|
|
3848
|
+
const text3 = bytes.toString("utf8");
|
|
3849
|
+
if (mediaType(contentType) === "text/markdown") return { format: "markdown", md: text3, title: null, ext: "md" };
|
|
3850
|
+
if (isHtml(contentType, text3)) {
|
|
3851
|
+
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
3852
|
+
let md;
|
|
3853
|
+
try {
|
|
3854
|
+
md = htmlToMarkdown(body);
|
|
3855
|
+
} catch {
|
|
3856
|
+
md = "";
|
|
3857
|
+
}
|
|
3858
|
+
md = md.trim() ? withPageTitle(text3, md) : SIMPLIFY_FAILED;
|
|
3859
|
+
return { format: "html", md, title: pageTitle(text3), ext: "html" };
|
|
3860
|
+
}
|
|
3861
|
+
return { format: "text", md: null, title: null, ext: "txt" };
|
|
3862
|
+
}
|
|
3863
|
+
function fetchAnswer(url, body, prefix, maxLength, startIndex) {
|
|
3864
|
+
const original = body.length;
|
|
3865
|
+
let content;
|
|
3866
|
+
if (startIndex >= original) content = "<error>No more content available.</error>";
|
|
3867
|
+
else {
|
|
3868
|
+
const cut = body.slice(startIndex, startIndex + maxLength);
|
|
3869
|
+
content = cut;
|
|
3870
|
+
const remaining = original - (startIndex + cut.length);
|
|
3871
|
+
if (cut.length === maxLength && remaining > 0) content += `
|
|
3872
|
+
|
|
3873
|
+
<error>Content truncated. Call the fetch tool with a start_index of ${startIndex + cut.length} to get more content.</error>`;
|
|
3874
|
+
}
|
|
3875
|
+
return `${prefix}Contents of ${url}:
|
|
3876
|
+
${content}`;
|
|
3877
|
+
}
|
|
3878
|
+
var ID_WIDTH = 13;
|
|
3879
|
+
function logId(url) {
|
|
3880
|
+
const bare2 = url.replace(/^https?:\/\//i, "");
|
|
3881
|
+
return bare2.length > ID_WIDTH ? bare2.slice(0, ID_WIDTH - 1) + "\u2026" : bare2;
|
|
3882
|
+
}
|
|
3883
|
+
var failureText = (url, reason) => {
|
|
3884
|
+
const m = /^HTTP (\d{3})\b/.exec(reason);
|
|
3885
|
+
return m ? `Failed to fetch ${url} - status code ${m[1]}` : `Failed to fetch ${url}: ${reason}`;
|
|
3886
|
+
};
|
|
3887
|
+
function titleLine(m, md) {
|
|
3888
|
+
if (m.title) return `# ${m.title}`;
|
|
3889
|
+
const first = md.split("\n").find((l) => l.trim());
|
|
3890
|
+
if (first) return first.trim();
|
|
3891
|
+
return `(${m.content_type || "no content type"}, no title)`;
|
|
3892
|
+
}
|
|
3893
|
+
function serve(requested, p, m, args, cached) {
|
|
3894
|
+
const maxLength = args.max_length ?? 5e3, startIndex = args.start_index ?? 0;
|
|
3895
|
+
const read = (file2) => existsSync8(file2) ? readFileSync8(file2, "utf8") : "";
|
|
3896
|
+
if (args.title_only) return { ok: true, text: `Contents of ${requested}:
|
|
3897
|
+
${titleLine(m, read(p.md))}`, cached, meta: m };
|
|
3898
|
+
const rawFile = m.files.find((f) => !f.endsWith("-meta.md") && !f.endsWith("-page.md"));
|
|
3899
|
+
let body, prefix = "";
|
|
3900
|
+
if (m.format === "markdown") body = read(p.md);
|
|
3901
|
+
else if (m.format === "text" || args.raw) {
|
|
3902
|
+
body = rawFile ? read(join9(p.entryDir, rawFile)) : read(p.md);
|
|
3903
|
+
prefix = `Content type ${m.content_type || "(not declared)"} cannot be simplified to markdown, but here is the raw content:
|
|
3904
|
+
`;
|
|
3905
|
+
} else body = read(p.md);
|
|
3906
|
+
return { ok: true, text: fetchAnswer(requested, body, prefix, maxLength, startIndex), cached, meta: m };
|
|
3907
|
+
}
|
|
3908
|
+
var inflight = /* @__PURE__ */ new Map();
|
|
3909
|
+
async function libraryFetch(libraryDir, requested, args, converter, logs = {}, fetchImpl = climb) {
|
|
3910
|
+
const url = normalizeUrl(requested);
|
|
3911
|
+
if (!url) return { ok: false, error: `Failed to fetch ${requested}: invalid URL`, cached: false, meta: null };
|
|
3912
|
+
const p = urlPaths(libraryDir, url);
|
|
3913
|
+
const s = settings();
|
|
3914
|
+
const fresh = (m2) => daysSince(m2.fetched) < (m2.format === "failed" ? s.retry_after_days : s.refresh_after_days);
|
|
3915
|
+
const answer = (m2, cached) => m2.format === "failed" ? { ok: false, error: failureText(requested, m2.error ?? "unknown error"), cached, meta: m2 } : serve(requested, p, m2, args, cached);
|
|
3916
|
+
const stored = readUrlMeta(p);
|
|
3917
|
+
if (stored && fresh(stored)) return answer(stored, true);
|
|
3918
|
+
let run = inflight.get(p.hash);
|
|
3919
|
+
if (!run) {
|
|
3920
|
+
run = (async () => {
|
|
3921
|
+
const r = await fetchImpl(url, logs.onTry, logs.onAlert);
|
|
3922
|
+
const base = { url, fetched: today(), converted_by: converter };
|
|
3923
|
+
if (!r.ok) {
|
|
3924
|
+
const error = r.first && /^HTTP \d{3}/.test(r.first) ? r.first : r.reason;
|
|
3925
|
+
writeUrlMeta(p, { ...base, final_url: r.url ?? url, title: null, format: "failed", content_type: "", level: "title", retrieved_by: r.via, error, files: [`${p.entry}-meta.md`] });
|
|
3926
|
+
return;
|
|
3927
|
+
}
|
|
3928
|
+
const page = await convert(r.bytes, r.contentType);
|
|
3929
|
+
mkdirSync5(p.entryDir, { recursive: true });
|
|
3930
|
+
const files = [`${p.entry}-meta.md`];
|
|
3931
|
+
if (page.md !== null) {
|
|
3932
|
+
writeAtomic(p.md, page.md);
|
|
3933
|
+
files.push(`${p.entry}-page.md`);
|
|
3934
|
+
}
|
|
3935
|
+
if (page.ext !== "md") {
|
|
3936
|
+
writeAtomic(p.raw(page.ext), r.bytes);
|
|
3937
|
+
files.push(`${p.entry}-page.${page.ext}`);
|
|
3938
|
+
}
|
|
3939
|
+
writeUrlMeta(p, { ...base, final_url: r.url, title: page.title, format: page.format, content_type: mediaType(r.contentType), level: "full_text", retrieved_by: r.via, error: null, files });
|
|
3940
|
+
})().finally(() => inflight.delete(p.hash));
|
|
3941
|
+
inflight.set(p.hash, run);
|
|
3942
|
+
}
|
|
3943
|
+
await run;
|
|
3944
|
+
const m = readUrlMeta(p);
|
|
3945
|
+
if (!m) return { ok: false, error: `Failed to fetch ${requested}: nothing stored`, cached: false, meta: null };
|
|
3946
|
+
return answer(m, false);
|
|
3947
|
+
}
|
|
3948
|
+
|
|
3949
|
+
// src/verifycache.ts
|
|
3950
|
+
import { createHash as createHash3 } from "crypto";
|
|
3951
|
+
import { existsSync as existsSync9, mkdirSync as mkdirSync6, readFileSync as readFileSync9 } from "fs";
|
|
3952
|
+
import { dirname as dirname3, join as join10 } from "path";
|
|
3953
|
+
var sha = (s) => createHash3("sha1").update(s).digest("hex");
|
|
3954
|
+
var normalizeClaim = (claim) => claim.replace(/\s+/g, " ").trim();
|
|
3955
|
+
function foldClaim(claim) {
|
|
3956
|
+
return claim.normalize("NFKC").toLowerCase().replace(/[\u2018\u2019\u201a\u201b\u2032]/g, "'").replace(/[\u201c\u201d\u201e\u201f\u2033]/g, '"').replace(/[\u2010-\u2015\u2212]/g, "-").replace(/\+\/-|\+-/g, "\xB1").replace(/\s+/g, " ").trim().replace(/[.;:,]+$/, "").trim();
|
|
3957
|
+
}
|
|
3958
|
+
var textHash = (text3) => sha(text3).slice(0, 16);
|
|
3959
|
+
var cacheKey = (claim, id, model, threshold, text3) => sha(`${foldClaim(claim)}
|
|
3960
|
+
${id}
|
|
3961
|
+
${model}
|
|
3962
|
+
${threshold}
|
|
3963
|
+
${text3}`).slice(0, 16);
|
|
3964
|
+
var verdictFile = (libraryDir, key) => join10(libraryDir, "verify", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
|
|
3965
|
+
function readVerdict(file2) {
|
|
3966
|
+
if (!existsSync9(file2)) return null;
|
|
3967
|
+
try {
|
|
3968
|
+
return JSON.parse(readFileSync9(file2, "utf8"));
|
|
3969
|
+
} catch {
|
|
3970
|
+
return null;
|
|
3971
|
+
}
|
|
3972
|
+
}
|
|
3973
|
+
function lookup(file2, maxDays) {
|
|
3974
|
+
const c = readVerdict(file2);
|
|
3975
|
+
return c && daysSince(c.date) < maxDays ? c : null;
|
|
3976
|
+
}
|
|
3977
|
+
function remember(file2, c) {
|
|
3978
|
+
mkdirSync6(dirname3(file2), { recursive: true });
|
|
3979
|
+
writeAtomic(file2, JSON.stringify(c, null, 2) + "\n");
|
|
3980
|
+
}
|
|
3981
|
+
async function verifyCached(claim, id, cfg, decide, libraryDir = FH_LIBRARY_DIR, read = readPaper) {
|
|
3982
|
+
const paper = await read(id, libraryDir);
|
|
3983
|
+
const once = () => paper;
|
|
3984
|
+
if ("error" in paper || !cfg) return { ...await verifyClaim(claim, id, cfg, decide, once), cached: false };
|
|
3985
|
+
const text3 = textHash(paper.text);
|
|
3986
|
+
const file2 = verdictFile(libraryDir, cacheKey(claim, paper.id, cfg.model, cfg.threshold, text3));
|
|
3987
|
+
const hit = lookup(file2, settings().refresh_after_days);
|
|
3988
|
+
if (hit) {
|
|
3989
|
+
return { claim, id: paper.id, verdict: hit.verdict, reason: hit.reason, score: hit.score, basis: hit.basis, windows: hit.windows, glyphs_unresolved: hit.glyphs_unresolved, superseded: hit.superseded, model: hit.model, cached: true };
|
|
3990
|
+
}
|
|
3991
|
+
const r = await verifyClaim(claim, id, cfg, decide, once);
|
|
3992
|
+
if (!r.error && r.score !== null) {
|
|
3993
|
+
remember(file2, { key: file2.slice(-21, -5), id: paper.id, claim: normalizeClaim(claim), model: cfg.model, threshold: cfg.threshold, text: text3, date: today(), verdict: r.verdict, reason: r.reason, score: r.score, basis: r.basis, windows: r.windows, glyphs_unresolved: r.glyphs_unresolved, superseded: r.superseded });
|
|
3994
|
+
}
|
|
3995
|
+
return { ...r, cached: false };
|
|
3996
|
+
}
|
|
3997
|
+
|
|
3998
|
+
// src/json.ts
|
|
3999
|
+
var DROP = /* @__PURE__ */ new Set(["entry_dir", "library_dir", "files"]);
|
|
4000
|
+
var compact2 = (o) => JSON.stringify(o);
|
|
4001
|
+
function lean(o) {
|
|
4002
|
+
if (Array.isArray(o)) return o.map((row2) => row2 && typeof row2 === "object" && !Array.isArray(row2) ? leanObject(row2) : row2);
|
|
4003
|
+
if (o && typeof o === "object") return leanObject(o);
|
|
4004
|
+
return o;
|
|
4005
|
+
}
|
|
4006
|
+
function leanObject(o) {
|
|
4007
|
+
const out = {};
|
|
4008
|
+
for (const [k, v] of Object.entries(o)) {
|
|
4009
|
+
if (DROP.has(k) || v === null || v === void 0 || Array.isArray(v) && v.length === 0) continue;
|
|
4010
|
+
out[k] = Array.isArray(v) ? lean(v) : v;
|
|
4011
|
+
}
|
|
4012
|
+
return out;
|
|
4013
|
+
}
|
|
4014
|
+
var leanJson = (o) => compact2(lean(o));
|
|
4015
|
+
|
|
4016
|
+
// src/trials.ts
|
|
4017
|
+
import { createHash as createHash4 } from "crypto";
|
|
4018
|
+
import { existsSync as existsSync10, mkdirSync as mkdirSync7, readFileSync as readFileSync10 } from "fs";
|
|
4019
|
+
import { dirname as dirname4, join as join11 } from "path";
|
|
4020
|
+
var SEARCH_AFTER_DAYS = 7;
|
|
4021
|
+
var API2 = "https://clinicaltrials.gov/api/v2/studies";
|
|
4022
|
+
var FIELDS = ["NCTId", "BriefTitle", "OverallStatus", "WhyStopped", "Phase", "StudyType", "EnrollmentCount", "EnrollmentType", "StartDate", "PrimaryCompletionDate", "CompletionDate", "LastUpdatePostDate", "PrimaryOutcomeMeasure", "PrimaryOutcomeTimeFrame", "Condition", "InterventionName", "LeadSponsorName", "HasResults"];
|
|
4023
|
+
var ONGOING = ["RECRUITING", "NOT_YET_RECRUITING", "ACTIVE_NOT_RECRUITING", "ENROLLING_BY_INVITATION"];
|
|
4024
|
+
var MAX_RESULTS = 100;
|
|
4025
|
+
var DEFAULT_RESULTS = 20;
|
|
4026
|
+
var SORTS = { relevance: null, newest: "LastUpdatePostDate:desc", largest: "EnrollmentCount:desc", completion: "CompletionDate:desc" };
|
|
4027
|
+
var COLUMNS = [
|
|
4028
|
+
"nct",
|
|
4029
|
+
"title",
|
|
4030
|
+
"status",
|
|
4031
|
+
// "Terminated — slow accrual": a trial past its completion date is not "ongoing" (ER.md 28.7)
|
|
4032
|
+
"phase",
|
|
4033
|
+
"enrollment",
|
|
4034
|
+
// "100 (estimated)" / "48 (actual)" — participant count (28.5)
|
|
4035
|
+
"start",
|
|
4036
|
+
"primary_completion",
|
|
4037
|
+
"primary_outcome",
|
|
4038
|
+
// the first primary outcome with its time frame, cut at 80 characters — the primary endpoint (28.5)
|
|
4039
|
+
"conditions",
|
|
4040
|
+
"interventions",
|
|
4041
|
+
// "IVF; Infertility, Female" — one string each
|
|
4042
|
+
"has_results"
|
|
4043
|
+
// posted results: get_study_results (28.8)
|
|
4044
|
+
];
|
|
4045
|
+
var clean = (s) => (s ?? "").replace(/\s+/g, " ").trim();
|
|
4046
|
+
function normalizeQuery(q) {
|
|
4047
|
+
const out = { status: q.status ?? "all", sort: q.sort ?? "relevance", max_results: Math.min(MAX_RESULTS, Math.max(1, Math.floor(q.max_results ?? DEFAULT_RESULTS))) };
|
|
4048
|
+
if (clean(q.query)) out.query = clean(q.query);
|
|
4049
|
+
if (clean(q.intervention)) out.intervention = clean(q.intervention);
|
|
4050
|
+
if (clean(q.condition)) out.condition = clean(q.condition);
|
|
4051
|
+
if (!out.query && !out.intervention && !out.condition) return { error: "empty search: give query, intervention or condition" };
|
|
4052
|
+
if (!["ongoing", "completed", "all"].includes(out.status)) return { error: `status must be ongoing, completed or all, not "${String(q.status)}"` };
|
|
4053
|
+
if (!(out.sort in SORTS)) return { error: `sort must be relevance, newest, largest or completion, not "${String(q.sort)}"` };
|
|
4054
|
+
return out;
|
|
4055
|
+
}
|
|
4056
|
+
var searchKey = (q) => createHash4("sha1").update(JSON.stringify(["clinicaltrials", q.query ?? "", q.intervention ?? "", q.condition ?? "", q.status, q.sort, q.max_results])).digest("hex").slice(0, 16);
|
|
4057
|
+
var searchFile = (libraryDir, key) => join11(libraryDir, "search", key.slice(0, 2), key.slice(0, 4), `${key}.json`);
|
|
4058
|
+
function searchUrl(q) {
|
|
4059
|
+
const p = new URLSearchParams();
|
|
4060
|
+
if (q.query) p.set("query.term", q.query);
|
|
4061
|
+
if (q.intervention) p.set("query.intr", q.intervention);
|
|
4062
|
+
if (q.condition) p.set("query.cond", q.condition);
|
|
4063
|
+
if (q.status === "ongoing") p.set("filter.overallStatus", ONGOING.join(","));
|
|
4064
|
+
if (q.status === "completed") p.set("filter.overallStatus", "COMPLETED");
|
|
4065
|
+
if (SORTS[q.sort]) p.set("sort", SORTS[q.sort]);
|
|
4066
|
+
p.set("fields", FIELDS.join(","));
|
|
4067
|
+
p.set("pageSize", String(q.max_results));
|
|
4068
|
+
p.set("countTotal", "true");
|
|
4069
|
+
return `${API2}?${p.toString()}`;
|
|
4070
|
+
}
|
|
4071
|
+
var STATUS2 = { NOT_YET_RECRUITING: "Not yet recruiting", RECRUITING: "Recruiting", ENROLLING_BY_INVITATION: "Enrolling by invitation", ACTIVE_NOT_RECRUITING: "Active, not recruiting", SUSPENDED: "Suspended", TERMINATED: "Terminated", COMPLETED: "Completed", WITHDRAWN: "Withdrawn", UNKNOWN: "Unknown status", AVAILABLE: "Available", NO_LONGER_AVAILABLE: "No longer available", TEMPORARILY_NOT_AVAILABLE: "Temporarily not available", APPROVED_FOR_MARKETING: "Approved for marketing", WITHHELD: "Withheld" };
|
|
4072
|
+
var PHASE2 = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
4073
|
+
var joined = (xs) => xs.length ? xs.map((x) => x.replace(/\s+/g, " ").trim()).filter(Boolean).join("; ") || null : null;
|
|
4074
|
+
function parseRows(json2) {
|
|
4075
|
+
const d = JSON.parse(json2);
|
|
4076
|
+
const rows = [];
|
|
4077
|
+
for (const s of Array.isArray(d.studies) ? d.studies : []) {
|
|
4078
|
+
const p = s.protocolSection ?? {};
|
|
4079
|
+
const id = p.identificationModule ?? {}, st = p.statusModule ?? {}, des = p.designModule ?? {};
|
|
4080
|
+
const phases = Array.isArray(des.phases) ? des.phases : [];
|
|
4081
|
+
const nct = String(id.nctId ?? "");
|
|
4082
|
+
const en = des.enrollmentInfo ?? {};
|
|
4083
|
+
const outcomes = Array.isArray(p.outcomesModule?.primaryOutcomes) ? p.outcomesModule.primaryOutcomes : [];
|
|
4084
|
+
const first = outcomes[0];
|
|
4085
|
+
const cut = (t, n) => t.length > n ? t.slice(0, n - 1).trimEnd() + "\u2026" : t;
|
|
4086
|
+
const primary = first?.measure ? cut(`${String(first.measure).replace(/\s+/g, " ").trim()}${first.timeFrame ? ` (${String(first.timeFrame).replace(/\s+/g, " ").trim()})` : ""}`, 80) + (outcomes.length > 1 ? ` +${outcomes.length - 1}` : "") : null;
|
|
4087
|
+
const status = (STATUS2[st.overallStatus] ?? String(st.overallStatus ?? "")) + (st.whyStopped ? ` \u2014 ${String(st.whyStopped).replace(/\s+/g, " ").trim()}` : "");
|
|
4088
|
+
rows.push([
|
|
4089
|
+
nct,
|
|
4090
|
+
String(id.briefTitle ?? ""),
|
|
4091
|
+
status,
|
|
4092
|
+
phases.length ? phases.map((x) => PHASE2[x] ?? x).join(" / ") : null,
|
|
4093
|
+
typeof en.count === "number" ? `${en.count}${en.type ? ` (${String(en.type).toLowerCase()})` : ""}` : null,
|
|
4094
|
+
st.startDateStruct?.date ?? null,
|
|
4095
|
+
st.primaryCompletionDateStruct?.date ?? null,
|
|
4096
|
+
primary,
|
|
4097
|
+
joined(Array.isArray(p.conditionsModule?.conditions) ? p.conditionsModule.conditions.map(String) : []),
|
|
4098
|
+
joined(Array.isArray(p.armsInterventionsModule?.interventions) ? p.armsInterventionsModule.interventions.map((i) => String(i?.name ?? "")).filter(Boolean) : []),
|
|
4099
|
+
s.hasResults === true
|
|
4100
|
+
]);
|
|
4101
|
+
}
|
|
4102
|
+
return { total: typeof d.totalCount === "number" ? d.totalCount : rows.length, rows };
|
|
4103
|
+
}
|
|
4104
|
+
var liveFetch = (url) => fetchText(url, { "User-Agent": LIBRARY_UA, Accept: "application/json" });
|
|
4105
|
+
async function searchTrials(raw, libraryDir = FH_LIBRARY_DIR, fetchImpl = liveFetch) {
|
|
4106
|
+
const q = normalizeQuery(raw);
|
|
4107
|
+
if ("error" in q) return { ok: false, error: q.error };
|
|
4108
|
+
const file2 = searchFile(libraryDir, searchKey(q));
|
|
4109
|
+
if (existsSync10(file2)) {
|
|
4110
|
+
try {
|
|
4111
|
+
const r = JSON.parse(readFileSync10(file2, "utf8"));
|
|
4112
|
+
if (daysSince(r.searched) < SEARCH_AFTER_DAYS) return { ok: true, result: r, cached: true };
|
|
4113
|
+
} catch {
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
let json2;
|
|
4117
|
+
try {
|
|
4118
|
+
json2 = await fetchImpl(searchUrl(q));
|
|
4119
|
+
} catch (e) {
|
|
4120
|
+
return { ok: false, error: `clinicaltrials.gov: ${e instanceof Error ? e.message : String(e)}` };
|
|
4121
|
+
}
|
|
4122
|
+
let parsed;
|
|
4123
|
+
try {
|
|
4124
|
+
parsed = parseRows(json2);
|
|
4125
|
+
} catch {
|
|
4126
|
+
return { ok: false, error: "clinicaltrials.gov: not a studies answer" };
|
|
4127
|
+
}
|
|
4128
|
+
const result = { source: "clinicaltrials", query: q, total: parsed.total, columns: COLUMNS, rows: parsed.rows, searched: today() };
|
|
4129
|
+
try {
|
|
4130
|
+
mkdirSync7(dirname4(file2), { recursive: true });
|
|
4131
|
+
writeAtomic(file2, JSON.stringify(result, null, 2) + "\n");
|
|
4132
|
+
} catch {
|
|
4133
|
+
}
|
|
4134
|
+
return { ok: true, result, cached: false };
|
|
4135
|
+
}
|
|
4136
|
+
function trialsLogId(q) {
|
|
4137
|
+
const s = (q.intervention ?? q.condition ?? q.query ?? "").replace(/\s+/g, " ");
|
|
4138
|
+
return s.length > 10 ? s.slice(0, 9) + "\u2026" : s || "-";
|
|
4139
|
+
}
|
|
4140
|
+
|
|
4141
|
+
// src/trialrecord.ts
|
|
4142
|
+
var list3 = (a) => Array.isArray(a) ? a : [];
|
|
4143
|
+
var para2 = (s, max = Infinity) => {
|
|
4144
|
+
const t = String(s ?? "").replace(/\s+/g, " ").trim();
|
|
4145
|
+
if (!t) return null;
|
|
4146
|
+
return t.length > max ? t.slice(0, max - 1).trimEnd() + "\u2026" : t;
|
|
4147
|
+
};
|
|
4148
|
+
var joined2 = (xs) => {
|
|
4149
|
+
const ys = xs.filter((x) => !!x);
|
|
4150
|
+
return ys.length ? ys.join("; ") : null;
|
|
4151
|
+
};
|
|
4152
|
+
var dated2 = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : null;
|
|
4153
|
+
function recordSummary(json2) {
|
|
4154
|
+
let d;
|
|
4155
|
+
try {
|
|
4156
|
+
d = JSON.parse(json2);
|
|
4157
|
+
} catch {
|
|
4158
|
+
return null;
|
|
4159
|
+
}
|
|
4160
|
+
const ps = d?.protocolSection;
|
|
4161
|
+
if (!ps) return null;
|
|
4162
|
+
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
4163
|
+
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
4164
|
+
const nct = String(idm.nctId ?? "").toUpperCase();
|
|
4165
|
+
const title = para2(idm.briefTitle) ?? nct;
|
|
4166
|
+
const official = para2(idm.officialTitle);
|
|
4167
|
+
const di = dm.designInfo ?? {}, mi = di.maskingInfo ?? {};
|
|
4168
|
+
const design = joined2([di.allocation ? words(di.allocation) : null, di.interventionModel ? words(di.interventionModel) : null, mi.masking ? `masking ${words(mi.masking).toLowerCase()}` : null, di.primaryPurpose ? `purpose ${words(di.primaryPurpose).toLowerCase()}` : null]);
|
|
4169
|
+
const outcome = (o) => `${para2(o.measure) ?? ""}${o.timeFrame ? ` (${para2(o.timeFrame)})` : ""}`.trim();
|
|
4170
|
+
const countries = [...new Set(list3(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
4171
|
+
const refs = list3(rf.references);
|
|
4172
|
+
const age = el.minimumAge || el.maximumAge ? `${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : null;
|
|
4173
|
+
return {
|
|
4174
|
+
nct,
|
|
4175
|
+
title,
|
|
4176
|
+
official_title: official && official !== title ? official : null,
|
|
4177
|
+
acronym: para2(idm.acronym),
|
|
4178
|
+
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
4179
|
+
why_stopped: para2(st.whyStopped),
|
|
4180
|
+
phase: list3(dm.phases).length ? list3(dm.phases).map(words).join(" / ") : null,
|
|
4181
|
+
study_type: dm.studyType ? words(dm.studyType) : null,
|
|
4182
|
+
design,
|
|
4183
|
+
enrollment: dm.enrollmentInfo?.count !== void 0 ? `${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null,
|
|
4184
|
+
start: dated2(st.startDateStruct),
|
|
4185
|
+
primary_completion: dated2(st.primaryCompletionDateStruct),
|
|
4186
|
+
completion: dated2(st.completionDateStruct),
|
|
4187
|
+
first_posted: st.studyFirstPostDateStruct?.date ?? null,
|
|
4188
|
+
last_update: st.lastUpdatePostDateStruct?.date ?? null,
|
|
4189
|
+
results_first_posted: st.resultsFirstPostDateStruct?.date ?? null,
|
|
4190
|
+
sponsor: para2(sp.leadSponsor?.name),
|
|
4191
|
+
collaborators: joined2(list3(sp.collaborators).map((c) => para2(c.name))),
|
|
4192
|
+
conditions: joined2(list3(ps.conditionsModule?.conditions).map((c) => para2(c))),
|
|
4193
|
+
interventions: joined2(list3(ai.interventions).map((i) => para2(`${i.type ? `${words(i.type)}: ` : ""}${i.name ?? ""}`))),
|
|
4194
|
+
primary_outcomes: list3(om.primaryOutcomes).map(outcome).filter(Boolean),
|
|
4195
|
+
secondary_outcomes: list3(om.secondaryOutcomes).length,
|
|
4196
|
+
eligibility: joined2([el.sex ? `sex ${words(el.sex).toLowerCase()}` : null, age ? `age ${age}` : null, el.healthyVolunteers !== void 0 ? `healthy volunteers ${el.healthyVolunteers ? "yes" : "no"}` : null]),
|
|
4197
|
+
locations: countries.length ? `${list3(cl.locations).length} site${list3(cl.locations).length === 1 ? "" : "s"} in ${countries.join(", ")}` : null,
|
|
4198
|
+
publications: refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`),
|
|
4199
|
+
has_results: d.hasResults === true,
|
|
4200
|
+
summary: para2(de.briefSummary, 400)
|
|
4201
|
+
};
|
|
4202
|
+
}
|
|
4203
|
+
function resultsText(nct, md, maxChars) {
|
|
4204
|
+
const i = md.search(/^## Results\s*$/m);
|
|
4205
|
+
if (i < 0) return { nct, has_results: false, text: null, truncated: false, outcomes: [], note: "no posted results in the registry record" };
|
|
4206
|
+
const part = md.slice(i).replace(/^## Results\s*\n/, "").trim();
|
|
4207
|
+
const outcomes = [...part.matchAll(/^#### (.+)$/gm)].map((m) => m[1].trim());
|
|
4208
|
+
const truncated = part.length > maxChars;
|
|
4209
|
+
return { nct, has_results: true, text: truncated ? part.slice(0, maxChars).trimEnd() + "\n\n[truncated]" : part, truncated, outcomes, note: truncated ? "cut at max_chars; library_read(id, section: <an outcome title>) returns one measure whole" : null };
|
|
4210
|
+
}
|
|
4211
|
+
|
|
3560
4212
|
// src/tools.ts
|
|
3561
4213
|
var pkg = (() => {
|
|
3562
4214
|
try {
|
|
3563
|
-
return JSON.parse(
|
|
4215
|
+
return JSON.parse(readFileSync11(new URL("../package.json", import.meta.url), "utf8"));
|
|
3564
4216
|
} catch {
|
|
3565
4217
|
return { name: "fh-library", version: "0.0.0" };
|
|
3566
4218
|
}
|
|
3567
4219
|
})();
|
|
3568
|
-
var INSTRUCTIONS = `fh-library keeps a local library of scientific papers, FDA drug labels
|
|
4220
|
+
var INSTRUCTIONS = `fh-library keeps a local library of scientific papers, FDA drug labels, clinical trial records and web pages, so you can check what a source actually says, and searches ClinicalTrials.gov. The server downloads each record, converts it to Markdown and stores it on disk. Every tool returns a small JSON answer; the document itself never enters your context \u2014 except library_fetch, which is the standard fetch tool with a cache. One ID per call.
|
|
3569
4221
|
|
|
3570
4222
|
IDs \u2014 always "type:value"
|
|
3571
4223
|
- pmid:30955454 \u2014 a paper by PubMed ID. Bare digits are taken as a PMID.
|
|
@@ -3575,11 +4227,15 @@ IDs \u2014 always "type:value"
|
|
|
3575
4227
|
|
|
3576
4228
|
Tools
|
|
3577
4229
|
- library_version() \u2014 package name, version, library directory.
|
|
3578
|
-
- library_info(ids?) \u2014 the whole library: entry counts by type and level, the retrieval sources, browsers, the decision model, the configuration. ids=true lists every entry.
|
|
4230
|
+
- library_info(ids?) \u2014 the whole library: entry counts by type and level, the retrieval sources, browsers, the decision model, the configuration, this server's cache hits and misses since it started (cache). ids=true lists every entry.
|
|
3579
4231
|
- library_add(id) \u2014 download and store one record; returns its status. A stored full text is returned without any download; an abstract-level record retries only the sources that failed transiently; a record older than 30 days gets its correction notices re-read.
|
|
3580
4232
|
- library_status(id) \u2014 the status of one record without downloading; level "missing" when it is not stored.
|
|
3581
4233
|
- library_read(id, section?, query?, max_chars?, max_hits?) \u2014 one small piece of a stored document: its outline, one section, or the passages matching a query.
|
|
3582
4234
|
- library_verify(claim, id) \u2014 does this one source state this one finding? A decision model judges the stored text and returns a verdict.
|
|
4235
|
+
- library_clinicaltrials_search_studies(query | intervention | condition, status?, sort?, max_results?) \u2014 search ClinicalTrials.gov: the total and rows in columns order (NCT ID, title, status, phase, enrollment, dates, primary outcome, conditions, interventions, has_results). Cached a week.
|
|
4236
|
+
- library_clinicaltrials_get_study_record(nct) \u2014 one trial as a review needs it (status, design, enrollment, dates, outcomes, eligibility, result publications, summary), from the stored record; added first when missing.
|
|
4237
|
+
- library_clinicaltrials_get_study_results(nct, max_chars?) \u2014 a trial's posted results as Markdown, capped; the outcome titles listed for library_read.
|
|
4238
|
+
- library_fetch(url, max_length?, start_index?, raw?, title_only?) \u2014 a web page as Markdown, same call and same answer as the standard fetch tool ("Contents of <url>:" and the page, cut at max_length with a note on how to continue). Prefer it for every URL: a cached page answers from disk, a walled or script-rendered page climbs the browser and unlocker ladder in one call. PDFs as Markdown. title_only returns the title line alone \u2014 enough to verify a link, at a few tokens.
|
|
3583
4239
|
|
|
3584
4240
|
Hints
|
|
3585
4241
|
- library_read and library_verify store a missing record themselves; the library_read answer then carries added: true. No library_add necessary first.
|
|
@@ -3587,6 +4243,8 @@ Hints
|
|
|
3587
4243
|
- level \u2014 what is stored: "full_text", "abstract" (no full text could be found; attempts names each source tried and why it failed) or "title" (not even an abstract; citable false).
|
|
3588
4244
|
- error "pmid not found" or "invalid id" \u2014 the ID itself is wrong; the same call gives the same answer.
|
|
3589
4245
|
- Calls may run in parallel: library_add, library_read and library_verify. Two calls on the same missing record share one add.
|
|
4246
|
+
- library_fetch takes any URL that is not a paper, label or trial record; those have IDs, and library_add, library_read and library_verify work on the stored document. A second pass over the same review reads every page from disk; a failed fetch is remembered for a day.
|
|
4247
|
+
- To verify that a web page exists and what it is: library_fetch(url, title_only: true). The answer is "Contents of <url>:" and the page's title line \u2014 a few tokens \u2014 or the fetch error; the page itself is stored in full, so a later library_fetch of the same URL reads from disk.
|
|
3590
4248
|
|
|
3591
4249
|
Verifying a claim
|
|
3592
4250
|
- library_verify judges one claim against one stored document. A finding backed by three papers takes three calls with the same claim.
|
|
@@ -3598,6 +4256,7 @@ Verifying a claim
|
|
|
3598
4256
|
- verdict "unverifiable" \u2014 only the abstract is stored and it does not state the claim; the full text may (reason says so). A retry gives the same answer until a full text is stored.
|
|
3599
4257
|
- glyphs_unresolved above 0 \u2014 the PDF font lost symbols (minus, \xB1, \u2264); a fail on a number may be the font. library_read shows the passage.
|
|
3600
4258
|
- superseded true \u2014 a published correction replaced the figure the claim uses; the status's corrections holds the notice.
|
|
4259
|
+
- Verdicts are kept for 30 days: the same claim against the same stored text answers from disk, no model call. A paper that gains its full text is judged afresh.
|
|
3601
4260
|
|
|
3602
4261
|
Reading a document
|
|
3603
4262
|
- library_read with only the ID returns the outline: every heading with its word count, and whether the full text or only the abstract is stored.
|
|
@@ -3608,12 +4267,12 @@ Reading a document
|
|
|
3608
4267
|
- section and query together search inside that section only.
|
|
3609
4268
|
- section "Abstract" on a full text without one returns the stored abstract. section "Declarations" returns funding, conflicts of interest, acknowledgements and ethics statements, assembled when the document has no section of that name.
|
|
3610
4269
|
|
|
3611
|
-
Status fields: requested, type, id, entry,
|
|
4270
|
+
Status fields: requested, type, id, entry, pmid, pmcid, doi, setid, nctid, publications, title, first_author, year, publication_types, retracted, corrections, coi (conflict-of-interest statement), funding, citable, comment, has_abstract, level, version, retrieved_by, retrieved_date, pubmed_checked, attempts, glyphs_unresolved, error. Answers are compact JSON; a field that is null or an empty list is left out (library_verify keeps its nulls: score and reason null mean something).
|
|
3612
4271
|
|
|
3613
4272
|
Where records come from: PubMed for the record and abstract; the full text from a file you drop into <library>/inbox/ (PMID in the file name), PubMed Central, Europe PMC, publishers' text-mining links, Unpaywall, OpenAlex, CORE, and a proxy URL template from the configuration. Every action is appended as one line to <library>/library.log.`;
|
|
3614
4273
|
var idSchema = z.string().describe('ID as "type:value": a paper "pmid:30955454" (bare digits count as pmid) or "doi:10.5812/asjsm.110753" (a paper PubMed does not index; a DOI PubMed indexes is taken as its PMID), a drug label "dailymed:<set ID>" (the set ID from its DailyMed page URL), or a trial record "nct:05173896" (the eight digits; nct:NCT05173896 and a bare NCT05173896 are accepted).');
|
|
3615
4274
|
var levelWord = (m) => m.level === "abstract" && !m.has_abstract ? "title" : m.level;
|
|
3616
|
-
var json = (s) => ({ content: [{ type: "text", text:
|
|
4275
|
+
var json = (s) => ({ content: [{ type: "text", text: leanJson(s) }] });
|
|
3617
4276
|
function noticeTag(m) {
|
|
3618
4277
|
const n = (t) => m.corrections.filter((c) => c.type === t).length;
|
|
3619
4278
|
const parts = [
|
|
@@ -3737,13 +4396,13 @@ function registerTools(server2) {
|
|
|
3737
4396
|
{ id: idSchema },
|
|
3738
4397
|
({ id }) => addEntry(id)
|
|
3739
4398
|
);
|
|
3740
|
-
const
|
|
4399
|
+
const inflight2 = /* @__PURE__ */ new Map();
|
|
3741
4400
|
function addEntry(id) {
|
|
3742
4401
|
const key = parseId(id).requested;
|
|
3743
|
-
let p =
|
|
4402
|
+
let p = inflight2.get(key);
|
|
3744
4403
|
if (!p) {
|
|
3745
|
-
p = addEntryNow(id).finally(() =>
|
|
3746
|
-
|
|
4404
|
+
p = addEntryNow(id).finally(() => inflight2.delete(key));
|
|
4405
|
+
inflight2.set(key, p);
|
|
3747
4406
|
}
|
|
3748
4407
|
return p;
|
|
3749
4408
|
}
|
|
@@ -3914,7 +4573,7 @@ function registerTools(server2) {
|
|
|
3914
4573
|
},
|
|
3915
4574
|
async ({ id, section: section2, query, max_chars, max_hits }) => {
|
|
3916
4575
|
const parsed = parseId(id);
|
|
3917
|
-
const out = (o2) => ({ content: [{ type: "text", text:
|
|
4576
|
+
const out = (o2) => ({ content: [{ type: "text", text: leanJson(o2) }] });
|
|
3918
4577
|
if (parsed.error) {
|
|
3919
4578
|
log("library_read", parsed.requested, `error ${parsed.error}`);
|
|
3920
4579
|
return out({ requested: parsed.requested, error: parsed.error });
|
|
@@ -3930,8 +4589,8 @@ function registerTools(server2) {
|
|
|
3930
4589
|
}
|
|
3931
4590
|
added = true;
|
|
3932
4591
|
}
|
|
3933
|
-
const full =
|
|
3934
|
-
const md =
|
|
4592
|
+
const full = existsSync11(paths.fulltextMd);
|
|
4593
|
+
const md = readFileSync11(full ? paths.fulltextMd : paths.md, "utf8");
|
|
3935
4594
|
const text3 = full ? "full_text" : "abstract";
|
|
3936
4595
|
const base = added ? { requested: parsed.requested, entry: paths.entry, text: text3, added: true } : { requested: parsed.requested, entry: paths.entry, text: text3 };
|
|
3937
4596
|
const headings2 = () => outline(md, text3).sections;
|
|
@@ -3951,8 +4610,8 @@ function registerTools(server2) {
|
|
|
3951
4610
|
const cap = max_chars ?? 6e3;
|
|
3952
4611
|
let sec = readSection(md, section2, cap);
|
|
3953
4612
|
let via = "";
|
|
3954
|
-
if (!sec && /abstract/i.test(section2) && full &&
|
|
3955
|
-
sec = readSection(
|
|
4613
|
+
if (!sec && /abstract/i.test(section2) && full && existsSync11(paths.md)) {
|
|
4614
|
+
sec = readSection(readFileSync11(paths.md, "utf8"), "abstract", cap);
|
|
3956
4615
|
via = " (from the abstract file)";
|
|
3957
4616
|
}
|
|
3958
4617
|
if (!sec && /declaration/i.test(section2)) {
|
|
@@ -3996,15 +4655,16 @@ function registerTools(server2) {
|
|
|
3996
4655
|
log("library_verify", pid.requested, "missing adding first");
|
|
3997
4656
|
await addEntry(id);
|
|
3998
4657
|
}
|
|
3999
|
-
const r = await
|
|
4658
|
+
const { cached, ...r } = await verifyCached(claim, id, cfg, cfg ? decider(cfg) : null, FH_LIBRARY_DIR);
|
|
4659
|
+
if (!r.error && r.score !== null) bump("verify", cached);
|
|
4000
4660
|
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
4001
|
-
log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
4002
|
-
return { content: [{ type: "text", text:
|
|
4661
|
+
log("library_verify", r.id, r.error ? `${modelFailure(r.error, cfg) ? "!! " : ""}${secs} verdict=${"error".padEnd(12)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(12)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""}${cached ? " cached" : ""} | ${claim}`);
|
|
4662
|
+
return { content: [{ type: "text", text: compact2(r) }] };
|
|
4003
4663
|
}
|
|
4004
4664
|
);
|
|
4005
4665
|
server2.tool(
|
|
4006
4666
|
"library_info",
|
|
4007
|
-
"The whole library as JSON: library directory and host; entry count and size on disk; per type (pmid, doi, dailymed, nct) the entries by level (full_text, abstract, title) and retractions; counts of retracted and not-citable entries and of errata, expressions of concern and updates; the retrieval sources in run order and which are off (tiers); the download ladder (download): the Playwright servers with the version they must run, the Chrome rungs, the Bright Data unlocker; the configuration and where each value came from \u2014 environment, .env or default, never the value (config); the decision model behind library_verify and whether it answers right now (verify). ids=true also lists every entry ID.",
|
|
4667
|
+
"The whole library as JSON: library directory and host; entry count and size on disk; per type (pmid, doi, dailymed, nct) the entries by level (full_text, abstract, title) and retractions; counts of retracted and not-citable entries and of errata, expressions of concern and updates; the retrieval sources in run order and which are off (tiers); the download ladder (download): the Playwright servers with the version they must run, the Chrome rungs, the Bright Data unlocker; the configuration and where each value came from \u2014 environment, .env or default, never the value (config); the decision model behind library_verify and whether it answers right now (verify); this server's cache hits and misses for library_verify and library_fetch since it started (cache). ids=true also lists every entry ID.",
|
|
4008
4668
|
{ ids: z.boolean().optional().describe("Also list every entry ID (type:value).") },
|
|
4009
4669
|
async ({ ids }) => {
|
|
4010
4670
|
const info = libraryInfo(FH_LIBRARY_DIR, HOST, ids === true);
|
|
@@ -4019,7 +4679,110 @@ function registerTools(server2) {
|
|
|
4019
4679
|
}
|
|
4020
4680
|
}
|
|
4021
4681
|
log("library_info", null, `entries=${info.entries}${info.verify ? ` verify ${info.verify.model} ${info.verify.ok ? `ok ${info.verify.ms} ms` : `down: ${info.verify.error}`}` : " verify not configured"}`);
|
|
4022
|
-
|
|
4682
|
+
const s = currentStats();
|
|
4683
|
+
const cache = s ? { since: s.started, verify: s.verify, fetch: s.fetch, search: s.search } : null;
|
|
4684
|
+
return { content: [{ type: "text", text: compact2({ ...info, cache }) }] };
|
|
4685
|
+
}
|
|
4686
|
+
);
|
|
4687
|
+
server2.tool(
|
|
4688
|
+
"library_fetch",
|
|
4689
|
+
`Fetch a web page as Markdown: the standard fetch tool, cached and hardened. Use it for any URL instead of a plain fetch or a browser. A page fetched in the last ${settings().refresh_after_days} days is answered from disk in milliseconds; a page a plain fetch cannot get (403, rate limit, bot wall, script-rendered) is fetched through the configured browsers and unlocker in this one call. PDFs come back as Markdown. To verify a link, title_only returns just its title line.`,
|
|
4690
|
+
{
|
|
4691
|
+
url: z.string().describe("URL to fetch"),
|
|
4692
|
+
max_length: z.number().int().min(1).max(999999).default(5e3).describe("Maximum number of characters to return."),
|
|
4693
|
+
start_index: z.number().int().min(0).default(0).describe("On return output starting at this character index, useful if a previous fetch was truncated and more context is required."),
|
|
4694
|
+
raw: z.boolean().default(false).describe("Get the actual HTML content of the requested page, without simplification."),
|
|
4695
|
+
title_only: z.boolean().default(false).describe("Return only the page's title line. The whole page is still fetched and stored; use this to verify that a link resolves and what it points to.")
|
|
4696
|
+
},
|
|
4697
|
+
async ({ url, max_length, start_index, raw, title_only }) => {
|
|
4698
|
+
const t0 = Date.now();
|
|
4699
|
+
const id = logId(url), put = (line) => log("library_fetch", id, `${line} ${url}`);
|
|
4700
|
+
const r = await libraryFetch(FH_LIBRARY_DIR, url, { max_length, start_index, raw, title_only }, `${pkg.name} ${pkg.version}`, {
|
|
4701
|
+
onTry: put,
|
|
4702
|
+
onAlert: (line) => put(`!! ${line}`)
|
|
4703
|
+
});
|
|
4704
|
+
const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
|
|
4705
|
+
if (r.meta) bump("fetch", r.cached);
|
|
4706
|
+
if (r.ok) {
|
|
4707
|
+
put(`${r.cached ? "cached" : `fetched ${secs}`} ${r.meta.format} retrieved_by=${r.meta.retrieved_by} fetched=${r.meta.fetched} chars=${r.text.length}${raw ? " raw" : ""}${title_only ? " title" : ""}${start_index ? ` start=${start_index}` : ""}`);
|
|
4708
|
+
return { content: [{ type: "text", text: r.text }] };
|
|
4709
|
+
}
|
|
4710
|
+
put(`${r.cached ? "cached failure" : `failed ${secs}`}: ${r.error.replace(url, "<url>")}`);
|
|
4711
|
+
return { content: [{ type: "text", text: r.error }], isError: true };
|
|
4712
|
+
}
|
|
4713
|
+
);
|
|
4714
|
+
server2.tool(
|
|
4715
|
+
"library_clinicaltrials_search_studies",
|
|
4716
|
+
`Search ClinicalTrials.gov by free text, intervention or condition; status ongoing | completed | all; sort relevance | newest | largest | completion. Returns the total and up to ${MAX_RESULTS} rows as arrays in columns order: NCT ID, title, status, phase, enrollment, start, primary completion, primary outcome, conditions, interventions, has_results. Link: https://clinicaltrials.gov/study/<NCT ID>. Cached ${SEARCH_AFTER_DAYS} days. One trial in detail: library_clinicaltrials_get_study_record.`,
|
|
4717
|
+
{
|
|
4718
|
+
query: z.string().optional().describe("Free-text search across all fields (title, conditions, interventions, summary, \u2026); AND / OR / NOT allowed."),
|
|
4719
|
+
intervention: z.string().optional().describe('Intervention or treatment, e.g. "tadalafil", "cognitive behavioral therapy".'),
|
|
4720
|
+
condition: z.string().optional().describe('Condition or disease, e.g. "hypertension", "type 2 diabetes".'),
|
|
4721
|
+
status: z.enum(["ongoing", "completed", "all"]).default("all").describe("ongoing: recruiting, not yet recruiting, active not recruiting, enrolling by invitation. completed: completed. all: no filter."),
|
|
4722
|
+
sort: z.enum(["relevance", "newest", "largest", "completion"]).default("relevance").describe("relevance (the registry's ranking), newest (last update first), largest (enrollment first), completion (completion date, latest first)."),
|
|
4723
|
+
max_results: z.number().int().min(1).max(MAX_RESULTS).default(20).describe("Rows to return (1\u2013100).")
|
|
4724
|
+
},
|
|
4725
|
+
async (args) => {
|
|
4726
|
+
const t0 = Date.now();
|
|
4727
|
+
const r = await searchTrials(args, FH_LIBRARY_DIR);
|
|
4728
|
+
const secs = ((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0") + "s";
|
|
4729
|
+
if (!r.ok) {
|
|
4730
|
+
log("library_search", "ct:-", `error: ${r.error}`);
|
|
4731
|
+
return { content: [{ type: "text", text: compact2({ error: r.error }) }], isError: true };
|
|
4732
|
+
}
|
|
4733
|
+
bump("search", r.cached);
|
|
4734
|
+
const q = r.result.query, parts = [q.intervention && `intervention=${q.intervention}`, q.condition && `condition=${q.condition}`, q.query && `query=${q.query}`].filter(Boolean).join(" ");
|
|
4735
|
+
log("library_search", `ct:${trialsLogId(q)}`, `${r.cached ? "cached" : `searched ${secs}`} rows=${r.result.rows.length} total=${r.result.total} status=${q.status}${q.sort !== "relevance" ? ` sort=${q.sort}` : ""} | ${parts}`);
|
|
4736
|
+
return { content: [{ type: "text", text: leanJson(r.result) }] };
|
|
4737
|
+
}
|
|
4738
|
+
);
|
|
4739
|
+
const nctEntry = async (id) => {
|
|
4740
|
+
const p = parseId(/^\d{8}$/.test(id.trim()) ? `nct:${id.trim()}` : id);
|
|
4741
|
+
if (p.error) return { error: p.error };
|
|
4742
|
+
if (p.type !== "nct") return { error: `not a trial ID: ${p.requested}` };
|
|
4743
|
+
const paths = entryPaths(FH_LIBRARY_DIR, p.type, p.id);
|
|
4744
|
+
if (!entryExists(paths)) {
|
|
4745
|
+
const r = await addEntry(p.requested);
|
|
4746
|
+
const s = JSON.parse(r.content[0].text);
|
|
4747
|
+
if (s.error) return { error: s.error };
|
|
4748
|
+
}
|
|
4749
|
+
return { paths, requested: p.requested };
|
|
4750
|
+
};
|
|
4751
|
+
server2.tool(
|
|
4752
|
+
"library_clinicaltrials_get_study_record",
|
|
4753
|
+
'One ClinicalTrials.gov trial as a review needs it: titles, status and why stopped, phase, design, enrollment, dates, sponsor, conditions, interventions, primary outcomes, eligibility, sites, result publications as pmid: IDs, has_results, brief summary. From the stored record; a missing trial is added first. The whole record: library_read("nct:\u2026").',
|
|
4754
|
+
{ nct: z.string().describe('The NCT ID: "nct:05173896", "NCT05173896" or the eight digits.') },
|
|
4755
|
+
async ({ nct }) => {
|
|
4756
|
+
const t0 = Date.now();
|
|
4757
|
+
const e = await nctEntry(nct);
|
|
4758
|
+
if ("error" in e) {
|
|
4759
|
+
log("library_trials", nct, `record error: ${e.error}`);
|
|
4760
|
+
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
4761
|
+
}
|
|
4762
|
+
const raw = e.paths.fulltextRaw("json");
|
|
4763
|
+
const r = existsSync11(raw) ? recordSummary(readFileSync11(raw, "utf8")) : null;
|
|
4764
|
+
if (!r) {
|
|
4765
|
+
log("library_trials", e.requested, "record error: no stored record");
|
|
4766
|
+
return { content: [{ type: "text", text: compact2({ error: "no stored record" }) }], isError: true };
|
|
4767
|
+
}
|
|
4768
|
+
log("library_trials", e.requested, `record ${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s ${r.status ?? ""}${r.has_results ? " results" : ""} | ${r.title}`);
|
|
4769
|
+
return { content: [{ type: "text", text: leanJson(r) }] };
|
|
4770
|
+
}
|
|
4771
|
+
);
|
|
4772
|
+
server2.tool(
|
|
4773
|
+
"library_clinicaltrials_get_study_results",
|
|
4774
|
+
'The posted results of one ClinicalTrials.gov trial \u2014 participant flow, baseline, every outcome measure as a table, adverse events \u2014 as Markdown from the stored record, capped at max_chars, outcome titles listed; library_read("nct:\u2026", section: <title>) returns one measure whole. has_results false when the registry holds none. A missing trial is added first.',
|
|
4775
|
+
{ nct: z.string().describe('The NCT ID: "nct:05173896", "NCT05173896" or the eight digits.'), max_chars: z.number().int().min(200).max(6e4).default(6e3).describe("Cap for the results text (default 6000).") },
|
|
4776
|
+
async ({ nct, max_chars }) => {
|
|
4777
|
+
const e = await nctEntry(nct);
|
|
4778
|
+
if ("error" in e) {
|
|
4779
|
+
log("library_trials", nct, `results error: ${e.error}`);
|
|
4780
|
+
return { content: [{ type: "text", text: compact2({ error: e.error }) }], isError: true };
|
|
4781
|
+
}
|
|
4782
|
+
const md = existsSync11(e.paths.fulltextMd) ? readFileSync11(e.paths.fulltextMd, "utf8") : "";
|
|
4783
|
+
const r = resultsText(e.requested.replace(/^nct:/, "NCT"), md, max_chars);
|
|
4784
|
+
log("library_trials", e.requested, `results ${r.has_results ? `${r.outcomes.length} outcomes${r.truncated ? " truncated" : ""}` : "none"}`);
|
|
4785
|
+
return { content: [{ type: "text", text: leanJson(r) }] };
|
|
4023
4786
|
}
|
|
4024
4787
|
);
|
|
4025
4788
|
server2.tool(
|
|
@@ -4039,6 +4802,8 @@ var server = new McpServer(
|
|
|
4039
4802
|
{ instructions: INSTRUCTIONS }
|
|
4040
4803
|
);
|
|
4041
4804
|
registerTools(server);
|
|
4805
|
+
startStats(pkg);
|
|
4806
|
+
log("startup", null, `${pkg.name} ${pkg.version}`);
|
|
4042
4807
|
for (const problem of resolveTiers().problems) log("startup", null, `!! ${problem}`);
|
|
4043
4808
|
var transport = new StdioServerTransport();
|
|
4044
4809
|
await server.connect(transport);
|