ai4l-library-dev 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +3642 -223
- package/package.json +5 -1
package/dist/index.js
CHANGED
|
@@ -1,218 +1,191 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
+
// src/env.ts
|
|
4
|
+
import { existsSync, readFileSync } from "fs";
|
|
5
|
+
import { join, resolve } from "path";
|
|
6
|
+
function dotenvFile(env = process.env) {
|
|
7
|
+
const named = env.AI4L_LIBRARY_ENV?.trim();
|
|
8
|
+
if (named) return named;
|
|
9
|
+
if (existsSync(resolve(".env"))) return resolve(".env");
|
|
10
|
+
const lib = env.AI4L_LIBRARY_DIR?.trim() || (existsSync("/library") ? "/library" : "library");
|
|
11
|
+
return join(resolve(lib), ".env");
|
|
12
|
+
}
|
|
13
|
+
function loadDotenv(file = dotenvFile()) {
|
|
14
|
+
if (!existsSync(file)) return [];
|
|
15
|
+
const loaded = [];
|
|
16
|
+
for (const raw of readFileSync(file, "utf8").split(/\r?\n/)) {
|
|
17
|
+
const line = raw.trim();
|
|
18
|
+
if (!line || line.startsWith("#")) continue;
|
|
19
|
+
const m = /^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(line);
|
|
20
|
+
if (!m) continue;
|
|
21
|
+
let v = m[2].trim();
|
|
22
|
+
if (v.startsWith('"') && v.endsWith('"') || v.startsWith("'") && v.endsWith("'")) v = v.slice(1, -1);
|
|
23
|
+
else v = v.replace(/\s+#.*$/, "");
|
|
24
|
+
if (process.env[m[1]] === void 0) {
|
|
25
|
+
process.env[m[1]] = v;
|
|
26
|
+
loaded.push(m[1]);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return loaded;
|
|
30
|
+
}
|
|
31
|
+
|
|
3
32
|
// src/index.ts
|
|
4
33
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
5
34
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
6
35
|
|
|
7
36
|
// src/tools.ts
|
|
8
|
-
import { readFileSync as
|
|
37
|
+
import { existsSync as existsSync10, readFileSync as readFileSync9 } from "fs";
|
|
9
38
|
import { z } from "zod";
|
|
10
39
|
|
|
11
40
|
// src/entry.ts
|
|
12
|
-
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
13
|
-
import {
|
|
14
|
-
|
|
15
|
-
return {
|
|
16
|
-
requested,
|
|
17
|
-
type,
|
|
18
|
-
id,
|
|
19
|
-
entry: null,
|
|
20
|
-
entry_dir: null,
|
|
21
|
-
library_dir: libraryDir,
|
|
22
|
-
pmid: null,
|
|
23
|
-
pmcid: null,
|
|
24
|
-
doi: null,
|
|
25
|
-
title: null,
|
|
26
|
-
first_author: null,
|
|
27
|
-
year: null,
|
|
28
|
-
publication_types: [],
|
|
29
|
-
retracted: false,
|
|
30
|
-
level: null,
|
|
31
|
-
version: null,
|
|
32
|
-
retrieved_by: null,
|
|
33
|
-
retrieved_date: null,
|
|
34
|
-
attempts: [],
|
|
35
|
-
files: []
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
function renderAbstractMd(r) {
|
|
39
|
-
const line = [
|
|
40
|
-
r.authors.join(", ") || null,
|
|
41
|
-
r.journal,
|
|
42
|
-
r.year,
|
|
43
|
-
`PMID ${r.pmid}`,
|
|
44
|
-
r.doi ? `DOI ${r.doi}` : null,
|
|
45
|
-
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
46
|
-
].filter(Boolean).join(" \xB7 ");
|
|
47
|
-
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
48
|
-
return `# ${r.title}
|
|
49
|
-
|
|
50
|
-
${line}
|
|
51
|
-
|
|
52
|
-
## Abstract
|
|
41
|
+
import { copyFileSync, existsSync as existsSync4, mkdirSync, readdirSync, readFileSync as readFileSync3, renameSync, rmSync, writeFileSync } from "fs";
|
|
42
|
+
import { join as join4 } from "path";
|
|
43
|
+
import { basename, dirname } from "path";
|
|
53
44
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
57
|
-
function renderMeta(r, files, retrievedDate, converter) {
|
|
58
|
-
const lines = [
|
|
59
|
-
["pmid", r.pmid],
|
|
60
|
-
["pmcid", r.pmcid ?? ""],
|
|
61
|
-
["doi", r.doi ?? ""],
|
|
62
|
-
["title", JSON.stringify(r.title)],
|
|
63
|
-
["first_author", r.firstAuthor ?? ""],
|
|
64
|
-
["year", r.year ?? ""],
|
|
65
|
-
["publication_types", r.publicationTypes.join(", ")],
|
|
66
|
-
["retracted", r.publicationTypes.includes("Retracted Publication") ? "yes" : "no"],
|
|
67
|
-
["level", "abstract"],
|
|
68
|
-
["version", ""],
|
|
69
|
-
["files", files.join(", ")],
|
|
70
|
-
["retrieved_by", "pubmed"],
|
|
71
|
-
["retrieved_date", retrievedDate],
|
|
72
|
-
["converted_by", converter],
|
|
73
|
-
["attempts", ""]
|
|
74
|
-
];
|
|
75
|
-
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
76
|
-
}
|
|
77
|
-
function entryExists(p) {
|
|
78
|
-
return existsSync(p.meta);
|
|
79
|
-
}
|
|
80
|
-
function writeEntry(p, r, xml, converter) {
|
|
81
|
-
mkdirSync(p.entryDir, { recursive: true });
|
|
82
|
-
const files = [basename(p.xml), basename(p.md)];
|
|
83
|
-
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
84
|
-
writeFileSync(p.xml, xml);
|
|
85
|
-
writeFileSync(p.md, renderAbstractMd(r));
|
|
86
|
-
writeFileSync(p.meta, renderMeta(r, files, today, converter));
|
|
87
|
-
}
|
|
88
|
-
function parseFrontmatter(md) {
|
|
89
|
-
const m = /^---\n([\s\S]*?)\n---/.exec(md);
|
|
90
|
-
const out = {};
|
|
91
|
-
for (const line of (m?.[1] ?? "").split("\n")) {
|
|
92
|
-
const i = line.indexOf(":");
|
|
93
|
-
if (i > 0) out[line.slice(0, i).trim()] = line.slice(i + 1).trim();
|
|
94
|
-
}
|
|
95
|
-
return out;
|
|
96
|
-
}
|
|
97
|
-
var list = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
98
|
-
var opt = (v) => v ? v : null;
|
|
99
|
-
function readStatus(p, base) {
|
|
100
|
-
const fm = parseFrontmatter(readFileSync(p.meta, "utf8"));
|
|
101
|
-
let title = opt(fm.title);
|
|
102
|
-
if (title?.startsWith('"')) {
|
|
103
|
-
try {
|
|
104
|
-
title = JSON.parse(title);
|
|
105
|
-
} catch {
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return {
|
|
109
|
-
...base,
|
|
110
|
-
entry: p.entry,
|
|
111
|
-
entry_dir: p.entryDir,
|
|
112
|
-
pmid: opt(fm.pmid),
|
|
113
|
-
pmcid: opt(fm.pmcid),
|
|
114
|
-
doi: opt(fm.doi),
|
|
115
|
-
title,
|
|
116
|
-
first_author: opt(fm.first_author),
|
|
117
|
-
year: opt(fm.year),
|
|
118
|
-
publication_types: list(fm.publication_types),
|
|
119
|
-
retracted: fm.retracted === "yes",
|
|
120
|
-
level: opt(fm.level) ?? null,
|
|
121
|
-
version: opt(fm.version),
|
|
122
|
-
retrieved_by: opt(fm.retrieved_by),
|
|
123
|
-
retrieved_date: opt(fm.retrieved_date),
|
|
124
|
-
attempts: list(fm.attempts),
|
|
125
|
-
files: list(fm.files)
|
|
126
|
-
};
|
|
127
|
-
}
|
|
45
|
+
// src/pubmed.ts
|
|
46
|
+
import { XMLParser } from "fast-xml-parser";
|
|
128
47
|
|
|
129
|
-
// src/
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
function parseId(raw) {
|
|
133
|
-
const requested = String(raw ?? "").trim();
|
|
134
|
-
const bare = /^\d+$/.test(requested);
|
|
135
|
-
const m = bare ? null : /^([A-Za-z][A-Za-z0-9_-]*):(.+)$/.exec(requested);
|
|
136
|
-
if (!bare && !m) return { requested, type: null, id: null, error: "invalid id" };
|
|
137
|
-
const type = bare ? "pmid" : m[1].toLowerCase();
|
|
138
|
-
const value = bare ? requested : m[2].trim();
|
|
139
|
-
if (!KNOWN.has(type)) return { requested, type, id: value, error: "invalid id" };
|
|
140
|
-
if (!SUPPORTED.has(type)) return { requested, type, id: value, error: "type not supported yet" };
|
|
141
|
-
if (!/^[1-9]\d*$/.test(value)) return { requested, type, id: value, error: "invalid id" };
|
|
142
|
-
return { requested, type, id: value };
|
|
143
|
-
}
|
|
48
|
+
// src/contact.ts
|
|
49
|
+
import { existsSync as existsSync3, readFileSync as readFileSync2 } from "fs";
|
|
50
|
+
import { join as join3 } from "path";
|
|
144
51
|
|
|
145
52
|
// src/layout.ts
|
|
53
|
+
import { createHash } from "crypto";
|
|
146
54
|
import { existsSync as existsSync2 } from "fs";
|
|
147
|
-
import { join, resolve } from "path";
|
|
148
|
-
var
|
|
149
|
-
const env = process.env.
|
|
150
|
-
if (env && env.trim()) return
|
|
55
|
+
import { join as join2, resolve as resolve2 } from "path";
|
|
56
|
+
var AI4L_LIBRARY_DIR = (() => {
|
|
57
|
+
const env = process.env.AI4L_LIBRARY_DIR;
|
|
58
|
+
if (env && env.trim()) return resolve2(env.trim());
|
|
151
59
|
if (existsSync2("/library")) return "/library";
|
|
152
|
-
return
|
|
60
|
+
return resolve2("library");
|
|
153
61
|
})();
|
|
62
|
+
var doiSlug = (doi) => doi.toLowerCase().replace(/[^a-z0-9.-]+/g, "_");
|
|
154
63
|
function entryDirRel(type, num) {
|
|
64
|
+
if (type === "doi") {
|
|
65
|
+
const h = createHash("sha1").update(num).digest("hex");
|
|
66
|
+
return `${type}/${h.slice(0, 2)}/${h.slice(0, 4)}/${num}`;
|
|
67
|
+
}
|
|
68
|
+
if (!/^\d+$/.test(num)) return `${type}/${num.slice(0, 2)}/${num.slice(0, 4)}/${num}`;
|
|
155
69
|
const l1 = num.length > 6 ? num.slice(0, -6) : "0";
|
|
156
70
|
const l2 = num.length > 4 ? num.slice(0, -4) : "0";
|
|
157
71
|
return `${type}/${l1}/${l2}/${num}`;
|
|
158
72
|
}
|
|
159
73
|
function entryPaths(libraryDir, type, num) {
|
|
74
|
+
if (type === "doi") num = doiSlug(num);
|
|
160
75
|
const entry = `${type}-${num}`;
|
|
161
|
-
const entryDir =
|
|
76
|
+
const entryDir = join2(libraryDir, entryDirRel(type, num));
|
|
162
77
|
return {
|
|
163
78
|
entry,
|
|
164
79
|
entryDir,
|
|
165
|
-
xml:
|
|
166
|
-
md:
|
|
167
|
-
meta:
|
|
80
|
+
xml: join2(entryDir, `${entry}-abstract.xml`),
|
|
81
|
+
md: join2(entryDir, `${entry}-abstract.md`),
|
|
82
|
+
meta: join2(entryDir, `${entry}-meta.md`),
|
|
83
|
+
fulltextMd: join2(entryDir, `${entry}-fulltext.md`),
|
|
84
|
+
fulltextRaw: (format) => join2(entryDir, `${entry}-fulltext.${format}`)
|
|
168
85
|
};
|
|
169
86
|
}
|
|
170
87
|
|
|
88
|
+
// src/contact.ts
|
|
89
|
+
var DEFAULT = "hello@forever-healthy.org";
|
|
90
|
+
function contactEmail() {
|
|
91
|
+
const file = process.env.AI4L_LIBRARY_CONFIG?.trim() || join3(AI4L_LIBRARY_DIR, "config.json");
|
|
92
|
+
let v = "";
|
|
93
|
+
if (existsSync3(file)) {
|
|
94
|
+
try {
|
|
95
|
+
v = String(JSON.parse(readFileSync2(file, "utf8")).contact_email ?? "");
|
|
96
|
+
} catch {
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
v = v.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, n) => process.env[n] ?? "").trim();
|
|
100
|
+
return v || process.env.CONTACT_EMAIL?.trim() || DEFAULT;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// src/http.ts
|
|
104
|
+
var TIMEOUT_MS = 3e4;
|
|
105
|
+
var CHROME_UA = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
|
106
|
+
var LIBRARY_UA = `ai4l-library (+https://github.com/forever-healthy/AI4L-library; ${contactEmail()})`;
|
|
107
|
+
function httpReason(status, body) {
|
|
108
|
+
const m = body.length < 4e3 && /<statusCode>([A-Z_]+)<\/statusCode>|"statusCode"\s*:\s*"([A-Z_]+)"/.exec(body);
|
|
109
|
+
return m ? `HTTP ${status} ${m[1] ?? m[2]}` : `HTTP ${status}`;
|
|
110
|
+
}
|
|
111
|
+
var DOWNLOAD_HEADERS = { "User-Agent": CHROME_UA, "Accept": "application/pdf,text/html,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.9" };
|
|
112
|
+
async function fetchBytes(url, headers = DOWNLOAD_HEADERS) {
|
|
113
|
+
let last = "unknown error", finalUrl = "", lastBody = "", lastType = "";
|
|
114
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
115
|
+
try {
|
|
116
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "follow" });
|
|
117
|
+
if (res.ok) return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "", url: res.url };
|
|
118
|
+
lastBody = (await res.text().catch(() => "")).slice(0, 4e3);
|
|
119
|
+
lastType = res.headers.get("content-type") ?? "";
|
|
120
|
+
last = res.status < 500 ? httpReason(res.status, lastBody) : `HTTP ${res.status}`;
|
|
121
|
+
finalUrl = res.url;
|
|
122
|
+
if (res.status !== 429 && res.status < 500) break;
|
|
123
|
+
} catch (e) {
|
|
124
|
+
last = e instanceof Error && e.name === "TimeoutError" ? "timeout" : e instanceof Error ? e.message : String(e);
|
|
125
|
+
}
|
|
126
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
127
|
+
}
|
|
128
|
+
throw Object.assign(new Error(last), finalUrl ? { url: finalUrl } : {}, lastBody ? { body: lastBody, contentType: lastType } : {});
|
|
129
|
+
}
|
|
130
|
+
async function fetchText(url, headers = {}) {
|
|
131
|
+
let last = "unknown error";
|
|
132
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
133
|
+
try {
|
|
134
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS), redirect: "follow" });
|
|
135
|
+
if (res.ok) return await res.text();
|
|
136
|
+
last = res.status < 500 ? httpReason(res.status, await res.text().catch(() => "")) : `HTTP ${res.status}`;
|
|
137
|
+
if (res.status !== 429 && res.status < 500) break;
|
|
138
|
+
} catch (e) {
|
|
139
|
+
last = e instanceof Error && e.name === "TimeoutError" ? "timeout" : e instanceof Error ? e.message : String(e);
|
|
140
|
+
}
|
|
141
|
+
await new Promise((r) => setTimeout(r, 1500));
|
|
142
|
+
}
|
|
143
|
+
throw new Error(last);
|
|
144
|
+
}
|
|
145
|
+
|
|
171
146
|
// src/pubmed.ts
|
|
172
|
-
|
|
147
|
+
var CORRECTION_TYPES = {
|
|
148
|
+
ErratumIn: "erratum_in",
|
|
149
|
+
RetractionIn: "retraction_in",
|
|
150
|
+
PartialRetractionIn: "partial_retraction_in",
|
|
151
|
+
RetractedandRepublishedIn: "retracted_and_republished_in",
|
|
152
|
+
ExpressionOfConcernIn: "expression_of_concern_in",
|
|
153
|
+
UpdateIn: "update_in"
|
|
154
|
+
};
|
|
155
|
+
var RETRACTION_TYPES = /* @__PURE__ */ new Set(["retraction_in", "partial_retraction_in", "retracted_and_republished_in"]);
|
|
156
|
+
var isRetractionNotice = (c) => RETRACTION_TYPES.has(c.type);
|
|
173
157
|
var EFETCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi";
|
|
174
|
-
var TIMEOUT_MS = 3e4;
|
|
175
158
|
function efetchUrl(pmid) {
|
|
176
159
|
const u = new URL(EFETCH);
|
|
177
160
|
u.searchParams.set("db", "pubmed");
|
|
178
161
|
u.searchParams.set("id", pmid);
|
|
179
162
|
u.searchParams.set("retmode", "xml");
|
|
180
163
|
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
u.searchParams.set("email", process.env.CONTACT_EMAIL);
|
|
184
|
-
}
|
|
164
|
+
u.searchParams.set("tool", "ai4l-library");
|
|
165
|
+
u.searchParams.set("email", contactEmail());
|
|
185
166
|
return u.toString();
|
|
186
167
|
}
|
|
187
168
|
async function fetchPubmedXml(pmid) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
const res = await fetch(url, { signal: AbortSignal.timeout(TIMEOUT_MS) });
|
|
193
|
-
if (res.ok) return await res.text();
|
|
194
|
-
last = new Error(`HTTP ${res.status}`);
|
|
195
|
-
if (res.status !== 429 && res.status < 500) break;
|
|
196
|
-
} catch (e) {
|
|
197
|
-
last = e;
|
|
198
|
-
}
|
|
169
|
+
try {
|
|
170
|
+
return await fetchText(efetchUrl(pmid));
|
|
171
|
+
} catch (e) {
|
|
172
|
+
throw new Error(`pubmed fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
199
173
|
}
|
|
200
|
-
const msg = last instanceof Error ? last.message : String(last);
|
|
201
|
-
throw new Error(`pubmed fetch failed: ${msg}`);
|
|
202
174
|
}
|
|
203
175
|
var parser = new XMLParser({
|
|
204
176
|
ignoreAttributes: false,
|
|
205
177
|
attributeNamePrefix: "@_",
|
|
206
178
|
parseTagValue: false,
|
|
207
179
|
processEntities: true,
|
|
208
|
-
stopNodes: ["*.ArticleTitle", "*.AbstractText"],
|
|
209
|
-
isArray: (name) => ["PubmedArticle", "AbstractText", "Author", "ArticleId", "PublicationType"].includes(name)
|
|
180
|
+
stopNodes: ["*.ArticleTitle", "*.AbstractText", "*.CoiStatement"],
|
|
181
|
+
isArray: (name) => ["PubmedArticle", "PubmedBookArticle", "AbstractText", "Author", "ArticleId", "PublicationType", "CommentsCorrections", "Grant", "AuthorList"].includes(name)
|
|
210
182
|
});
|
|
211
183
|
function text(n) {
|
|
212
184
|
if (n == null) return "";
|
|
213
185
|
if (typeof n === "object") return text(n["#text"]);
|
|
214
186
|
return String(n);
|
|
215
187
|
}
|
|
188
|
+
var plain = (n) => decodeEntities(text(n)).trim();
|
|
216
189
|
function inlineText(raw) {
|
|
217
190
|
return decodeEntities(raw.replace(/<[^>]+>/g, "")).replace(/\s+/g, " ").trim();
|
|
218
191
|
}
|
|
@@ -222,7 +195,10 @@ function decodeEntities(s) {
|
|
|
222
195
|
function parsePubmedXml(xml) {
|
|
223
196
|
const doc = parser.parse(xml);
|
|
224
197
|
const article = doc?.PubmedArticleSet?.PubmedArticle?.[0];
|
|
225
|
-
if (!article)
|
|
198
|
+
if (!article) {
|
|
199
|
+
const book = doc?.PubmedArticleSet?.PubmedBookArticle?.[0];
|
|
200
|
+
return book ? parseBook(book) : null;
|
|
201
|
+
}
|
|
226
202
|
const cit = article.MedlineCitation ?? {};
|
|
227
203
|
const art = cit.Article ?? {};
|
|
228
204
|
const ids = article.PubmedData?.ArticleIdList?.ArticleId ?? [];
|
|
@@ -230,15 +206,15 @@ function parsePubmedXml(xml) {
|
|
|
230
206
|
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
231
207
|
return hit ? text(hit).trim() || null : null;
|
|
232
208
|
};
|
|
233
|
-
const authorList = art.AuthorList?.Author ?? [];
|
|
209
|
+
const authorList = art.AuthorList?.[0]?.Author ?? [];
|
|
234
210
|
const authors = authorList.map((a) => {
|
|
235
|
-
if (a.CollectiveName) return
|
|
236
|
-
const last =
|
|
237
|
-
const ini =
|
|
211
|
+
if (a.CollectiveName) return plain(a.CollectiveName);
|
|
212
|
+
const last = plain(a.LastName);
|
|
213
|
+
const ini = plain(a.Initials);
|
|
238
214
|
return [last, ini].filter(Boolean).join(" ");
|
|
239
215
|
}).filter(Boolean);
|
|
240
216
|
const first = authorList[0];
|
|
241
|
-
const firstAuthor = first ?
|
|
217
|
+
const firstAuthor = first ? plain(first.CollectiveName ?? first.LastName) || null : null;
|
|
242
218
|
const pub = art.Journal?.JournalIssue?.PubDate ?? {};
|
|
243
219
|
let year = text(pub.Year).trim() || null;
|
|
244
220
|
if (!year) year = /\d{4}/.exec(text(pub.MedlineDate))?.[0] ?? null;
|
|
@@ -246,85 +222,3528 @@ function parsePubmedXml(xml) {
|
|
|
246
222
|
label: typeof a === "object" ? a["@_Label"] ?? null : null,
|
|
247
223
|
text: inlineText(text(a))
|
|
248
224
|
})).filter((a) => a.text);
|
|
225
|
+
const corrections = (cit.CommentsCorrectionsList?.CommentsCorrections ?? []).flatMap((c) => {
|
|
226
|
+
const type = CORRECTION_TYPES[String(c["@_RefType"] ?? "")];
|
|
227
|
+
if (!type) return [];
|
|
228
|
+
return [{ type, pmid: text(c.PMID).trim() || null, source: inlineText(text(c.RefSource)) }];
|
|
229
|
+
});
|
|
230
|
+
const funding = [];
|
|
231
|
+
for (const g of art.GrantList?.Grant ?? []) {
|
|
232
|
+
const a = plain(g.Agency);
|
|
233
|
+
if (a && !funding.includes(a)) funding.push(a);
|
|
234
|
+
}
|
|
249
235
|
return {
|
|
236
|
+
coi: inlineText(text(cit.CoiStatement)) || null,
|
|
237
|
+
funding,
|
|
250
238
|
pmid: text(cit.PMID).trim() || idOf("pubmed") || "",
|
|
251
239
|
pmcid: idOf("pmc"),
|
|
252
240
|
doi: idOf("doi"),
|
|
253
241
|
title: inlineText(text(art.ArticleTitle)),
|
|
254
242
|
authors,
|
|
255
243
|
firstAuthor,
|
|
256
|
-
journal:
|
|
244
|
+
journal: plain(art.Journal?.Title) || null,
|
|
257
245
|
year,
|
|
258
|
-
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(
|
|
259
|
-
abstract
|
|
246
|
+
publicationTypes: (art.PublicationTypeList?.PublicationType ?? []).map(plain).filter(Boolean),
|
|
247
|
+
abstract,
|
|
248
|
+
corrections
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
function parseBook(b) {
|
|
252
|
+
const d = b.BookDocument ?? {};
|
|
253
|
+
const book = d.Book ?? {};
|
|
254
|
+
const ids = [...d.ArticleIdList?.ArticleId ?? [], ...b.PubmedBookData?.ArticleIdList?.ArticleId ?? []];
|
|
255
|
+
const idOf = (t) => {
|
|
256
|
+
const hit = ids.find((i) => i["@_IdType"] === t);
|
|
257
|
+
return hit ? text(hit).trim() || null : null;
|
|
258
|
+
};
|
|
259
|
+
const lists = [...d.AuthorList ?? [], ...book.AuthorList ?? []];
|
|
260
|
+
const pick = lists.find((l) => (l["@_Type"] ?? "authors") === "authors" && (l.Author ?? []).length) ?? lists.find((l) => (l.Author ?? []).length);
|
|
261
|
+
const people = pick?.Author ?? [];
|
|
262
|
+
const authors = people.map((a) => a.CollectiveName ? plain(a.CollectiveName) : [plain(a.LastName), plain(a.Initials)].filter(Boolean).join(" ")).filter(Boolean);
|
|
263
|
+
const first = people[0];
|
|
264
|
+
const abstract = (d.Abstract?.AbstractText ?? []).map((a) => ({ label: typeof a === "object" ? a["@_Label"] ?? null : null, text: inlineText(text(a)) })).filter((a) => a.text);
|
|
265
|
+
const types = ["Book", ...(d.PublicationType ?? []).map(plain).filter(Boolean)];
|
|
266
|
+
return {
|
|
267
|
+
pmid: text(d.PMID).trim() || idOf("pubmed") || "",
|
|
268
|
+
pmcid: null,
|
|
269
|
+
doi: idOf("doi") ?? (text(book.ELocationID).trim() || null),
|
|
270
|
+
title: inlineText(text(d.ArticleTitle)) || inlineText(text(book.BookTitle)),
|
|
271
|
+
authors,
|
|
272
|
+
firstAuthor: first ? plain(first.CollectiveName ?? first.LastName) || null : null,
|
|
273
|
+
journal: plain(book.Publisher?.PublisherName) || null,
|
|
274
|
+
year: text(book.PubDate?.Year).trim() || null,
|
|
275
|
+
publicationTypes: [...new Set(types)],
|
|
276
|
+
abstract,
|
|
277
|
+
corrections: [],
|
|
278
|
+
coi: null,
|
|
279
|
+
funding: []
|
|
260
280
|
};
|
|
261
281
|
}
|
|
262
282
|
|
|
263
|
-
// src/
|
|
264
|
-
|
|
283
|
+
// src/spl.ts
|
|
284
|
+
import { XMLParser as XMLParser2 } from "fast-xml-parser";
|
|
285
|
+
var DAILYMED_PAGE = (setid) => `https://dailymed.nlm.nih.gov/dailymed/drugInfo.cfm?setid=${setid}`;
|
|
286
|
+
var SPL_URL = (setid) => `https://dailymed.nlm.nih.gov/dailymed/services/v2/spls/${setid}.xml`;
|
|
287
|
+
async function fetchSpl(setid) {
|
|
288
|
+
const xml = await fetchText(SPL_URL(setid), { "User-Agent": LIBRARY_UA, Accept: "application/xml, */*;q=0.8" });
|
|
289
|
+
if (!/<document[\s>]/.test(xml.slice(0, 3e3))) throw new Error("dailymed: not an SPL document");
|
|
290
|
+
return xml;
|
|
291
|
+
}
|
|
292
|
+
var parser2 = new XMLParser2({ preserveOrder: true, ignoreAttributes: false, attributeNamePrefix: "@_", parseTagValue: false, trimValues: false, processEntities: true, htmlEntities: true });
|
|
293
|
+
var tagOf = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
294
|
+
var kids = (n) => n[tagOf(n)] ?? [];
|
|
295
|
+
var attr = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
296
|
+
var squash = (s) => s.replace(/\s+/g, " ").trim();
|
|
297
|
+
var child = (n, tag) => kids(n).find((x) => tagOf(x) === tag);
|
|
298
|
+
function findDeep(nodes, tag) {
|
|
299
|
+
for (const n of nodes) {
|
|
300
|
+
if ("#text" in n) continue;
|
|
301
|
+
if (tagOf(n) === tag) return n;
|
|
302
|
+
const h = findDeep(kids(n), tag);
|
|
303
|
+
if (h) return h;
|
|
304
|
+
}
|
|
305
|
+
return void 0;
|
|
306
|
+
}
|
|
307
|
+
function inline(nodes) {
|
|
308
|
+
let out = "";
|
|
309
|
+
for (const n of nodes) {
|
|
310
|
+
if ("#text" in n) {
|
|
311
|
+
out += String(n["#text"]);
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
const t = tagOf(n);
|
|
315
|
+
const inner = () => inline(kids(n));
|
|
316
|
+
switch (t) {
|
|
317
|
+
case "content": {
|
|
318
|
+
const style = attr(n, "styleCode").toLowerCase();
|
|
319
|
+
const s = inner();
|
|
320
|
+
out += /bold/.test(style) ? `**${s}**` : /italic/.test(style) ? `*${s}*` : s;
|
|
321
|
+
break;
|
|
322
|
+
}
|
|
323
|
+
case "sup":
|
|
324
|
+
out += `^${inner()}^`;
|
|
325
|
+
break;
|
|
326
|
+
case "sub":
|
|
327
|
+
out += `~${inner()}~`;
|
|
328
|
+
break;
|
|
329
|
+
case "br":
|
|
330
|
+
out += " ";
|
|
331
|
+
break;
|
|
332
|
+
case "footnote":
|
|
333
|
+
out += ` (${squash(inner())})`;
|
|
334
|
+
break;
|
|
335
|
+
case "footnoteRef":
|
|
336
|
+
case "renderMultiMedia":
|
|
337
|
+
break;
|
|
338
|
+
default:
|
|
339
|
+
out += inner();
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
return out;
|
|
343
|
+
}
|
|
344
|
+
function table(n) {
|
|
345
|
+
const row2 = (x) => kids(x).filter((c) => ["td", "th"].includes(tagOf(c))).map((c) => squash(inline(kids(c))).replace(/\|/g, "\\|"));
|
|
346
|
+
const collect = (nodes, into) => {
|
|
347
|
+
for (const x of nodes) {
|
|
348
|
+
if ("#text" in x) continue;
|
|
349
|
+
if (tagOf(x) === "tr") into.push(row2(x));
|
|
350
|
+
else if (!["thead", "tfoot"].includes(tagOf(x))) collect(kids(x), into);
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
const head = [], body = [], foot = [];
|
|
354
|
+
for (const x of kids(n)) {
|
|
355
|
+
const t = tagOf(x);
|
|
356
|
+
if (t === "thead") collect(kids(x), head);
|
|
357
|
+
else if (t === "tfoot") collect(kids(x), foot);
|
|
358
|
+
else if (t === "tr") body.push(row2(x));
|
|
359
|
+
else if (!("#text" in x) && t !== "caption") collect(kids(x), body);
|
|
360
|
+
}
|
|
361
|
+
const rows = [...head, ...body];
|
|
362
|
+
if (!rows.length) return "";
|
|
363
|
+
const width = Math.max(...rows.map((r) => r.length));
|
|
364
|
+
const line = (r) => `| ${[...r, ...Array(width - r.length).fill("")].join(" | ")} |`;
|
|
365
|
+
const cap = child(n, "caption");
|
|
366
|
+
const title = cap ? squash(inline(kids(cap))) : "";
|
|
367
|
+
const notes = foot.map((r) => r.filter(Boolean).join(" ")).filter(Boolean);
|
|
368
|
+
return [title ? `**${title}**` : "", line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line), ...notes.length ? ["", ...notes] : []].filter((l, i) => l || i > 0).join("\n");
|
|
369
|
+
}
|
|
370
|
+
function blocks(nodes, depth) {
|
|
371
|
+
const out = [];
|
|
372
|
+
for (const n of nodes) {
|
|
373
|
+
if ("#text" in n) {
|
|
374
|
+
const t2 = squash(String(n["#text"]));
|
|
375
|
+
if (t2) out.push(t2);
|
|
376
|
+
continue;
|
|
377
|
+
}
|
|
378
|
+
const t = tagOf(n);
|
|
379
|
+
const c = kids(n);
|
|
380
|
+
switch (t) {
|
|
381
|
+
case "paragraph": {
|
|
382
|
+
const s = squash(inline(c.filter((x) => !["table", "list"].includes(tagOf(x)))));
|
|
383
|
+
if (s) out.push(s);
|
|
384
|
+
out.push(...blocks(c.filter((x) => ["table", "list"].includes(tagOf(x))), depth));
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
387
|
+
case "list": {
|
|
388
|
+
const ordered = /^(ordered|numbered|arabic|roman|alpha)/i.test(attr(n, "listType"));
|
|
389
|
+
let i = 0;
|
|
390
|
+
for (const it of c.filter((x) => tagOf(x) === "item")) {
|
|
391
|
+
i++;
|
|
392
|
+
const body = blocks(kids(it).filter((x) => tagOf(x) !== "caption"), depth).join(" ");
|
|
393
|
+
if (body) out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
394
|
+
}
|
|
395
|
+
break;
|
|
396
|
+
}
|
|
397
|
+
case "table": {
|
|
398
|
+
const md = table(n);
|
|
399
|
+
if (md) out.push(md);
|
|
400
|
+
break;
|
|
401
|
+
}
|
|
402
|
+
case "renderMultiMedia":
|
|
403
|
+
case "observationMedia":
|
|
404
|
+
case "subject":
|
|
405
|
+
break;
|
|
406
|
+
case "section":
|
|
407
|
+
out.push(...section(n, depth));
|
|
408
|
+
break;
|
|
409
|
+
case "component":
|
|
410
|
+
case "text":
|
|
411
|
+
case "excerpt":
|
|
412
|
+
case "highlight":
|
|
413
|
+
default:
|
|
414
|
+
out.push(...blocks(c, depth));
|
|
415
|
+
break;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
return out;
|
|
419
|
+
}
|
|
420
|
+
var SKIP_CODES = /* @__PURE__ */ new Set(["48780-1", "51945-4"]);
|
|
421
|
+
var CODE_NAMES = { "34066-1": "Boxed Warning", "43683-2": "Recent Major Changes" };
|
|
422
|
+
var plain2 = (nodes) => {
|
|
423
|
+
let out = "";
|
|
424
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : tagOf(n) === "br" ? " " : plain2(kids(n));
|
|
425
|
+
return out;
|
|
426
|
+
};
|
|
427
|
+
function section(n, depth) {
|
|
428
|
+
const code = child(n, "code");
|
|
429
|
+
const c = code ? attr(code, "code") : "";
|
|
430
|
+
if (SKIP_CODES.has(c)) return [];
|
|
431
|
+
const title = child(n, "title");
|
|
432
|
+
let head = title ? squash(plain2(kids(title))) : "";
|
|
433
|
+
if (c === "34066-1" && head && !/boxed warning/i.test(head)) head = `Boxed Warning: ${head}`;
|
|
434
|
+
if (!head && CODE_NAMES[c]) head = CODE_NAMES[c];
|
|
435
|
+
const out = [];
|
|
436
|
+
if (head) out.push(`${"#".repeat(Math.min(depth, 6))} ${head}`);
|
|
437
|
+
const rest = kids(n).filter((x) => !["title", "code", "id", "effectiveTime"].includes(tagOf(x)));
|
|
438
|
+
out.push(...blocks(rest, head ? depth + 1 : depth));
|
|
439
|
+
return out;
|
|
440
|
+
}
|
|
441
|
+
var isoDate = (v) => /^\d{8}$/.test(v) ? `${v.slice(0, 4)}-${v.slice(4, 6)}-${v.slice(6, 8)}` : v || null;
|
|
442
|
+
function parseSpl(xml) {
|
|
443
|
+
const doc = parser2.parse(xml);
|
|
444
|
+
const d = doc.find((x) => tagOf(x) === "document");
|
|
445
|
+
if (!d) return null;
|
|
446
|
+
const k = kids(d);
|
|
447
|
+
const setid = attr(child(d, "setId") ?? {}, "root").toLowerCase();
|
|
448
|
+
const version = attr(child(d, "versionNumber") ?? {}, "value") || null;
|
|
449
|
+
const effective = isoDate(attr(child(d, "effectiveTime") ?? {}, "value"));
|
|
450
|
+
const named = (nodes) => {
|
|
451
|
+
for (const n of nodes) {
|
|
452
|
+
if ("#text" in n) continue;
|
|
453
|
+
if (tagOf(n) === "manufacturedProduct") {
|
|
454
|
+
const inner2 = child(n, "manufacturedProduct") ?? n;
|
|
455
|
+
if (child(inner2, "name")) return inner2;
|
|
456
|
+
}
|
|
457
|
+
const h = named(kids(n));
|
|
458
|
+
if (h) return h;
|
|
459
|
+
}
|
|
460
|
+
return void 0;
|
|
461
|
+
};
|
|
462
|
+
const inner = named(k);
|
|
463
|
+
const brand = inner ? squash(inline(kids(child(inner, "name") ?? { name: [] }))) || null : null;
|
|
464
|
+
const genericNode = inner ? findDeep(kids(inner), "genericMedicine") : void 0;
|
|
465
|
+
const generic = genericNode ? squash(inline(kids(child(genericNode, "name") ?? { name: [] }))) || null : null;
|
|
466
|
+
const formNode = inner ? child(inner, "formCode") : void 0;
|
|
467
|
+
const form = formNode ? attr(formNode, "displayName").toLowerCase() || null : null;
|
|
468
|
+
const org = findDeep(k, "representedOrganization");
|
|
469
|
+
const labeler = org ? squash(inline(kids(child(org, "name") ?? { name: [] }))) || null : null;
|
|
470
|
+
const body = findDeep(k, "structuredBody");
|
|
471
|
+
const sections2 = body ? blocks(kids(body), 2) : [];
|
|
472
|
+
const docTitle = child(d, "title");
|
|
473
|
+
const lines = [];
|
|
474
|
+
let cur = "";
|
|
475
|
+
for (const x of docTitle ? kids(docTitle) : []) {
|
|
476
|
+
if (!("#text" in x) && tagOf(x) === "br") {
|
|
477
|
+
lines.push(cur);
|
|
478
|
+
cur = "";
|
|
479
|
+
} else cur += "#text" in x ? String(x["#text"]) : inline([x]);
|
|
480
|
+
}
|
|
481
|
+
lines.push(cur);
|
|
482
|
+
const highlights = lines.map(squash).filter((l) => l && !/^these highlights do not include/i.test(l) && !/^see full prescribing information/i.test(l)).map((l) => l.replace(/^see full prescribing information for [^.]+\.\s*/i, "")).filter(Boolean);
|
|
483
|
+
const approvalYear = /initial u\.?s\.? approval:?\s*(\d{4})/i.exec(highlights.join(" "))?.[1] ?? null;
|
|
484
|
+
const productLine = highlights.find((l) => !/initial u\.?s\.? approval/i.test(l)) ?? null;
|
|
485
|
+
const title = [brand, generic ? `(${generic})` : "", form].filter(Boolean).join(" ") || productLine || setid;
|
|
486
|
+
return { setid, version, effective, brand, generic, form, labeler, title: `${title} \u2014 prescribing information`, highlights, approvalYear, sections: sections2 };
|
|
487
|
+
}
|
|
488
|
+
function headLineSpl(s) {
|
|
489
|
+
return [s.labeler, `DailyMed set ID ${s.setid}`, s.version ? `version ${s.version}` : "", s.effective ? `effective ${s.effective}` : "", s.approvalYear ? `initial U.S. approval ${s.approvalYear}` : "", DAILYMED_PAGE(s.setid)].filter(Boolean).join(" \xB7 ");
|
|
490
|
+
}
|
|
491
|
+
function renderLabelMd(s) {
|
|
492
|
+
const parts = [`# ${s.title}`, "", headLineSpl(s), ""];
|
|
493
|
+
if (s.highlights.length) parts.push("## Highlights", "", ...s.highlights.flatMap((l) => [l, ""]));
|
|
494
|
+
parts.push(...s.sections.flatMap((b) => [b, ""]));
|
|
495
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// src/ctgov.ts
|
|
499
|
+
var CTGOV_PAGE = (nctid) => `https://clinicaltrials.gov/study/${nctid}`;
|
|
500
|
+
var API = (nctid) => `https://clinicaltrials.gov/api/v2/studies/${nctid}?format=json`;
|
|
501
|
+
async function fetchStudy(nctid) {
|
|
502
|
+
const json2 = await fetchText(API(nctid), { "User-Agent": LIBRARY_UA, Accept: "application/json" });
|
|
503
|
+
if (!/"protocolSection"/.test(json2.slice(0, 2e3))) throw new Error("ctgov: not a study record");
|
|
504
|
+
return json2;
|
|
505
|
+
}
|
|
506
|
+
var STATUS = {
|
|
507
|
+
NOT_YET_RECRUITING: "Not yet recruiting",
|
|
508
|
+
RECRUITING: "Recruiting",
|
|
509
|
+
ENROLLING_BY_INVITATION: "Enrolling by invitation",
|
|
510
|
+
ACTIVE_NOT_RECRUITING: "Active, not recruiting",
|
|
511
|
+
SUSPENDED: "Suspended",
|
|
512
|
+
TERMINATED: "Terminated",
|
|
513
|
+
COMPLETED: "Completed",
|
|
514
|
+
WITHDRAWN: "Withdrawn",
|
|
515
|
+
UNKNOWN: "Unknown status",
|
|
516
|
+
AVAILABLE: "Available",
|
|
517
|
+
NO_LONGER_AVAILABLE: "No longer available",
|
|
518
|
+
TEMPORARILY_NOT_AVAILABLE: "Temporarily not available",
|
|
519
|
+
APPROVED_FOR_MARKETING: "Approved for marketing",
|
|
520
|
+
WITHHELD: "Withheld"
|
|
521
|
+
};
|
|
522
|
+
var PHASE = { EARLY_PHASE1: "Early Phase 1", PHASE1: "Phase 1", PHASE2: "Phase 2", PHASE3: "Phase 3", PHASE4: "Phase 4", NA: "Not applicable" };
|
|
523
|
+
var words = (v) => {
|
|
524
|
+
const s = String(v ?? "");
|
|
525
|
+
if (!s) return "";
|
|
526
|
+
if (STATUS[s]) return STATUS[s];
|
|
527
|
+
if (PHASE[s]) return PHASE[s];
|
|
528
|
+
const w = s.toLowerCase().replace(/_/g, " ");
|
|
529
|
+
return w.charAt(0).toUpperCase() + w.slice(1);
|
|
530
|
+
};
|
|
531
|
+
var dated = (d) => d?.date ? `${d.date}${d.type ? ` (${String(d.type).toLowerCase()})` : ""}` : "";
|
|
532
|
+
var list = (a) => Array.isArray(a) ? a : [];
|
|
533
|
+
var row = (cells) => `| ${cells.map((c) => c.replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()).join(" | ")} |`;
|
|
534
|
+
var table2 = (head, rows) => rows.length ? [row(head), `| ${head.map(() => "---").join(" | ")} |`, ...rows.map(row)].join("\n") : "";
|
|
535
|
+
var para = (s) => String(s ?? "").replace(/\r/g, "").trim();
|
|
536
|
+
function parseStudy(json2) {
|
|
537
|
+
let d;
|
|
265
538
|
try {
|
|
266
|
-
|
|
539
|
+
d = JSON.parse(json2);
|
|
267
540
|
} catch {
|
|
268
|
-
return
|
|
541
|
+
return null;
|
|
269
542
|
}
|
|
270
|
-
|
|
271
|
-
|
|
543
|
+
const ps = d?.protocolSection;
|
|
544
|
+
if (!ps) return null;
|
|
545
|
+
const idm = ps.identificationModule ?? {}, st = ps.statusModule ?? {}, sp = ps.sponsorCollaboratorsModule ?? {}, de = ps.descriptionModule ?? {};
|
|
546
|
+
const dm = ps.designModule ?? {}, ai = ps.armsInterventionsModule ?? {}, om = ps.outcomesModule ?? {}, el = ps.eligibilityModule ?? {}, cl = ps.contactsLocationsModule ?? {}, rf = ps.referencesModule ?? {};
|
|
547
|
+
const nctid = String(idm.nctId ?? "").toUpperCase();
|
|
548
|
+
const out = [];
|
|
549
|
+
const sec = (title, lines) => {
|
|
550
|
+
const body = lines.filter((l) => !!l && l.trim() !== "");
|
|
551
|
+
if (body.length) out.push(`## ${title}`, ...body);
|
|
552
|
+
};
|
|
553
|
+
const secondary = list(idm.secondaryIdInfos).map((s) => `${words(s.type)}${s.domain ? ` (${s.domain})` : ""}: ${s.id}`);
|
|
554
|
+
sec("Identification", [
|
|
555
|
+
idm.officialTitle ? `Official title: ${para(idm.officialTitle)}` : null,
|
|
556
|
+
idm.acronym ? `Acronym: ${idm.acronym}` : null,
|
|
557
|
+
idm.organization?.fullName ? `Organization: ${idm.organization.fullName}` : null,
|
|
558
|
+
secondary.length ? `Secondary IDs: ${secondary.join("; ")}` : null
|
|
559
|
+
]);
|
|
560
|
+
sec("Status", [
|
|
561
|
+
`Overall status: ${words(st.overallStatus)}${st.whyStopped ? ` \u2014 why stopped: ${para(st.whyStopped)}` : ""}`,
|
|
562
|
+
st.startDateStruct ? `Start: ${dated(st.startDateStruct)}` : null,
|
|
563
|
+
st.primaryCompletionDateStruct ? `Primary completion: ${dated(st.primaryCompletionDateStruct)}` : null,
|
|
564
|
+
st.completionDateStruct ? `Completion: ${dated(st.completionDateStruct)}` : null,
|
|
565
|
+
st.studyFirstPostDateStruct ? `First posted: ${dated(st.studyFirstPostDateStruct)}` : null,
|
|
566
|
+
st.lastUpdatePostDateStruct ? `Last update posted: ${dated(st.lastUpdatePostDateStruct)}` : null,
|
|
567
|
+
st.statusVerifiedDate ? `Status verified: ${st.statusVerifiedDate}` : null,
|
|
568
|
+
`Results posted: ${d.hasResults ? "yes" : "no"}`
|
|
569
|
+
]);
|
|
570
|
+
const rp = sp.responsibleParty;
|
|
571
|
+
sec("Sponsor and collaborators", [
|
|
572
|
+
sp.leadSponsor?.name ? `Lead sponsor: ${sp.leadSponsor.name}${sp.leadSponsor.class ? ` (${words(sp.leadSponsor.class).toLowerCase()})` : ""}` : null,
|
|
573
|
+
list(sp.collaborators).length ? `Collaborators: ${list(sp.collaborators).map((c) => c.name).join("; ")}` : null,
|
|
574
|
+
rp ? `Responsible party: ${[words(rp.type).toLowerCase(), rp.investigatorFullName, rp.investigatorTitle, rp.investigatorAffiliation].filter(Boolean).join(", ")}` : null
|
|
575
|
+
]);
|
|
576
|
+
sec("Description", [de.briefSummary ? para(de.briefSummary) : null, de.detailedDescription ? para(de.detailedDescription) : null]);
|
|
577
|
+
sec("Conditions", [list(ps.conditionsModule?.conditions).length ? list(ps.conditionsModule.conditions).join("; ") : null, list(ps.conditionsModule?.keywords).length ? `Keywords: ${list(ps.conditionsModule.keywords).join("; ")}` : null]);
|
|
578
|
+
const di = dm.designInfo ?? {};
|
|
579
|
+
const mi = di.maskingInfo ?? {};
|
|
580
|
+
sec("Design", [
|
|
581
|
+
dm.studyType ? `Study type: ${words(dm.studyType)}` : null,
|
|
582
|
+
list(dm.phases).length ? `Phase: ${list(dm.phases).map(words).join(", ")}` : null,
|
|
583
|
+
di.allocation ? `Allocation: ${words(di.allocation)}` : null,
|
|
584
|
+
di.interventionModel ? `Intervention model: ${words(di.interventionModel)}${di.interventionModelDescription ? ` \u2014 ${para(di.interventionModelDescription)}` : ""}` : null,
|
|
585
|
+
di.primaryPurpose ? `Primary purpose: ${words(di.primaryPurpose)}` : null,
|
|
586
|
+
mi.masking ? `Masking: ${words(mi.masking)}${list(mi.whoMasked).length ? ` (${list(mi.whoMasked).map((w) => words(w).toLowerCase()).join(", ")})` : ""}${mi.maskingDescription ? ` \u2014 ${para(mi.maskingDescription)}` : ""}` : null,
|
|
587
|
+
dm.enrollmentInfo?.count !== void 0 ? `Enrollment: ${dm.enrollmentInfo.count}${dm.enrollmentInfo.type ? ` (${String(dm.enrollmentInfo.type).toLowerCase()})` : ""}` : null
|
|
588
|
+
]);
|
|
589
|
+
const arms = list(ai.armGroups).map((a) => `- **${a.label}**${a.type ? ` (${words(a.type).toLowerCase()})` : ""}${a.description ? `: ${para(a.description)}` : ""}${list(a.interventionNames).length ? ` \u2014 ${list(a.interventionNames).join("; ")}` : ""}`);
|
|
590
|
+
const ints = list(ai.interventions).map((i) => `- **${words(i.type)}: ${i.name}**${list(i.otherNames).length ? ` (${list(i.otherNames).join(", ")})` : ""}${i.description ? `: ${para(i.description)}` : ""}${list(i.armGroupLabels).length ? ` \u2014 arms: ${list(i.armGroupLabels).join("; ")}` : ""}`);
|
|
591
|
+
sec("Arms and interventions", [arms.length ? "Arms:" : null, ...arms, ints.length ? "Interventions:" : null, ...ints]);
|
|
592
|
+
const outcome = (o) => `- **${para(o.measure)}**${o.description ? `: ${para(o.description)}` : ""}${o.timeFrame ? ` Time frame: ${para(o.timeFrame)}.` : ""}`;
|
|
593
|
+
sec("Outcomes", [
|
|
594
|
+
list(om.primaryOutcomes).length ? "### Primary outcomes" : null,
|
|
595
|
+
...list(om.primaryOutcomes).map(outcome),
|
|
596
|
+
list(om.secondaryOutcomes).length ? "### Secondary outcomes" : null,
|
|
597
|
+
...list(om.secondaryOutcomes).map(outcome),
|
|
598
|
+
list(om.otherOutcomes).length ? "### Other outcomes" : null,
|
|
599
|
+
...list(om.otherOutcomes).map(outcome)
|
|
600
|
+
]);
|
|
601
|
+
sec("Eligibility", [
|
|
602
|
+
[el.sex ? `Sex: ${words(el.sex).toLowerCase()}` : "", el.minimumAge || el.maximumAge ? `Age: ${el.minimumAge ?? "no minimum"} to ${el.maximumAge ?? "no maximum"}` : "", el.healthyVolunteers !== void 0 ? `Healthy volunteers: ${el.healthyVolunteers ? "yes" : "no"}` : ""].filter(Boolean).join(" \xB7 "),
|
|
603
|
+
el.eligibilityCriteria ? para(el.eligibilityCriteria) : null
|
|
604
|
+
]);
|
|
605
|
+
const countries = [...new Set(list(cl.locations).map((l) => l.country).filter(Boolean))];
|
|
606
|
+
sec("Locations", [countries.length ? `${list(cl.locations).length} sites in ${countries.join(", ")}` : null, list(cl.overallOfficials).length ? `Overall officials: ${list(cl.overallOfficials).map((o) => [o.name, o.role ? words(o.role).toLowerCase() : "", o.affiliation].filter(Boolean).join(", ")).join("; ")}` : null]);
|
|
607
|
+
const refs = list(rf.references);
|
|
608
|
+
sec("References", refs.map((r) => `- ${para(r.citation)}${r.pmid ? ` pmid:${r.pmid}` : ""}${r.type ? ` (${words(r.type).toLowerCase()})` : ""}`));
|
|
609
|
+
const publications = refs.filter((r) => r.pmid && ["RESULT", "DERIVED"].includes(String(r.type))).map((r) => `pmid:${r.pmid}`);
|
|
610
|
+
if (d.hasResults && d.resultsSection) out.push(...results(d.resultsSection));
|
|
611
|
+
return {
|
|
612
|
+
nctid,
|
|
613
|
+
title: para(idm.briefTitle) || nctid,
|
|
614
|
+
acronym: idm.acronym ?? null,
|
|
615
|
+
sponsor: sp.leadSponsor?.name ?? null,
|
|
616
|
+
firstPosted: st.studyFirstPostDateStruct?.date ?? null,
|
|
617
|
+
lastUpdate: st.lastUpdatePostDateStruct?.date ?? null,
|
|
618
|
+
status: st.overallStatus ? words(st.overallStatus) : null,
|
|
619
|
+
hasResults: !!d.hasResults,
|
|
620
|
+
publications,
|
|
621
|
+
sections: out
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
function results(rs) {
|
|
625
|
+
const out = ["## Results"];
|
|
626
|
+
const pf = rs.participantFlowModule;
|
|
627
|
+
if (pf) {
|
|
628
|
+
out.push("### Participant flow");
|
|
629
|
+
const groups = list(pf.groups);
|
|
630
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
631
|
+
if (pf.recruitmentDetails) out.push(para(pf.recruitmentDetails));
|
|
632
|
+
for (const p of list(pf.periods)) {
|
|
633
|
+
const head = ["Period: " + (p.title ?? ""), ...groups.map((g) => g.title)];
|
|
634
|
+
const rows = list(p.milestones).map((m) => [words(m.type), ...groups.map((g) => list(m.achievements).find((a) => a.groupId === g.id)?.numSubjects ?? "")]);
|
|
635
|
+
for (const dw of list(p.dropWithdraws)) rows.push([`Not completed: ${dw.type}`, ...groups.map((g) => list(dw.reasons).find((r) => r.groupId === g.id)?.numSubjects ?? "")]);
|
|
636
|
+
out.push(table2(head, rows));
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
const bc = rs.baselineCharacteristicsModule;
|
|
640
|
+
if (bc) {
|
|
641
|
+
out.push("### Baseline characteristics");
|
|
642
|
+
const groups = list(bc.groups);
|
|
643
|
+
out.push(...groups.map((g) => `- **${g.title}**${g.description ? `: ${para(g.description)}` : ""}`));
|
|
644
|
+
for (const m of list(bc.measures)) {
|
|
645
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? words(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
646
|
+
const rows = [];
|
|
647
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
648
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
649
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
650
|
+
})]);
|
|
651
|
+
out.push(`**${m.title}**${unit ? ` (${unit})` : ""}`, table2(["", ...groups.map((g) => g.title)], rows));
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const om = rs.outcomeMeasuresModule;
|
|
655
|
+
if (om) {
|
|
656
|
+
out.push("### Outcome measures");
|
|
657
|
+
for (const m of list(om.outcomeMeasures)) {
|
|
658
|
+
const groups = list(m.groups);
|
|
659
|
+
const unit = [m.paramType ? words(m.paramType).toLowerCase() : "", m.dispersionType ? String(m.dispersionType).toLowerCase() : "", m.unitOfMeasure].filter(Boolean).join(", ");
|
|
660
|
+
out.push(`#### ${m.title}${m.type ? ` (${words(m.type).toLowerCase()})` : ""}`);
|
|
661
|
+
if (m.description) out.push(para(m.description));
|
|
662
|
+
out.push([m.timeFrame ? `Time frame: ${para(m.timeFrame)}.` : "", unit ? `Measure: ${unit}.` : "", m.populationDescription ? `Population: ${para(m.populationDescription)}` : ""].filter(Boolean).join(" "));
|
|
663
|
+
if (m.reportingStatus && m.reportingStatus !== "POSTED") out.push(`Reporting status: ${words(m.reportingStatus)}.`);
|
|
664
|
+
const rows = [];
|
|
665
|
+
for (const c of list(m.classes)) for (const cat of list(c.categories)) rows.push([[c.title, cat.title].filter(Boolean).join(" \u2014 ") || m.title, ...groups.map((g) => {
|
|
666
|
+
const x = list(cat.measurements).find((v) => v.groupId === g.id);
|
|
667
|
+
return x ? `${x.value ?? ""}${x.spread !== void 0 ? ` \xB1 ${x.spread}` : ""}${x.lowerLimit !== void 0 ? ` (${x.lowerLimit} to ${x.upperLimit})` : ""}` : "";
|
|
668
|
+
})]);
|
|
669
|
+
const denoms = list(m.denoms).flatMap((dn) => list(dn.counts)).filter((c) => c.groupId);
|
|
670
|
+
if (denoms.length) rows.unshift(["Participants analysed", ...groups.map((g) => denoms.find((c) => c.groupId === g.id)?.value ?? "")]);
|
|
671
|
+
out.push(table2(["", ...groups.map((g) => g.title)], rows));
|
|
672
|
+
for (const a of list(m.analyses)) out.push(`Analysis${a.groupDescription ? ` (${para(a.groupDescription)})` : ""}: ${[a.statisticalMethod, a.pValue !== void 0 ? `p ${a.pValueComment ?? ""}${a.pValue}`.trim() : "", a.paramType ? `${words(a.paramType).toLowerCase()} ${a.paramValue ?? ""}` : "", a.ciPctValue ? `${a.ciPctValue}% CI ${a.ciLowerLimit ?? ""} to ${a.ciUpperLimit ?? ""}` : ""].filter(Boolean).join("; ")}`);
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
const ae = rs.adverseEventsModule;
|
|
676
|
+
if (ae) {
|
|
677
|
+
out.push("### Adverse events");
|
|
678
|
+
const groups = list(ae.eventGroups);
|
|
679
|
+
out.push([ae.timeFrame ? `Time frame: ${para(ae.timeFrame)}.` : "", ae.frequencyThreshold ? `Threshold for other events: ${ae.frequencyThreshold}%.` : "", ae.description ? para(ae.description) : ""].filter(Boolean).join(" "));
|
|
680
|
+
const summary = [
|
|
681
|
+
["Serious events, participants affected / at risk", ...groups.map((g) => `${g.seriousNumAffected ?? ""} / ${g.seriousNumAtRisk ?? ""}`)],
|
|
682
|
+
["Other events, participants affected / at risk", ...groups.map((g) => `${g.otherNumAffected ?? ""} / ${g.otherNumAtRisk ?? ""}`)],
|
|
683
|
+
["Deaths, participants affected / at risk", ...groups.map((g) => g.deathsNumAffected !== void 0 ? `${g.deathsNumAffected} / ${g.deathsNumAtRisk ?? ""}` : "")]
|
|
684
|
+
].filter((r) => r.slice(1).some(Boolean));
|
|
685
|
+
out.push(table2(["", ...groups.map((g) => g.title)], summary));
|
|
686
|
+
const events = (title, evs) => {
|
|
687
|
+
if (!evs.length) return;
|
|
688
|
+
out.push(`**${title}** (participants affected / at risk)`, table2(["Event", "System", ...groups.map((g) => g.title)], evs.map((e) => [e.term ?? "", e.organSystem ?? "", ...groups.map((g) => {
|
|
689
|
+
const s = list(e.stats).find((x) => x.groupId === g.id);
|
|
690
|
+
return s ? `${s.numAffected ?? ""} / ${s.numAtRisk ?? ""}` : "";
|
|
691
|
+
})])));
|
|
692
|
+
};
|
|
693
|
+
events("Serious adverse events", list(ae.seriousEvents));
|
|
694
|
+
events("Other adverse events", list(ae.otherEvents));
|
|
695
|
+
}
|
|
696
|
+
return out;
|
|
697
|
+
}
|
|
698
|
+
function headLineStudy(s) {
|
|
699
|
+
return [s.sponsor, s.nctid, s.status ? s.status.toLowerCase() : "", s.firstPosted ? `first posted ${s.firstPosted}` : "", s.lastUpdate ? `last update posted ${s.lastUpdate}` : "", CTGOV_PAGE(s.nctid)].filter(Boolean).join(" \xB7 ");
|
|
700
|
+
}
|
|
701
|
+
function renderStudyMd(s) {
|
|
702
|
+
const parts = [`# ${s.title}${s.acronym ? ` (${s.acronym})` : ""} \u2014 ClinicalTrials.gov record`, "", headLineStudy(s), "", ...s.sections.flatMap((b) => [b, ""])];
|
|
703
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
704
|
+
}
|
|
272
705
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
- library_status(id) \u2014 read-only: the entry's status, or level "missing".
|
|
276
|
-
- get_version() \u2014 package name, version and the resolved library directory.
|
|
706
|
+
// src/pdf.ts
|
|
707
|
+
import { getDocumentProxy } from "unpdf";
|
|
277
708
|
|
|
278
|
-
|
|
709
|
+
// src/glyph-names.ts
|
|
710
|
+
var PI_NAMES = {
|
|
711
|
+
H11001: "+",
|
|
712
|
+
H11002: "\u2212",
|
|
713
|
+
H11003: "\xD7",
|
|
714
|
+
H11005: "=",
|
|
715
|
+
H11006: "\xB1",
|
|
716
|
+
H11009: "\u221E",
|
|
717
|
+
H11011: "~",
|
|
718
|
+
H11015: "~",
|
|
719
|
+
H11021: "<",
|
|
720
|
+
H11022: ">",
|
|
721
|
+
H11032: "\u2032",
|
|
722
|
+
H11034: "\xB0",
|
|
723
|
+
H11349: "\u2264",
|
|
724
|
+
H11350: "\u2265",
|
|
725
|
+
H11545: "+",
|
|
726
|
+
H11549: "=",
|
|
727
|
+
H11554: "\u2022",
|
|
728
|
+
H11569: "*",
|
|
729
|
+
H17004: "\u25CF",
|
|
730
|
+
H17039: "\u25AA",
|
|
731
|
+
H20648: "\u2016",
|
|
732
|
+
H20841: "|",
|
|
733
|
+
H20851: "[",
|
|
734
|
+
H20852: "]",
|
|
735
|
+
H20862: "/",
|
|
736
|
+
H20873: "(",
|
|
737
|
+
H20874: ")",
|
|
738
|
+
H9004: "\u0394",
|
|
739
|
+
H9018: "\u03A3",
|
|
740
|
+
H9251: "\u03B1",
|
|
741
|
+
H9252: "\u03B2",
|
|
742
|
+
H9253: "\u03B3",
|
|
743
|
+
H9260: "\u03BA",
|
|
744
|
+
H9262: "\u03BC",
|
|
745
|
+
H9270: "\u03C4",
|
|
746
|
+
H9273: "\u03C7",
|
|
747
|
+
L1151: "+",
|
|
748
|
+
L1152: "\u2212",
|
|
749
|
+
L1154: "\xD7",
|
|
750
|
+
L50140: ">",
|
|
751
|
+
L50141: "<",
|
|
752
|
+
L50512: "\xB1",
|
|
753
|
+
L50871: "\u2265",
|
|
754
|
+
afii9797: "\u0394",
|
|
755
|
+
afii9825: "\u03B1",
|
|
756
|
+
afii9826: "\u03B2",
|
|
757
|
+
afii9839: "\u03BC",
|
|
758
|
+
Delta1: "\u0394",
|
|
759
|
+
"221e": "\u221E",
|
|
760
|
+
"223c": "~",
|
|
761
|
+
uni00b5: "\u03BC"
|
|
762
|
+
};
|
|
763
|
+
var FAMILY_NAMES = {
|
|
764
|
+
AdvP4C4E74: { C0: "\u2212", C1: ".", C2: "\xD7", C3: "*", C6: "\xB1", C15: "\u2022", C20: "\u2264", C21: "\u2265", C24: "~" },
|
|
765
|
+
AdvMT_SY: { C0: "\u2212", C2: "\xD7", C6: "\xB1" },
|
|
766
|
+
AdvMacMthSyN: { C0: "\u2212", C2: "\xD7", C6: "\xB1" }
|
|
767
|
+
};
|
|
768
|
+
var fontFamily = (name) => name.replace(/^[A-Z]{6}\+/, "").replace(/[+-].*$/, "");
|
|
769
|
+
function glyphChar(fontName, glyph) {
|
|
770
|
+
return PI_NAMES[glyph] ?? FAMILY_NAMES[fontFamily(fontName)]?.[glyph] ?? null;
|
|
771
|
+
}
|
|
772
|
+
function emittedNames(differences) {
|
|
773
|
+
const out = /* @__PURE__ */ new Map();
|
|
774
|
+
for (const [code, name] of differences) {
|
|
775
|
+
const m = /^C(\d{2,3})$/.exec(name);
|
|
776
|
+
const emitted = m ? Number(m[1]) : code;
|
|
777
|
+
if (emitted < 32) out.set(emitted, name);
|
|
778
|
+
}
|
|
779
|
+
return out;
|
|
780
|
+
}
|
|
279
781
|
|
|
280
|
-
|
|
281
|
-
var
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
782
|
+
// src/pdf.ts
|
|
783
|
+
var SECTION_NAMES = [
|
|
784
|
+
"abstract",
|
|
785
|
+
"summary",
|
|
786
|
+
"background",
|
|
787
|
+
"introduction",
|
|
788
|
+
"method",
|
|
789
|
+
"methods",
|
|
790
|
+
"materials and methods",
|
|
791
|
+
"material and methods",
|
|
792
|
+
"methods and materials",
|
|
793
|
+
"subjects and methods",
|
|
794
|
+
"patients and methods",
|
|
795
|
+
"participants and methods",
|
|
796
|
+
"experimental",
|
|
797
|
+
"experimental section",
|
|
798
|
+
"experimental procedures",
|
|
799
|
+
"study design",
|
|
800
|
+
"study population",
|
|
801
|
+
"participants",
|
|
802
|
+
"procedure",
|
|
803
|
+
"procedures",
|
|
804
|
+
"statistical analysis",
|
|
805
|
+
"result",
|
|
806
|
+
"results",
|
|
807
|
+
"results and discussion",
|
|
808
|
+
"findings",
|
|
809
|
+
"discussion",
|
|
810
|
+
"conclusion",
|
|
811
|
+
"conclusions",
|
|
812
|
+
"limitations",
|
|
813
|
+
"references",
|
|
814
|
+
"acknowledgments",
|
|
815
|
+
"acknowledgements",
|
|
816
|
+
"funding",
|
|
817
|
+
"conflict of interest",
|
|
818
|
+
"conflicts of interest",
|
|
819
|
+
"supplementary material",
|
|
820
|
+
"supplementary information"
|
|
821
|
+
];
|
|
822
|
+
var squeeze = (s) => s.toLowerCase().replace(/&/g, "and").replace(/[^a-z]/g, "");
|
|
823
|
+
var BY_SQUEEZED = new Map(SECTION_NAMES.map((n) => [squeeze(n), n.replace(/\b\w/g, (c) => c.toUpperCase())]));
|
|
824
|
+
function headingOf(line) {
|
|
825
|
+
let bare2 = line.trim().replace(/^\d+(\.\d+)*\.?\s*[|.]?\s*/, "").replace(/\s*:$/, "").replace(/([A-Za-z])\d{1,4}$/, "$1");
|
|
826
|
+
if (bare2.length > 60 || !/^[A-Za-z][A-Za-z &]*$/.test(bare2)) return null;
|
|
827
|
+
return BY_SQUEEZED.get(squeeze(bare2)) ?? null;
|
|
828
|
+
}
|
|
829
|
+
async function pdfToMarkdown(bytes, title, headLine2, stats) {
|
|
830
|
+
const pdf = await getDocumentProxy(new Uint8Array(bytes), { fontExtraProperties: true });
|
|
831
|
+
const text3 = await extractPages(pdf);
|
|
832
|
+
if (stats) {
|
|
833
|
+
stats.pages = pdf.numPages;
|
|
834
|
+
stats.chars = text3.replace(/\s+/g, " ").trim().length;
|
|
835
|
+
}
|
|
836
|
+
const lines = text3.split(/\r?\n/).map((l) => l.replace(/\s+/g, " ").trim());
|
|
837
|
+
const out = [`# ${title}`, "", headLine2, ""];
|
|
838
|
+
let para2 = [];
|
|
839
|
+
const flush = () => {
|
|
840
|
+
if (para2.length) {
|
|
841
|
+
out.push(para2.join(" "), "");
|
|
842
|
+
para2 = [];
|
|
304
843
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
844
|
+
};
|
|
845
|
+
for (const line of lines) {
|
|
846
|
+
if (!line) {
|
|
847
|
+
flush();
|
|
848
|
+
continue;
|
|
849
|
+
}
|
|
850
|
+
const h = headingOf(line);
|
|
851
|
+
if (h) {
|
|
852
|
+
flush();
|
|
853
|
+
out.push(`## ${h}`, "");
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
para2.push(line);
|
|
857
|
+
}
|
|
858
|
+
flush();
|
|
859
|
+
return fixGlyphs(out.join("\n")).replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
860
|
+
}
|
|
861
|
+
var CONTROL = /[\u0001-\u0008\u000b\u000c\u000e-\u001f]/;
|
|
862
|
+
async function extractPages(pdf) {
|
|
863
|
+
const pages = [];
|
|
864
|
+
for (let i = 1; i <= pdf.numPages; i++) {
|
|
865
|
+
const page = await pdf.getPage(i);
|
|
866
|
+
const fonts = /* @__PURE__ */ new Map();
|
|
867
|
+
const fontOf = (loaded2) => {
|
|
868
|
+
if (!fonts.has(loaded2)) {
|
|
869
|
+
let f;
|
|
870
|
+
try {
|
|
871
|
+
f = page.commonObjs.get(loaded2);
|
|
872
|
+
} catch {
|
|
873
|
+
f = void 0;
|
|
874
|
+
}
|
|
875
|
+
fonts.set(loaded2, f?.differences instanceof Map ? { name: f.name ?? "", emitted: emittedNames(f.differences) } : null);
|
|
876
|
+
}
|
|
877
|
+
return fonts.get(loaded2) ?? null;
|
|
878
|
+
};
|
|
879
|
+
let loaded = false, text3 = "";
|
|
880
|
+
for (const it of (await page.getTextContent()).items) {
|
|
881
|
+
if (it.str == null) continue;
|
|
882
|
+
let s = it.str;
|
|
883
|
+
if (CONTROL.test(s) && it.fontName) {
|
|
884
|
+
if (!loaded) {
|
|
885
|
+
try {
|
|
886
|
+
await page.getOperatorList();
|
|
887
|
+
} catch {
|
|
888
|
+
}
|
|
889
|
+
loaded = true;
|
|
890
|
+
}
|
|
891
|
+
const font = fontOf(it.fontName);
|
|
892
|
+
if (font) s = s.replace(new RegExp(CONTROL.source, "g"), (ch) => {
|
|
893
|
+
const name = font.emitted.get(ch.charCodeAt(0));
|
|
894
|
+
return (name && glyphChar(font.name, name)) ?? ch;
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
text3 += s;
|
|
898
|
+
if (it.hasEOL) {
|
|
899
|
+
let i2 = text3.length;
|
|
900
|
+
while (i2 > 0 && (text3[i2 - 1] === " " || text3[i2 - 1] === " ")) i2--;
|
|
901
|
+
if (i2 >= 2 && text3[i2 - 1] === "-" && /[A-Za-z]/.test(text3[i2 - 2])) text3 = text3.slice(0, i2 - 1) + BREAK + text3.slice(i2);
|
|
902
|
+
text3 += "\n";
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
pages.push(text3);
|
|
906
|
+
}
|
|
907
|
+
return dehyphenate(pages.join("\n").replace(/[^\S\n]+/g, " ").replace(/ ?\n ?/g, "\n").replace(/\n{3,}/g, "\n\n"));
|
|
908
|
+
}
|
|
909
|
+
var BREAK = "\0";
|
|
910
|
+
var SUSPENDED = /* @__PURE__ */ new Set(["and", "or", "to", "vs", "but", "nor"]);
|
|
911
|
+
var COMPOUND_PREFIX = /* @__PURE__ */ new Set([
|
|
912
|
+
"anti",
|
|
913
|
+
"non",
|
|
914
|
+
"pre",
|
|
915
|
+
"post",
|
|
916
|
+
"self",
|
|
917
|
+
"meta",
|
|
918
|
+
"cross",
|
|
919
|
+
"long",
|
|
920
|
+
"short",
|
|
921
|
+
"high",
|
|
922
|
+
"low",
|
|
923
|
+
"well",
|
|
924
|
+
"placebo",
|
|
925
|
+
"double",
|
|
926
|
+
"single",
|
|
927
|
+
"randomized",
|
|
928
|
+
"randomised",
|
|
929
|
+
"multi",
|
|
930
|
+
"co",
|
|
931
|
+
"inter",
|
|
932
|
+
"intra",
|
|
933
|
+
"sub",
|
|
934
|
+
"semi",
|
|
935
|
+
"dose",
|
|
936
|
+
"time",
|
|
937
|
+
"age",
|
|
938
|
+
"sex",
|
|
939
|
+
"over",
|
|
940
|
+
"under",
|
|
941
|
+
"half",
|
|
942
|
+
"full",
|
|
943
|
+
"all",
|
|
944
|
+
"one",
|
|
945
|
+
"two",
|
|
946
|
+
"three",
|
|
947
|
+
"first",
|
|
948
|
+
"second",
|
|
949
|
+
"third",
|
|
950
|
+
"end",
|
|
951
|
+
"mid",
|
|
952
|
+
"late",
|
|
953
|
+
"early",
|
|
954
|
+
"whole",
|
|
955
|
+
"small",
|
|
956
|
+
"large",
|
|
957
|
+
"vitamin",
|
|
958
|
+
"omega",
|
|
959
|
+
"alpha",
|
|
960
|
+
"beta",
|
|
961
|
+
"gamma",
|
|
962
|
+
"type",
|
|
963
|
+
"wild",
|
|
964
|
+
"ex",
|
|
965
|
+
"quasi",
|
|
966
|
+
"pseudo",
|
|
967
|
+
"peri",
|
|
968
|
+
"re"
|
|
969
|
+
]);
|
|
970
|
+
function dehyphenate(text3) {
|
|
971
|
+
if (!text3.includes(BREAK)) return text3;
|
|
972
|
+
const lower = text3.toLowerCase();
|
|
973
|
+
const seen = (s) => new RegExp(`(?<![a-z])${s}(?![a-z])`).test(lower);
|
|
974
|
+
return text3.replace(/([A-Za-z]+)\u0000\n([A-Za-z]+)/g, (m, x, y) => {
|
|
975
|
+
if (!/^[a-z]/.test(y)) return `${x}-${y}`;
|
|
976
|
+
if (SUSPENDED.has(y.toLowerCase())) return `${x}- ${y}`;
|
|
977
|
+
const hyph = `${x}-${y}`.toLowerCase(), joined = `${x}${y}`.toLowerCase();
|
|
978
|
+
if (seen(hyph)) return `${x}-${y}`;
|
|
979
|
+
if (seen(joined)) return `${x}${y}`;
|
|
980
|
+
return COMPOUND_PREFIX.has(x.toLowerCase()) ? `${x}-${y}` : `${x}${y}`;
|
|
981
|
+
}).replace(/\u0000\n/g, "-\n").replace(/\u0000/g, "-");
|
|
982
|
+
}
|
|
983
|
+
function fixGlyphs(md) {
|
|
984
|
+
return md.replace(/¡(?=\.?\d)/g, "\u2212").replace(/(?<=[\p{L}\p{N}])\u0008(?=[\p{L}\p{N}])/gu, " ");
|
|
985
|
+
}
|
|
986
|
+
var UNRESOLVED_GLYPH = /[\u0001-\u0008\u000b\u000c\u000e-\u001f]/g;
|
|
987
|
+
var countUnresolved = (text3) => (text3.match(UNRESOLVED_GLYPH) ?? []).length;
|
|
988
|
+
var looksLikePdf = (bytes) => bytes.subarray(0, 5).toString("latin1") === "%PDF-";
|
|
989
|
+
|
|
990
|
+
// src/entry.ts
|
|
991
|
+
function emptyStatus(requested, type, id, libraryDir) {
|
|
992
|
+
return {
|
|
993
|
+
requested,
|
|
994
|
+
type,
|
|
995
|
+
id,
|
|
996
|
+
entry: null,
|
|
997
|
+
entry_dir: null,
|
|
998
|
+
library_dir: libraryDir,
|
|
999
|
+
pmid: null,
|
|
1000
|
+
setid: null,
|
|
1001
|
+
nctid: null,
|
|
1002
|
+
publications: [],
|
|
1003
|
+
pmcid: null,
|
|
1004
|
+
doi: null,
|
|
1005
|
+
title: null,
|
|
1006
|
+
first_author: null,
|
|
1007
|
+
year: null,
|
|
1008
|
+
publication_types: [],
|
|
1009
|
+
retracted: false,
|
|
1010
|
+
corrections: [],
|
|
1011
|
+
coi: null,
|
|
1012
|
+
funding: [],
|
|
1013
|
+
citable: true,
|
|
1014
|
+
comment: null,
|
|
1015
|
+
has_abstract: null,
|
|
1016
|
+
level: null,
|
|
1017
|
+
version: null,
|
|
1018
|
+
retrieved_by: null,
|
|
1019
|
+
retrieved_date: null,
|
|
1020
|
+
pubmed_checked: null,
|
|
1021
|
+
attempts: [],
|
|
1022
|
+
files: [],
|
|
1023
|
+
glyphs_unresolved: null
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
var today = () => (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1027
|
+
function headLine(r) {
|
|
1028
|
+
return [
|
|
1029
|
+
r.authors.join(", ") || null,
|
|
1030
|
+
r.journal,
|
|
1031
|
+
r.year,
|
|
1032
|
+
r.pmid ? `PMID ${r.pmid}` : "not in PubMed",
|
|
1033
|
+
r.doi ? `DOI ${r.doi}` : null,
|
|
1034
|
+
r.pmcid ? `PMCID ${r.pmcid}` : null
|
|
1035
|
+
].filter(Boolean).join(" \xB7 ");
|
|
1036
|
+
}
|
|
1037
|
+
function renderAbstractMd(r) {
|
|
1038
|
+
const body = r.abstract.length ? r.abstract.map((a) => a.label ? `**${a.label}.** ${a.text}` : a.text).join("\n\n") : "_No abstract available._";
|
|
1039
|
+
return `# ${r.title}
|
|
1040
|
+
|
|
1041
|
+
${headLine(r)}
|
|
1042
|
+
|
|
1043
|
+
## Abstract
|
|
1044
|
+
|
|
1045
|
+
${body}
|
|
1046
|
+
`;
|
|
1047
|
+
}
|
|
1048
|
+
function readHeadLine(p) {
|
|
1049
|
+
try {
|
|
1050
|
+
const lines = readFileSync3(p.md, "utf8").split("\n").filter((l) => l.trim());
|
|
1051
|
+
return lines[1] ?? "";
|
|
1052
|
+
} catch {
|
|
1053
|
+
return "";
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
function metaFromRecord(r, files, converter) {
|
|
1057
|
+
return {
|
|
1058
|
+
pmid: r.pmid,
|
|
1059
|
+
setid: null,
|
|
1060
|
+
nctid: null,
|
|
1061
|
+
publications: [],
|
|
1062
|
+
pmcid: r.pmcid,
|
|
1063
|
+
doi: r.doi,
|
|
1064
|
+
title: r.title,
|
|
1065
|
+
first_author: r.firstAuthor,
|
|
1066
|
+
year: r.year,
|
|
1067
|
+
publication_types: r.publicationTypes,
|
|
1068
|
+
retracted: isRetracted(r),
|
|
1069
|
+
corrections: r.corrections,
|
|
1070
|
+
coi: r.coi,
|
|
1071
|
+
funding: r.funding,
|
|
1072
|
+
has_abstract: r.abstract.length > 0,
|
|
1073
|
+
level: "abstract",
|
|
1074
|
+
version: null,
|
|
1075
|
+
files,
|
|
1076
|
+
retrieved_by: "pubmed",
|
|
1077
|
+
retrieved_date: today(),
|
|
1078
|
+
converted_by: converter,
|
|
1079
|
+
pubmed_checked: today(),
|
|
1080
|
+
attempts: []
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
var isRetracted = (r) => r.publicationTypes.includes("Retracted Publication") || r.corrections.some(isRetractionNotice);
|
|
1084
|
+
var renderCorrection = (c) => `${c.type} ${c.pmid ?? "-"} ${JSON.stringify(c.source)}`;
|
|
1085
|
+
function parseCorrection(v) {
|
|
1086
|
+
const m = /^(\S+) (\S+) ("[\s\S]*")$/.exec(v.trim());
|
|
1087
|
+
if (!m) return null;
|
|
1088
|
+
let source;
|
|
1089
|
+
try {
|
|
1090
|
+
source = JSON.parse(m[3]);
|
|
1091
|
+
} catch {
|
|
1092
|
+
return null;
|
|
1093
|
+
}
|
|
1094
|
+
return { type: m[1], pmid: m[2] === "-" ? null : m[2], source };
|
|
1095
|
+
}
|
|
1096
|
+
function renderMeta(m) {
|
|
1097
|
+
const lines = [
|
|
1098
|
+
["pmid", m.pmid],
|
|
1099
|
+
...m.setid ? [["setid", m.setid]] : [],
|
|
1100
|
+
...m.nctid ? [["nctid", m.nctid]] : [],
|
|
1101
|
+
...m.nctid ? [["publications", m.publications.join(", ")]] : [],
|
|
1102
|
+
["pmcid", m.pmcid ?? ""],
|
|
1103
|
+
["doi", m.doi ?? ""],
|
|
1104
|
+
["title", JSON.stringify(m.title)],
|
|
1105
|
+
["first_author", m.first_author ?? ""],
|
|
1106
|
+
["year", m.year ?? ""],
|
|
1107
|
+
["publication_types", m.publication_types.join("; ")],
|
|
1108
|
+
["retracted", m.retracted ? "yes" : "no"],
|
|
1109
|
+
...m.corrections.map((c) => ["corrections", renderCorrection(c)]),
|
|
1110
|
+
["coi", m.coi === null ? "" : JSON.stringify(m.coi)],
|
|
1111
|
+
["funding", m.funding.length ? JSON.stringify(m.funding) : ""],
|
|
1112
|
+
// a JSON array: a statement may hold semicolons
|
|
1113
|
+
["has_abstract", m.has_abstract ? "yes" : "no"],
|
|
1114
|
+
["level", m.level],
|
|
1115
|
+
["version", m.version ?? ""],
|
|
1116
|
+
["files", m.files.join(", ")],
|
|
1117
|
+
["retrieved_by", m.retrieved_by],
|
|
1118
|
+
["retrieved_date", m.retrieved_date],
|
|
1119
|
+
["converted_by", m.converted_by],
|
|
1120
|
+
["pubmed_checked", m.pubmed_checked],
|
|
1121
|
+
["attempts", m.attempts.join(", ")]
|
|
1122
|
+
];
|
|
1123
|
+
return "---\n" + lines.map(([k, v]) => v === "" ? `${k}:` : `${k}: ${v}`).join("\n") + "\n---\n";
|
|
1124
|
+
}
|
|
1125
|
+
function parseFrontmatter(md) {
|
|
1126
|
+
const m = /^---\n([\s\S]*?)\n---/.exec(md);
|
|
1127
|
+
const fm = {}, all = {};
|
|
1128
|
+
for (const line of (m?.[1] ?? "").split("\n")) {
|
|
1129
|
+
const i = line.indexOf(":");
|
|
1130
|
+
if (i <= 0) continue;
|
|
1131
|
+
const k = line.slice(0, i).trim(), v = line.slice(i + 1).trim();
|
|
1132
|
+
fm[k] = v;
|
|
1133
|
+
(all[k] ??= []).push(v);
|
|
1134
|
+
}
|
|
1135
|
+
return { fm, all };
|
|
1136
|
+
}
|
|
1137
|
+
var list2 = (v) => v ? v.split(",").map((s) => s.trim()).filter(Boolean) : [];
|
|
1138
|
+
function splitAttempts(v) {
|
|
1139
|
+
if (!v) return [];
|
|
1140
|
+
const out = [];
|
|
1141
|
+
let cur = "", depth = 0;
|
|
1142
|
+
for (const ch of v) {
|
|
1143
|
+
if (ch === "(") depth++;
|
|
1144
|
+
else if (ch === ")") depth = Math.max(0, depth - 1);
|
|
1145
|
+
if (ch === "," && depth === 0) {
|
|
1146
|
+
out.push(cur);
|
|
1147
|
+
cur = "";
|
|
1148
|
+
} else cur += ch;
|
|
1149
|
+
}
|
|
1150
|
+
out.push(cur);
|
|
1151
|
+
const merged = [];
|
|
1152
|
+
for (const piece of out.map((s) => s.trim()).filter(Boolean)) {
|
|
1153
|
+
if (merged.length && !/^\S+ \(/.test(piece)) merged[merged.length - 1] += ", " + piece;
|
|
1154
|
+
else merged.push(piece);
|
|
1155
|
+
}
|
|
1156
|
+
return merged;
|
|
1157
|
+
}
|
|
1158
|
+
var opt = (v) => v ? v : null;
|
|
1159
|
+
var TYPE_CONTINUATION = /* @__PURE__ */ new Set(["N.I.H.", "Extramural", "Intramural", "Non-U.S. Gov't", "U.S. Gov't", "U.S. Government", "P.H.S.", "Non-P.H.S.", "Phase I", "Phase II", "Phase III", "Phase IV", "Veterinary", "NIH", "American Recovery and Reinvestment Act"]);
|
|
1160
|
+
function splitPublicationTypes(v) {
|
|
1161
|
+
if (!v) return [];
|
|
1162
|
+
if (v.includes(";")) return v.split(";").map((s) => s.trim()).filter(Boolean);
|
|
1163
|
+
const out = [];
|
|
1164
|
+
for (const piece of list2(v)) {
|
|
1165
|
+
if (out.length && TYPE_CONTINUATION.has(piece)) out[out.length - 1] += ", " + piece;
|
|
1166
|
+
else out.push(piece);
|
|
1167
|
+
}
|
|
1168
|
+
return out;
|
|
1169
|
+
}
|
|
1170
|
+
function abstractMissing(p) {
|
|
1171
|
+
try {
|
|
1172
|
+
return readFileSync3(p.md, "utf8").includes("_No abstract available._");
|
|
1173
|
+
} catch {
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
function correctionsFromXml(p) {
|
|
1178
|
+
try {
|
|
1179
|
+
return parsePubmedXml(readFileSync3(p.xml, "utf8"))?.corrections ?? [];
|
|
1180
|
+
} catch {
|
|
1181
|
+
return [];
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
function declarationsFromXml(p) {
|
|
1185
|
+
try {
|
|
1186
|
+
const r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
1187
|
+
return { coi: r?.coi ?? null, funding: r?.funding ?? [] };
|
|
1188
|
+
} catch {
|
|
1189
|
+
return { coi: null, funding: [] };
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
function readMeta(p) {
|
|
1193
|
+
const { fm, all } = parseFrontmatter(readFileSync3(p.meta, "utf8"));
|
|
1194
|
+
let title = fm.title ?? "";
|
|
1195
|
+
if (title.startsWith('"')) {
|
|
1196
|
+
try {
|
|
1197
|
+
title = JSON.parse(title);
|
|
1198
|
+
} catch {
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
const legacy = fm.pubmed_checked === void 0;
|
|
1202
|
+
const corrections = legacy ? correctionsFromXml(p) : (all.corrections ?? []).map(parseCorrection).filter((c) => c !== null);
|
|
1203
|
+
let coi = null, funding = [];
|
|
1204
|
+
if (fm.coi === void 0) ({ coi, funding } = declarationsFromXml(p));
|
|
1205
|
+
else {
|
|
1206
|
+
if (fm.coi.startsWith('"')) {
|
|
1207
|
+
try {
|
|
1208
|
+
coi = JSON.parse(fm.coi);
|
|
1209
|
+
} catch {
|
|
1210
|
+
coi = fm.coi;
|
|
1211
|
+
}
|
|
1212
|
+
} else coi = fm.coi || null;
|
|
1213
|
+
funding = !fm.funding ? [] : fm.funding.startsWith("[") ? JSON.parse(fm.funding) : fm.funding.split(";").map((s) => s.trim()).filter(Boolean);
|
|
1214
|
+
}
|
|
1215
|
+
return {
|
|
1216
|
+
pmid: fm.pmid ?? "",
|
|
1217
|
+
setid: opt(fm.setid),
|
|
1218
|
+
nctid: opt(fm.nctid),
|
|
1219
|
+
publications: list2(fm.publications),
|
|
1220
|
+
pmcid: opt(fm.pmcid),
|
|
1221
|
+
doi: opt(fm.doi),
|
|
1222
|
+
title,
|
|
1223
|
+
first_author: opt(fm.first_author),
|
|
1224
|
+
year: opt(fm.year),
|
|
1225
|
+
publication_types: splitPublicationTypes(fm.publication_types),
|
|
1226
|
+
retracted: fm.retracted === "yes" || corrections.some(isRetractionNotice),
|
|
1227
|
+
corrections,
|
|
1228
|
+
coi,
|
|
1229
|
+
funding,
|
|
1230
|
+
has_abstract: fm.has_abstract ? fm.has_abstract === "yes" : !abstractMissing(p),
|
|
1231
|
+
level: fm.level === "full_text" ? "full_text" : "abstract",
|
|
1232
|
+
version: opt(fm.version),
|
|
1233
|
+
files: list2(fm.files),
|
|
1234
|
+
retrieved_by: fm.retrieved_by ?? "",
|
|
1235
|
+
retrieved_date: fm.retrieved_date ?? "",
|
|
1236
|
+
converted_by: fm.converted_by ?? "",
|
|
1237
|
+
pubmed_checked: legacy ? fm.retrieved_date ?? "" : fm.pubmed_checked,
|
|
1238
|
+
attempts: splitAttempts(fm.attempts)
|
|
1239
|
+
};
|
|
1240
|
+
}
|
|
1241
|
+
function writeAtomic(file, data) {
|
|
1242
|
+
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
1243
|
+
writeFileSync(tmp, data);
|
|
1244
|
+
renameSync(tmp, file);
|
|
1245
|
+
}
|
|
1246
|
+
function writeMeta(p, m) {
|
|
1247
|
+
writeAtomic(p.meta, renderMeta(m));
|
|
1248
|
+
}
|
|
1249
|
+
function copyEntry(from, to) {
|
|
1250
|
+
const tmp = `${to.entryDir}.${process.pid}.tmp`, old = `${to.entryDir}.${process.pid}.old`;
|
|
1251
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
1252
|
+
mkdirSync(tmp, { recursive: true });
|
|
1253
|
+
const rename = (n) => n.startsWith(from.entry) ? to.entry + n.slice(from.entry.length) : n;
|
|
1254
|
+
for (const n of readdirSync(from.entryDir)) if (join4(from.entryDir, n) !== from.meta && !n.endsWith(".tmp")) copyFileSync(join4(from.entryDir, n), join4(tmp, rename(n)));
|
|
1255
|
+
const m = readMeta(from);
|
|
1256
|
+
const copy = { ...m, files: m.files.map(rename) };
|
|
1257
|
+
writeFileSync(join4(tmp, basename(to.meta)), renderMeta(copy));
|
|
1258
|
+
mkdirSync(dirname(to.entryDir), { recursive: true });
|
|
1259
|
+
if (existsSync4(to.entryDir)) renameSync(to.entryDir, old);
|
|
1260
|
+
renameSync(tmp, to.entryDir);
|
|
1261
|
+
rmSync(old, { recursive: true, force: true });
|
|
1262
|
+
return copy;
|
|
1263
|
+
}
|
|
1264
|
+
function citability(m) {
|
|
1265
|
+
const of = (t) => m.corrections.filter((c) => c.type === t);
|
|
1266
|
+
const cite = (c) => `${c.source}${c.pmid ? ` (pmid:${c.pmid})` : ""}`;
|
|
1267
|
+
if (m.retracted) {
|
|
1268
|
+
const r = m.corrections.filter(isRetractionNotice);
|
|
1269
|
+
return { citable: false, comment: r.length ? `retracted: ${r.map(cite).join("; ")}` : "retracted (PubMed publication type)" };
|
|
1270
|
+
}
|
|
1271
|
+
const notes = [
|
|
1272
|
+
...of("erratum_in").map((c) => `erratum, not assessed: ${cite(c)}`),
|
|
1273
|
+
...of("expression_of_concern_in").map((c) => `expression of concern: ${cite(c)}`),
|
|
1274
|
+
...of("update_in").map((c) => `superseded by a newer version: ${cite(c)}`)
|
|
1275
|
+
];
|
|
1276
|
+
return { citable: true, comment: notes.length ? notes.join("; ") : null };
|
|
1277
|
+
}
|
|
1278
|
+
function statusFromMeta(p, m, base) {
|
|
1279
|
+
const full = existsSync4(p.fulltextMd) ? readFileSync3(p.fulltextMd, "utf8") : null;
|
|
1280
|
+
const cit = citability(m);
|
|
1281
|
+
if (full?.includes(`
|
|
1282
|
+
${CORRECTED_HEADING}
|
|
1283
|
+
`) && cit.comment) cit.comment = cit.comment.replace(/erratum, not assessed: /g, "erratum, corrected figures stored (Corrected in PubMed abstract): ");
|
|
1284
|
+
return {
|
|
1285
|
+
...base,
|
|
1286
|
+
entry: p.entry,
|
|
1287
|
+
entry_dir: p.entryDir,
|
|
1288
|
+
pmid: m.pmid || null,
|
|
1289
|
+
setid: m.setid,
|
|
1290
|
+
nctid: m.nctid,
|
|
1291
|
+
publications: m.publications,
|
|
1292
|
+
pmcid: m.pmcid,
|
|
1293
|
+
doi: m.doi,
|
|
1294
|
+
title: m.title || null,
|
|
1295
|
+
first_author: m.first_author,
|
|
1296
|
+
year: m.year,
|
|
1297
|
+
publication_types: m.publication_types,
|
|
1298
|
+
retracted: m.retracted,
|
|
1299
|
+
corrections: m.corrections,
|
|
1300
|
+
coi: m.coi,
|
|
1301
|
+
funding: m.funding,
|
|
1302
|
+
...cit,
|
|
1303
|
+
// What the entry holds: full_text, abstract, or only the title (the meta's
|
|
1304
|
+
// level is "abstract" until a full text arrives; has_abstract says whether
|
|
1305
|
+
// there is abstract text).
|
|
1306
|
+
has_abstract: m.has_abstract,
|
|
1307
|
+
level: m.level === "abstract" && !m.has_abstract ? "title" : m.level,
|
|
1308
|
+
version: m.version,
|
|
1309
|
+
retrieved_by: m.retrieved_by || null,
|
|
1310
|
+
retrieved_date: m.retrieved_date || null,
|
|
1311
|
+
pubmed_checked: m.pubmed_checked || null,
|
|
1312
|
+
attempts: m.attempts,
|
|
1313
|
+
files: m.files,
|
|
1314
|
+
glyphs_unresolved: full === null ? null : countUnresolved(full)
|
|
1315
|
+
};
|
|
1316
|
+
}
|
|
1317
|
+
function readStatus(p, base) {
|
|
1318
|
+
return statusFromMeta(p, readMeta(p), base);
|
|
1319
|
+
}
|
|
1320
|
+
function entryExists(p) {
|
|
1321
|
+
return existsSync4(p.meta);
|
|
1322
|
+
}
|
|
1323
|
+
function writeEntry(p, r, xml, converter) {
|
|
1324
|
+
mkdirSync(p.entryDir, { recursive: true });
|
|
1325
|
+
writeAtomic(p.xml, xml);
|
|
1326
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
1327
|
+
const meta = metaFromRecord(r, [basename(p.xml), basename(p.md)], converter);
|
|
1328
|
+
writeMeta(p, meta);
|
|
1329
|
+
return meta;
|
|
1330
|
+
}
|
|
1331
|
+
function refreshRecord(p, m, r, xml) {
|
|
1332
|
+
writeAtomic(p.xml, xml);
|
|
1333
|
+
syncCorrectedBlock(p);
|
|
1334
|
+
const next = {
|
|
1335
|
+
...m,
|
|
1336
|
+
pmcid: r.pmcid ?? m.pmcid,
|
|
1337
|
+
doi: r.doi ?? m.doi,
|
|
1338
|
+
publication_types: r.publicationTypes,
|
|
1339
|
+
retracted: isRetracted(r),
|
|
1340
|
+
corrections: r.corrections,
|
|
1341
|
+
coi: r.coi ?? m.coi,
|
|
1342
|
+
funding: r.funding.length ? r.funding : m.funding,
|
|
1343
|
+
// the paper's own statements stay when PubMed has none
|
|
1344
|
+
pubmed_checked: today()
|
|
1345
|
+
};
|
|
1346
|
+
writeMeta(p, next);
|
|
1347
|
+
return next;
|
|
1348
|
+
}
|
|
1349
|
+
function daysSince(date, now = /* @__PURE__ */ new Date()) {
|
|
1350
|
+
const t = new Date(date).getTime();
|
|
1351
|
+
return Number.isFinite(t) ? (now.getTime() - t) / 864e5 : Infinity;
|
|
1352
|
+
}
|
|
1353
|
+
var FUNDING_RE = /^[*_\s]*(funding|financial support|sources? of (financial )?support|support(ed by)?|grant support|this (work|study|research) was (supported|funded))\b[^:]*:?[*_\s]*(.*)$/i;
|
|
1354
|
+
var COI_RE = /^[*_\s]*(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?|conflict[- ]of[- ]interest statement)\b[^:]*:?[*_\s]*(.*)$/i;
|
|
1355
|
+
function declarationsFromMd(md) {
|
|
1356
|
+
let coi = null;
|
|
1357
|
+
const funding = [];
|
|
1358
|
+
const lines = md.split("\n");
|
|
1359
|
+
let section2 = "";
|
|
1360
|
+
for (let i = 0; i < lines.length; i++) {
|
|
1361
|
+
const l = lines[i].trim();
|
|
1362
|
+
const h = /^#{2,6}\s+(.*)$/.exec(l);
|
|
1363
|
+
if (h) {
|
|
1364
|
+
section2 = h[1];
|
|
1365
|
+
const body = lines.slice(i + 1).find((x) => x.trim() && !/^#/.test(x))?.trim() ?? "";
|
|
1366
|
+
if (body && /^(funding|financial support|sources? of funding|role of the funding source)/i.test(section2) && !funding.length) funding.push(body);
|
|
1367
|
+
if (body && /^(conflicts? of interest|competing interests?|disclosures?|financial disclosures?|declarations? of (competing |conflicting )?interests?)/i.test(section2) && coi === null) coi = body;
|
|
1368
|
+
continue;
|
|
1369
|
+
}
|
|
1370
|
+
if (section2 !== "Declarations" || !l) continue;
|
|
1371
|
+
const f = FUNDING_RE.exec(l);
|
|
1372
|
+
const c = COI_RE.exec(l);
|
|
1373
|
+
if (c && c[3]?.trim() && coi === null) coi = c[3].trim().replace(/[*_]+$/, "");
|
|
1374
|
+
else if (f && f[6]?.trim() && !funding.length) funding.push(f[6].trim().replace(/[*_]+$/, ""));
|
|
1375
|
+
}
|
|
1376
|
+
return { coi, funding };
|
|
1377
|
+
}
|
|
1378
|
+
var CORRECTED_HEADING = "## Corrected in PubMed abstract";
|
|
1379
|
+
function correctedSentences(r) {
|
|
1380
|
+
const out = [];
|
|
1381
|
+
for (const a of r.abstract) {
|
|
1382
|
+
for (const s of a.text.split(/(?<=[.!?\]])\s+(?=[A-Z(\u2212-])/)) {
|
|
1383
|
+
const i = s.indexOf("[corrected]");
|
|
1384
|
+
if (i >= 0) out.push(s.slice(0, i).trim().replace(/\s+([.,;])$/, "$1"));
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
return out.filter(Boolean);
|
|
1388
|
+
}
|
|
1389
|
+
function withCorrectedBlock(md, sentences) {
|
|
1390
|
+
const base = md.split(`
|
|
1391
|
+
${CORRECTED_HEADING}
|
|
1392
|
+
`)[0].trimEnd();
|
|
1393
|
+
if (!sentences.length) return base + "\n";
|
|
1394
|
+
return `${base}
|
|
1395
|
+
|
|
1396
|
+
${CORRECTED_HEADING}
|
|
1397
|
+
|
|
1398
|
+
NLM corrected these sentences of the abstract after an erratum; the text above may predate the correction.
|
|
1399
|
+
|
|
1400
|
+
${sentences.join("\n\n")}
|
|
1401
|
+
`;
|
|
1402
|
+
}
|
|
1403
|
+
function syncCorrectedBlock(p) {
|
|
1404
|
+
if (!existsSync4(p.fulltextMd) || !existsSync4(p.xml)) return;
|
|
1405
|
+
let r;
|
|
1406
|
+
try {
|
|
1407
|
+
r = parsePubmedXml(readFileSync3(p.xml, "utf8"));
|
|
1408
|
+
} catch {
|
|
1409
|
+
return;
|
|
1410
|
+
}
|
|
1411
|
+
if (!r) return;
|
|
1412
|
+
const md = readFileSync3(p.fulltextMd, "utf8");
|
|
1413
|
+
const next = withCorrectedBlock(md, correctedSentences(r));
|
|
1414
|
+
if (next !== md) writeAtomic(p.fulltextMd, next);
|
|
1415
|
+
}
|
|
1416
|
+
function applyFulltext(p, m, hit, converter) {
|
|
1417
|
+
const raw = p.fulltextRaw(hit.format);
|
|
1418
|
+
writeAtomic(raw, hit.raw);
|
|
1419
|
+
writeAtomic(p.fulltextMd, hit.md);
|
|
1420
|
+
syncCorrectedBlock(p);
|
|
1421
|
+
const files = [...m.files.filter((f) => !f.includes("-fulltext.")), basename(raw), basename(p.fulltextMd)];
|
|
1422
|
+
const own = declarationsFromMd(hit.md);
|
|
1423
|
+
const next = {
|
|
1424
|
+
...m,
|
|
1425
|
+
level: "full_text",
|
|
1426
|
+
version: hit.version,
|
|
1427
|
+
files,
|
|
1428
|
+
retrieved_by: hit.tier,
|
|
1429
|
+
retrieved_date: today(),
|
|
1430
|
+
converted_by: converter,
|
|
1431
|
+
coi: m.coi ?? own.coi,
|
|
1432
|
+
funding: m.funding.length ? m.funding : own.funding
|
|
1433
|
+
};
|
|
1434
|
+
writeMeta(p, next);
|
|
1435
|
+
return next;
|
|
1436
|
+
}
|
|
1437
|
+
function recordAttempts(p, m, labels) {
|
|
1438
|
+
if (!labels.length) return m;
|
|
1439
|
+
const tierOf = (a) => /^(\S+) \(/.exec(a.trim())?.[1] ?? a;
|
|
1440
|
+
const replaced = new Set(labels.map(tierOf));
|
|
1441
|
+
const next = { ...m, attempts: [...m.attempts.filter((a) => !replaced.has(tierOf(a))), ...labels] };
|
|
1442
|
+
writeMeta(p, next);
|
|
1443
|
+
return next;
|
|
1444
|
+
}
|
|
1445
|
+
function writeLabelEntry(p, s, xml, converter) {
|
|
1446
|
+
mkdirSync(p.entryDir, { recursive: true });
|
|
1447
|
+
const raw = p.fulltextRaw("xml");
|
|
1448
|
+
writeAtomic(raw, xml);
|
|
1449
|
+
writeAtomic(p.fulltextMd, renderLabelMd(s));
|
|
1450
|
+
const meta = {
|
|
1451
|
+
pmid: "",
|
|
1452
|
+
setid: s.setid,
|
|
1453
|
+
nctid: null,
|
|
1454
|
+
publications: [],
|
|
1455
|
+
pmcid: null,
|
|
1456
|
+
doi: null,
|
|
1457
|
+
title: s.title,
|
|
1458
|
+
first_author: s.labeler,
|
|
1459
|
+
year: s.effective?.slice(0, 4) ?? null,
|
|
1460
|
+
publication_types: ["Drug Label"],
|
|
1461
|
+
retracted: false,
|
|
1462
|
+
corrections: [],
|
|
1463
|
+
coi: null,
|
|
1464
|
+
funding: [],
|
|
1465
|
+
has_abstract: false,
|
|
1466
|
+
level: "full_text",
|
|
1467
|
+
version: s.version ? `spl ${s.version}` : null,
|
|
1468
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
1469
|
+
retrieved_by: "dailymed",
|
|
1470
|
+
retrieved_date: today(),
|
|
1471
|
+
converted_by: converter,
|
|
1472
|
+
pubmed_checked: today(),
|
|
1473
|
+
attempts: []
|
|
1474
|
+
};
|
|
1475
|
+
writeMeta(p, meta);
|
|
1476
|
+
return meta;
|
|
1477
|
+
}
|
|
1478
|
+
function writeStudyEntry(p, s, json2, converter) {
|
|
1479
|
+
mkdirSync(p.entryDir, { recursive: true });
|
|
1480
|
+
const raw = p.fulltextRaw("json");
|
|
1481
|
+
writeAtomic(raw, json2);
|
|
1482
|
+
writeAtomic(p.fulltextMd, renderStudyMd(s));
|
|
1483
|
+
const meta = {
|
|
1484
|
+
pmid: "",
|
|
1485
|
+
setid: null,
|
|
1486
|
+
nctid: s.nctid,
|
|
1487
|
+
publications: s.publications,
|
|
1488
|
+
pmcid: null,
|
|
1489
|
+
doi: null,
|
|
1490
|
+
title: s.title,
|
|
1491
|
+
first_author: s.sponsor,
|
|
1492
|
+
year: s.firstPosted?.slice(0, 4) ?? null,
|
|
1493
|
+
publication_types: ["Trial Registration"],
|
|
1494
|
+
retracted: false,
|
|
1495
|
+
corrections: [],
|
|
1496
|
+
coi: null,
|
|
1497
|
+
funding: [],
|
|
1498
|
+
has_abstract: false,
|
|
1499
|
+
level: "full_text",
|
|
1500
|
+
version: s.lastUpdate,
|
|
1501
|
+
files: [basename(raw), basename(p.fulltextMd)],
|
|
1502
|
+
retrieved_by: "ctgov",
|
|
1503
|
+
retrieved_date: today(),
|
|
1504
|
+
converted_by: converter,
|
|
1505
|
+
pubmed_checked: today(),
|
|
1506
|
+
attempts: []
|
|
1507
|
+
};
|
|
1508
|
+
writeMeta(p, meta);
|
|
1509
|
+
return meta;
|
|
1510
|
+
}
|
|
1511
|
+
function writeDoiEntry(p, r, json2, converter, abstractFrom = "crossref", recordFrom = "crossref") {
|
|
1512
|
+
mkdirSync(p.entryDir, { recursive: true });
|
|
1513
|
+
const raw = p.fulltextRaw("json").replace(/-fulltext\.json$/, recordFrom === "crossref" ? "-crossref.json" : "-csl.json");
|
|
1514
|
+
writeAtomic(raw, json2);
|
|
1515
|
+
writeAtomic(p.md, renderAbstractMd(r));
|
|
1516
|
+
const meta = { ...metaFromRecord(r, [basename(raw), basename(p.md)], converter), retrieved_by: r.abstract.length ? abstractFrom : recordFrom };
|
|
1517
|
+
writeMeta(p, meta);
|
|
1518
|
+
return meta;
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
// src/config.ts
|
|
1522
|
+
import { existsSync as existsSync7, readFileSync as readFileSync6 } from "fs";
|
|
1523
|
+
import { join as join7 } from "path";
|
|
1524
|
+
|
|
1525
|
+
// src/checks.ts
|
|
1526
|
+
var GREEK = { "\u03B1": "alpha", "\u03B2": "beta", "\xDF": "beta", "\u03B3": "gamma", "\u03B4": "delta", "\u03B5": "epsilon", "\u03B6": "zeta", "\u03B7": "eta", "\u03B8": "theta", "\u03B9": "iota", "\u03BA": "kappa", "\u03BB": "lambda", "\u03BC": "mu", "\xB5": "mu", "\u03BD": "nu", "\u03BE": "xi", "\u03C0": "pi", "\u03C1": "rho", "\u03C3": "sigma", "\u03C4": "tau", "\u03C5": "upsilon", "\u03C6": "phi", "\u03C7": "chi", "\u03C8": "psi", "\u03C9": "omega", "\u0391": "alpha", "\u0392": "beta", "\u0393": "gamma", "\u0394": "delta", "\u03A9": "omega", "\u03A3": "sigma", "\u039A": "kappa", "\u039B": "lambda", "\u03A0": "pi", "\u03A6": "phi", "\u03A8": "psi" };
|
|
1527
|
+
var greek = (s) => s.replace(/[\u0370-\u03ff\u00b5\u00df]/g, (c) => GREEK[c] ?? c);
|
|
1528
|
+
var norm = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
1529
|
+
var compact = (s) => greek(s).toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
1530
|
+
function hasTitle(md, title) {
|
|
1531
|
+
const t = norm(title);
|
|
1532
|
+
if (!t) return true;
|
|
1533
|
+
if (norm(md).includes(t.slice(0, 60))) return true;
|
|
1534
|
+
return compact(md).includes(compact(title).slice(0, 50));
|
|
1535
|
+
}
|
|
1536
|
+
var WALL = [
|
|
1537
|
+
/checking your browser/i,
|
|
1538
|
+
/recaptcha/i,
|
|
1539
|
+
/verify (that )?you are (a )?human/i,
|
|
1540
|
+
/human verification/i,
|
|
1541
|
+
/access denied/i,
|
|
1542
|
+
/enable javascript/i,
|
|
1543
|
+
/javascript is (disabled|required)/i,
|
|
1544
|
+
/cloudflare/i,
|
|
1545
|
+
/captcha/i,
|
|
1546
|
+
/sign in to (continue|access)/i,
|
|
1547
|
+
/purchase (access|this article|pdf)/i,
|
|
1548
|
+
/subscribe to (read|access)/i,
|
|
1549
|
+
/cookie.{0,40}(consent|wall)/i,
|
|
1550
|
+
/403 forbidden/i,
|
|
1551
|
+
/page not found/i
|
|
1552
|
+
];
|
|
1553
|
+
function isWallPage(text3) {
|
|
1554
|
+
const short = text3.length < 4e3;
|
|
1555
|
+
if (short && WALL.some((re) => re.test(text3))) return true;
|
|
1556
|
+
const title = /<title>([^<]*)<\/title>/i.exec(text3)?.[1] ?? "";
|
|
1557
|
+
return /checking your browser|recaptcha|just a moment|access denied|attention required|human verification/i.test(title);
|
|
1558
|
+
}
|
|
1559
|
+
var headings = (md) => md.split("\n").filter((l) => /^#{1,6}\s/.test(l)).map((l) => l.replace(/^#+\s*/, ""));
|
|
1560
|
+
function proseWords(md) {
|
|
1561
|
+
let n = 0;
|
|
1562
|
+
for (const p of md.split(/\n\s*\n/).map((x) => x.trim()).filter((x) => x && !/^#{1,6}\s/.test(x))) {
|
|
1563
|
+
const words2 = p.split(/\s+/).length;
|
|
1564
|
+
if (words2 < 40 || /^(-|\*|\d+\.)\s/.test(p)) continue;
|
|
1565
|
+
const linkChars = (p.match(/\[[^\]]*\]\([^)]*\)/g) ?? []).join("").length;
|
|
1566
|
+
if (linkChars / p.length > 0.3) continue;
|
|
1567
|
+
n += words2;
|
|
1568
|
+
}
|
|
1569
|
+
return n;
|
|
1570
|
+
}
|
|
1571
|
+
var MIN_PROSE = 800;
|
|
1572
|
+
function hasMethodsAndResults(md) {
|
|
1573
|
+
const hs = headings(md);
|
|
1574
|
+
const has = (re) => hs.some((h) => re.test(h));
|
|
1575
|
+
if (has(/method|material|procedure|design|experimental|patients? and|participants/i) && has(/result|finding|outcome/i)) return true;
|
|
1576
|
+
const words2 = md.split(/\s+/).length;
|
|
1577
|
+
if (words2 < 1200 || hs.length < 2) return false;
|
|
1578
|
+
const SECTION = /^(abstract|summary|introduction|background|discussion|conclusions?|references|acknowledg|limitations|key points|funding|participants|statistical|supplementary|data availability|author contributions|competing interests|conflicts? of interest|disclosure)/i;
|
|
1579
|
+
return hs.filter((h) => SECTION.test(h.trim())).length >= 2;
|
|
1580
|
+
}
|
|
1581
|
+
var hasDoi = (md, doi) => !!doi && compact(md).includes(compact(doi));
|
|
1582
|
+
function checkFulltext(md, title, titleSource, doi, trusted = false) {
|
|
1583
|
+
if (isWallPage(md)) return { ok: false, reason: "wall page" };
|
|
1584
|
+
const inBody = hasTitle(md, title);
|
|
1585
|
+
if (!inBody && !(titleSource && hasTitle(titleSource, title)) && !hasDoi(md, doi)) return { ok: false, reason: "title missing" };
|
|
1586
|
+
if (!trusted && proseWords(md) < MIN_PROSE) return { ok: false, reason: "no methods/results" };
|
|
1587
|
+
if (!hasMethodsAndResults(md)) return { ok: false, reason: "no methods/results" };
|
|
1588
|
+
return { ok: true };
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
// src/html.ts
|
|
1592
|
+
import TurndownService from "turndown";
|
|
1593
|
+
import { gfm } from "@joplin/turndown-plugin-gfm";
|
|
1594
|
+
var td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced", bulletListMarker: "-" });
|
|
1595
|
+
td.use(gfm);
|
|
1596
|
+
td.remove(["script", "style", "nav", "noscript", "iframe", "button", "input", "select", "textarea", "header", "footer", "aside"]);
|
|
1597
|
+
var FURNITURE = /gdpr|cookie|onetrust|consent|advert|\bad-slot\b/i;
|
|
1598
|
+
td.remove(((n) => ["div", "section"].includes(String(n.nodeName).toLowerCase()) && FURNITURE.test(`${n.className ?? ""} ${n.id ?? ""}`)));
|
|
1599
|
+
td.addRule("drop-svg", { filter: (n) => n.nodeName.toLowerCase() === "svg", replacement: () => "" });
|
|
1600
|
+
td.addRule("drop-images", { filter: ["img"], replacement: () => "" });
|
|
1601
|
+
td.addRule("div-headings", {
|
|
1602
|
+
filter: (n) => String(n.nodeName).toLowerCase() === "div" && /(^|\s)h[1-6](\s|$)/.test(String(n.className ?? "")),
|
|
1603
|
+
replacement: (content, n) => {
|
|
1604
|
+
const level = /(^|\s)h([1-6])(\s|$)/.exec(String(n.className))[2];
|
|
1605
|
+
return `
|
|
1606
|
+
|
|
1607
|
+
${"#".repeat(Number(level))} ${content.trim()}
|
|
1608
|
+
|
|
1609
|
+
`;
|
|
1610
|
+
}
|
|
1611
|
+
});
|
|
1612
|
+
function htmlToMarkdown(html) {
|
|
1613
|
+
return td.turndown(html).replace(/\n{3,}/g, "\n\n").trim() + "\n";
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
// src/jats.ts
|
|
1617
|
+
import { XMLParser as XMLParser3 } from "fast-xml-parser";
|
|
1618
|
+
var parser3 = new XMLParser3({
|
|
1619
|
+
preserveOrder: true,
|
|
1620
|
+
ignoreAttributes: false,
|
|
1621
|
+
attributeNamePrefix: "@_",
|
|
1622
|
+
parseTagValue: false,
|
|
1623
|
+
trimValues: false,
|
|
1624
|
+
processEntities: true,
|
|
1625
|
+
htmlEntities: true
|
|
1626
|
+
});
|
|
1627
|
+
var tagOf2 = (n) => Object.keys(n).find((k) => k !== ":@") ?? "";
|
|
1628
|
+
var kids2 = (n) => n[tagOf2(n)] ?? [];
|
|
1629
|
+
var attr2 = (n, a) => n[":@"]?.[`@_${a}`] ?? "";
|
|
1630
|
+
var SKIP_INLINE = /* @__PURE__ */ new Set(["inline-graphic", "graphic", "private-char"]);
|
|
1631
|
+
function formula(n) {
|
|
1632
|
+
const all = [n, ...descendants(n)];
|
|
1633
|
+
const tex = all.find((x) => tagOf2(x) === "tex-math");
|
|
1634
|
+
if (tex) {
|
|
1635
|
+
let t = plainText(kids2(tex)).trim();
|
|
1636
|
+
const m = /\\begin\{document\}([\s\S]*?)\\end\{document\}/.exec(t);
|
|
1637
|
+
if (m) t = m[1].trim();
|
|
1638
|
+
t = t.replace(/^\$+|\$+$/g, "").trim();
|
|
1639
|
+
return t ? `$${t}$` : "";
|
|
1640
|
+
}
|
|
1641
|
+
const mml = all.find((x) => /^(mml:)?math$/.test(tagOf2(x)));
|
|
1642
|
+
return mml ? plainText(kids2(mml)).replace(/\s+/g, "") : plainText(kids2(n));
|
|
1643
|
+
}
|
|
1644
|
+
function descendants(n) {
|
|
1645
|
+
if ("#text" in n) return [];
|
|
1646
|
+
const out = [];
|
|
1647
|
+
for (const k of kids2(n)) {
|
|
1648
|
+
out.push(k, ...descendants(k));
|
|
1649
|
+
}
|
|
1650
|
+
return out;
|
|
1651
|
+
}
|
|
1652
|
+
function inlineText2(nodes) {
|
|
1653
|
+
let out = "";
|
|
1654
|
+
for (const n of nodes) {
|
|
1655
|
+
if ("#text" in n) {
|
|
1656
|
+
out += String(n["#text"]);
|
|
1657
|
+
continue;
|
|
1658
|
+
}
|
|
1659
|
+
const t = tagOf2(n);
|
|
1660
|
+
if (SKIP_INLINE.has(t)) continue;
|
|
1661
|
+
const inner = () => inlineText2(kids2(n));
|
|
1662
|
+
switch (t) {
|
|
1663
|
+
case "italic":
|
|
1664
|
+
case "i":
|
|
1665
|
+
out += `*${inner()}*`;
|
|
1666
|
+
break;
|
|
1667
|
+
case "bold":
|
|
1668
|
+
case "b":
|
|
1669
|
+
out += `**${inner()}**`;
|
|
1670
|
+
break;
|
|
1671
|
+
case "sup":
|
|
1672
|
+
out += `^${inner()}^`;
|
|
1673
|
+
break;
|
|
1674
|
+
case "sub":
|
|
1675
|
+
out += `~${inner()}~`;
|
|
1676
|
+
break;
|
|
1677
|
+
case "break":
|
|
1678
|
+
out += " ";
|
|
1679
|
+
break;
|
|
1680
|
+
case "ext-link":
|
|
1681
|
+
case "uri": {
|
|
1682
|
+
const href = attr2(n, "xlink:href") || attr2(n, "href");
|
|
1683
|
+
const text3 = inner().trim() || href;
|
|
1684
|
+
out += href ? `[${text3}](${href})` : text3;
|
|
1685
|
+
break;
|
|
1686
|
+
}
|
|
1687
|
+
case "inline-formula":
|
|
1688
|
+
case "tex-math":
|
|
1689
|
+
case "mml:math":
|
|
1690
|
+
case "alternatives":
|
|
1691
|
+
out += formula(n);
|
|
1692
|
+
break;
|
|
1693
|
+
default:
|
|
1694
|
+
out += inner();
|
|
1695
|
+
}
|
|
1696
|
+
}
|
|
1697
|
+
return out;
|
|
1698
|
+
}
|
|
1699
|
+
function plainText(nodes) {
|
|
1700
|
+
let out = "";
|
|
1701
|
+
for (const n of nodes) out += "#text" in n ? String(n["#text"]) : plainText(kids2(n));
|
|
1702
|
+
return out;
|
|
1703
|
+
}
|
|
1704
|
+
var squash2 = (s) => s.replace(/\s+/g, " ").trim();
|
|
1705
|
+
var cell = (s) => squash2(s).replace(/\|/g, "\\|");
|
|
1706
|
+
function renderTable(table3) {
|
|
1707
|
+
const rows = [];
|
|
1708
|
+
const walk = (nodes) => {
|
|
1709
|
+
for (const n of nodes) {
|
|
1710
|
+
const t = tagOf2(n);
|
|
1711
|
+
if (t === "tr") rows.push(kids2(n).filter((c) => ["td", "th"].includes(tagOf2(c))).map((c) => cell(inlineText2(kids2(c)))));
|
|
1712
|
+
else if (!("#text" in n)) walk(kids2(n));
|
|
1713
|
+
}
|
|
1714
|
+
};
|
|
1715
|
+
walk(kids2(table3));
|
|
1716
|
+
if (!rows.length) return "";
|
|
1717
|
+
const width = Math.max(...rows.map((r) => r.length));
|
|
1718
|
+
const pad = (r) => [...r, ...Array(width - r.length).fill("")];
|
|
1719
|
+
const line = (r) => `| ${pad(r).join(" | ")} |`;
|
|
1720
|
+
return [line(rows[0]), `| ${Array(width).fill("---").join(" | ")} |`, ...rows.slice(1).map(line)].join("\n");
|
|
1721
|
+
}
|
|
1722
|
+
function labelCaption(n) {
|
|
1723
|
+
const label = n ? squash2(plainText(kids2(n).filter((c) => tagOf2(c) === "label"))) : "";
|
|
1724
|
+
const cap = kids2(n).find((c) => tagOf2(c) === "caption");
|
|
1725
|
+
const capText = cap ? squash2(inlineText2(kids2(cap))) : "";
|
|
1726
|
+
return [label ? `**${label}.**` : "", capText].filter(Boolean).join(" ");
|
|
1727
|
+
}
|
|
1728
|
+
function renderBlocks(nodes, depth) {
|
|
1729
|
+
const out = [];
|
|
1730
|
+
for (const n of nodes) {
|
|
1731
|
+
if ("#text" in n) {
|
|
1732
|
+
const t2 = squash2(String(n["#text"]));
|
|
1733
|
+
if (t2) out.push(t2);
|
|
1734
|
+
continue;
|
|
1735
|
+
}
|
|
1736
|
+
const t = tagOf2(n);
|
|
1737
|
+
const c = kids2(n);
|
|
1738
|
+
switch (t) {
|
|
1739
|
+
case "sec":
|
|
1740
|
+
case "app":
|
|
1741
|
+
case "ack":
|
|
1742
|
+
case "boxed-text":
|
|
1743
|
+
case "notes":
|
|
1744
|
+
case "glossary": {
|
|
1745
|
+
const title = c.find((x) => tagOf2(x) === "title");
|
|
1746
|
+
if (title) out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash2(inlineText2(kids2(title)))}`);
|
|
1747
|
+
else if (t === "ack") out.push(`${"#".repeat(Math.min(depth + 1, 6))} Acknowledgements`);
|
|
1748
|
+
out.push(...renderBlocks(c.filter((x) => tagOf2(x) !== "title"), depth + 1));
|
|
1749
|
+
break;
|
|
1750
|
+
}
|
|
1751
|
+
case "title":
|
|
1752
|
+
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${squash2(inlineText2(c))}`);
|
|
1753
|
+
break;
|
|
1754
|
+
case "p": {
|
|
1755
|
+
const BLOCK = /* @__PURE__ */ new Set(["table-wrap", "fig", "list", "disp-formula", "disp-quote", "boxed-text", "supplementary-material"]);
|
|
1756
|
+
const inline2 = c.filter((x) => "#text" in x || !BLOCK.has(tagOf2(x)));
|
|
1757
|
+
const blocks2 = c.filter((x) => !("#text" in x) && BLOCK.has(tagOf2(x)));
|
|
1758
|
+
const s = squash2(inlineText2(inline2));
|
|
1759
|
+
if (s) out.push(s);
|
|
1760
|
+
out.push(...renderBlocks(blocks2, depth));
|
|
1761
|
+
break;
|
|
1762
|
+
}
|
|
1763
|
+
case "list": {
|
|
1764
|
+
const ordered = /order|alpha|roman|number/i.test(attr2(n, "list-type"));
|
|
1765
|
+
let i = 0;
|
|
1766
|
+
for (const li of c.filter((x) => tagOf2(x) === "list-item")) {
|
|
1767
|
+
i++;
|
|
1768
|
+
const body = renderBlocks(kids2(li).filter((x) => tagOf2(x) !== "label"), depth).join(" ");
|
|
1769
|
+
out.push(`${ordered ? `${i}.` : "-"} ${body}`);
|
|
1770
|
+
}
|
|
1771
|
+
break;
|
|
1772
|
+
}
|
|
1773
|
+
case "table-wrap": {
|
|
1774
|
+
const head = labelCaption(n);
|
|
1775
|
+
if (head) out.push(head);
|
|
1776
|
+
for (const tbl of c.filter((x) => tagOf2(x) === "table")) {
|
|
1777
|
+
const md = renderTable(tbl);
|
|
1778
|
+
if (md) out.push(md);
|
|
1779
|
+
}
|
|
1780
|
+
const foot = c.find((x) => tagOf2(x) === "table-wrap-foot");
|
|
1781
|
+
if (foot) out.push(...renderBlocks(kids2(foot), depth));
|
|
1782
|
+
break;
|
|
1783
|
+
}
|
|
1784
|
+
case "fig": {
|
|
1785
|
+
const head = labelCaption(n);
|
|
1786
|
+
out.push(head || "*(figure)*");
|
|
1787
|
+
break;
|
|
1788
|
+
}
|
|
1789
|
+
case "disp-formula": {
|
|
1790
|
+
const s = squash2(formula(n));
|
|
1791
|
+
if (s) out.push(s);
|
|
1792
|
+
break;
|
|
1793
|
+
}
|
|
1794
|
+
case "disp-quote":
|
|
1795
|
+
out.push(...renderBlocks(c, depth).map((l) => `> ${l}`));
|
|
1796
|
+
break;
|
|
1797
|
+
case "def-list":
|
|
1798
|
+
for (const d of c.filter((x) => tagOf2(x) === "def-item")) {
|
|
1799
|
+
const term = d["def-item"].find((x) => tagOf2(x) === "term");
|
|
1800
|
+
const def = d["def-item"].find((x) => tagOf2(x) === "def");
|
|
1801
|
+
out.push(`**${squash2(inlineText2(term ? kids2(term) : []))}**: ${def ? renderBlocks(kids2(def), depth).join(" ") : ""}`);
|
|
1802
|
+
}
|
|
1803
|
+
break;
|
|
1804
|
+
case "ref-list": {
|
|
1805
|
+
const title = c.find((x) => tagOf2(x) === "title");
|
|
1806
|
+
out.push(`${"#".repeat(Math.min(depth + 1, 6))} ${title ? squash2(inlineText2(kids2(title))) : "References"}`);
|
|
1807
|
+
for (const r of c.filter((x) => tagOf2(x) === "ref")) {
|
|
1808
|
+
const rk = kids2(r);
|
|
1809
|
+
const label = squash2(plainText(rk.filter((x) => tagOf2(x) === "label")));
|
|
1810
|
+
out.push(`- ${label ? `${label} ` : ""}${renderCitation(rk.filter((x) => tagOf2(x) !== "label"))}`);
|
|
1811
|
+
}
|
|
1812
|
+
break;
|
|
1813
|
+
}
|
|
1814
|
+
case "supplementary-material":
|
|
1815
|
+
case "graphic":
|
|
1816
|
+
case "media":
|
|
1817
|
+
case "fn-group":
|
|
1818
|
+
case "label":
|
|
1819
|
+
break;
|
|
1820
|
+
default:
|
|
1821
|
+
out.push(...renderBlocks(c, depth));
|
|
1822
|
+
}
|
|
1823
|
+
}
|
|
1824
|
+
return out;
|
|
1825
|
+
}
|
|
1826
|
+
function renderCitation(nodes) {
|
|
1827
|
+
const alt = nodes.find((x) => tagOf2(x) === "citation-alternatives");
|
|
1828
|
+
if (alt) return renderCitation(kids2(alt));
|
|
1829
|
+
const mixed = nodes.find((x) => ["mixed-citation", "citation", "nlm-citation"].includes(tagOf2(x)) && !kids2(x).some((y) => tagOf2(y) === "person-group"));
|
|
1830
|
+
if (mixed) return squash2(inlineText2(kids2(mixed)));
|
|
1831
|
+
const el = nodes.find((x) => ["element-citation", "mixed-citation", "citation", "nlm-citation"].includes(tagOf2(x)));
|
|
1832
|
+
if (!el) return squash2(inlineText2(nodes));
|
|
1833
|
+
const f = kids2(el);
|
|
1834
|
+
const get = (tag) => f.filter((x) => tagOf2(x) === tag).map((x) => squash2(inlineText2(kids2(x))));
|
|
1835
|
+
const names = f.filter((x) => tagOf2(x) === "person-group").flatMap((g) => kids2(g)).map((x) => {
|
|
1836
|
+
const t = tagOf2(x);
|
|
1837
|
+
if (t === "etal") return "et al";
|
|
1838
|
+
if (t === "collab") return squash2(plainText(kids2(x)));
|
|
1839
|
+
if (t === "name" || t === "string-name") {
|
|
1840
|
+
const sur = squash2(plainText(kids2(x).filter((y) => tagOf2(y) === "surname")));
|
|
1841
|
+
const giv = squash2(plainText(kids2(x).filter((y) => tagOf2(y) === "given-names")));
|
|
1842
|
+
return [sur, giv].filter(Boolean).join(" ");
|
|
1843
|
+
}
|
|
1844
|
+
return "";
|
|
1845
|
+
}).filter(Boolean);
|
|
1846
|
+
const vol = get("volume")[0], issue = get("issue")[0], fp = get("fpage")[0], lp = get("lpage")[0];
|
|
1847
|
+
const pages = fp ? lp ? `${fp}-${lp}` : fp : get("elocation-id")[0];
|
|
1848
|
+
const where = [get("source")[0], get("year")[0] ? `${get("year")[0]};` : "", vol ? `${vol}${issue ? `(${issue})` : ""}` : "", pages ? `:${pages}` : ""].filter(Boolean).join("").replace(/;$/, "");
|
|
1849
|
+
const ids = f.filter((x) => tagOf2(x) === "pub-id").map((x) => `${attr2(x, "pub-id-type")}:${squash2(plainText(kids2(x)))}`);
|
|
1850
|
+
return [names.join(", "), [...get("article-title"), ...get("chapter-title")][0], where, ...ids].filter(Boolean).join(". ").replace(/\.\./g, ".");
|
|
1851
|
+
}
|
|
1852
|
+
var FN_LABELS = {
|
|
1853
|
+
"conflict": "Conflicts of interest",
|
|
1854
|
+
"COI-statement": "Conflicts of interest",
|
|
1855
|
+
"coi-statement": "Conflicts of interest",
|
|
1856
|
+
"financial-disclosure": "Financial disclosure",
|
|
1857
|
+
"supported-by": "Funding",
|
|
1858
|
+
"funding": "Funding",
|
|
1859
|
+
"equal": "Author note",
|
|
1860
|
+
"con": "Contributions",
|
|
1861
|
+
"contributed-by": "Contributions",
|
|
1862
|
+
"other": ""
|
|
1863
|
+
};
|
|
1864
|
+
function declarations(meta, back) {
|
|
1865
|
+
const out = [];
|
|
1866
|
+
const fg = meta ? findDeep2(kids2(meta), "funding-group") : void 0;
|
|
1867
|
+
if (fg) {
|
|
1868
|
+
const statements = kids2(fg).filter((x) => tagOf2(x) === "funding-statement").map((x) => squash2(inlineText2(kids2(x)))).filter(Boolean);
|
|
1869
|
+
const awards = kids2(fg).filter((x) => tagOf2(x) === "award-group").map((ag) => {
|
|
1870
|
+
const src = kids2(ag).filter((x) => tagOf2(x) === "funding-source").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
|
|
1871
|
+
const ids = kids2(ag).filter((x) => tagOf2(x) === "award-id").map((x) => squash2(plainText(kids2(x)))).filter(Boolean);
|
|
1872
|
+
return [src.join(", "), ids.length ? `(${ids.join(", ")})` : ""].filter(Boolean).join(" ");
|
|
1873
|
+
}).filter(Boolean);
|
|
1874
|
+
const text3 = statements.length ? statements.join(" ") : awards.join("; ");
|
|
1875
|
+
if (text3) out.push(`Funding: ${text3}`);
|
|
1876
|
+
}
|
|
1877
|
+
const fns = [];
|
|
1878
|
+
const notes = meta ? findDeep2(kids2(meta), "author-notes") : void 0;
|
|
1879
|
+
if (notes) fns.push(...kids2(notes).filter((x) => tagOf2(x) === "fn"));
|
|
1880
|
+
if (back) for (const g of kids2(back).filter((x) => tagOf2(x) === "fn-group")) fns.push(...kids2(g).filter((x) => tagOf2(x) === "fn"));
|
|
1881
|
+
for (const fn of fns) {
|
|
1882
|
+
const text3 = renderBlocks(kids2(fn).filter((x) => tagOf2(x) !== "label"), 1).join(" ").trim();
|
|
1883
|
+
if (!text3) continue;
|
|
1884
|
+
const label = FN_LABELS[attr2(fn, "fn-type")] ?? "";
|
|
1885
|
+
out.push(label && !new RegExp(`^[*_]*${label.split(" ")[0]}`, "i").test(text3) ? `${label}: ${text3}` : text3);
|
|
1886
|
+
}
|
|
1887
|
+
return out;
|
|
1888
|
+
}
|
|
1889
|
+
function find(nodes, path) {
|
|
1890
|
+
let cur = nodes;
|
|
1891
|
+
let hit;
|
|
1892
|
+
for (const tag of path) {
|
|
1893
|
+
hit = cur.find((x) => tagOf2(x) === tag);
|
|
1894
|
+
if (!hit) return void 0;
|
|
1895
|
+
cur = kids2(hit);
|
|
1896
|
+
}
|
|
1897
|
+
return hit;
|
|
1898
|
+
}
|
|
1899
|
+
function findDeep2(nodes, tag) {
|
|
1900
|
+
for (const n of nodes) {
|
|
1901
|
+
if ("#text" in n) continue;
|
|
1902
|
+
if (tagOf2(n) === tag) return n;
|
|
1903
|
+
const hit = findDeep2(kids2(n), tag);
|
|
1904
|
+
if (hit) return hit;
|
|
1905
|
+
}
|
|
1906
|
+
return void 0;
|
|
1907
|
+
}
|
|
1908
|
+
function parseJats(xml) {
|
|
1909
|
+
const doc = parser3.parse(xml);
|
|
1910
|
+
const article = find(doc, ["pmc-articleset", "article"]) ?? find(doc, ["article"]) ?? findDeep2(doc, "article");
|
|
1911
|
+
const a = article ? kids2(article) : [];
|
|
1912
|
+
const front = find(a, ["front"]);
|
|
1913
|
+
const meta = front ? find(kids2(front), ["article-meta"]) ?? front : void 0;
|
|
1914
|
+
const titleNode = meta ? find(kids2(meta), ["title-group", "article-title"]) ?? findDeep2(kids2(meta), "article-title") : void 0;
|
|
1915
|
+
const abstractNode = meta ? kids2(meta).find((x) => tagOf2(x) === "abstract" && !attr2(x, "abstract-type")) ?? findDeep2(kids2(meta), "abstract") : void 0;
|
|
1916
|
+
const body = find(a, ["body"]);
|
|
1917
|
+
const back = find(a, ["back"]);
|
|
1918
|
+
return {
|
|
1919
|
+
title: titleNode ? squash2(inlineText2(kids2(titleNode))) : "",
|
|
1920
|
+
abstract: abstractNode ? renderBlocks(kids2(abstractNode).filter((x) => tagOf2(x) !== "title"), 2) : [],
|
|
1921
|
+
body: body ? renderBlocks(kids2(body), 1) : [],
|
|
1922
|
+
declarations: declarations(meta, back),
|
|
1923
|
+
back: back ? renderBlocks(kids2(back), 1) : [],
|
|
1924
|
+
hasBody: !!body && kids2(body).length > 0
|
|
1925
|
+
};
|
|
1926
|
+
}
|
|
1927
|
+
function renderFulltextMd(doc, headLine2) {
|
|
1928
|
+
const parts = [`# ${doc.title}`, "", headLine2, ""];
|
|
1929
|
+
if (doc.abstract.length) parts.push("## Abstract", "", ...doc.abstract.flatMap((b) => [b, ""]));
|
|
1930
|
+
parts.push(...doc.body.flatMap((b) => [b, ""]));
|
|
1931
|
+
if (doc.declarations.length) parts.push("## Declarations", "", ...doc.declarations.flatMap((b) => [b, ""]));
|
|
1932
|
+
parts.push(...doc.back.flatMap((b) => [b, ""]));
|
|
1933
|
+
return parts.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd() + "\n";
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
// src/browser.ts
|
|
1937
|
+
import { existsSync as existsSync5, mkdirSync as mkdirSync2, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "fs";
|
|
1938
|
+
import { join as join5 } from "path";
|
|
1939
|
+
import { chromium } from "playwright-core";
|
|
1940
|
+
function browserConfig(file = CONFIG_FILE) {
|
|
1941
|
+
const out = { browsers: [], headless: "auto" };
|
|
1942
|
+
if (!existsSync5(file)) return out;
|
|
1943
|
+
try {
|
|
1944
|
+
const d = JSON.parse(readFileSync4(file, "utf8")).download ?? {};
|
|
1945
|
+
if (Array.isArray(d.browsers)) {
|
|
1946
|
+
for (const b of d.browsers) {
|
|
1947
|
+
if (typeof b !== "string") continue;
|
|
1948
|
+
const v = resolveEnv(b).trim();
|
|
1949
|
+
if (v && !/^\$\{[A-Z0-9_]+\}$/.test(v) && !out.browsers.includes(v)) out.browsers.push(v);
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
if (d.headless === "auto" || typeof d.headless === "boolean") out.headless = d.headless;
|
|
1953
|
+
} catch {
|
|
1954
|
+
}
|
|
1955
|
+
return out;
|
|
1956
|
+
}
|
|
1957
|
+
var INSTALL_HINT = "npx playwright-core install chromium";
|
|
1958
|
+
var isRemote = (endpoint) => /^wss?:\/\//i.test(endpoint);
|
|
1959
|
+
var isCdp = (endpoint) => /^cdp:\/\//i.test(endpoint);
|
|
1960
|
+
var cdpHttp = (endpoint) => endpoint.replace(/^cdp:\/\//i, "http://");
|
|
1961
|
+
var rungKind = (endpoint) => isCdp(endpoint) ? "chrome" : isRemote(endpoint) ? "server" : "local";
|
|
1962
|
+
function localChromium() {
|
|
1963
|
+
try {
|
|
1964
|
+
const p = chromium.executablePath();
|
|
1965
|
+
return existsSync5(p) ? p : null;
|
|
1966
|
+
} catch {
|
|
1967
|
+
return null;
|
|
1968
|
+
}
|
|
1969
|
+
}
|
|
1970
|
+
function browserLabel(endpoint) {
|
|
1971
|
+
if (rungKind(endpoint) === "local") return "local";
|
|
1972
|
+
try {
|
|
1973
|
+
const h = new URL(isCdp(endpoint) ? cdpHttp(endpoint) : endpoint).hostname;
|
|
1974
|
+
return isCdp(endpoint) ? `chrome@${h}` : h;
|
|
1975
|
+
} catch {
|
|
1976
|
+
return endpoint;
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
var hasDisplay = () => !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
|
|
1980
|
+
function browserHeadless(endpoint, cfg = browserConfig()) {
|
|
1981
|
+
if (isCdp(endpoint)) return false;
|
|
1982
|
+
return cfg.headless === "auto" ? isRemote(endpoint) ? false : !hasDisplay() : cfg.headless;
|
|
1983
|
+
}
|
|
1984
|
+
function browserRungs(cfg = browserConfig()) {
|
|
1985
|
+
return cfg.browsers.map((endpoint) => ({
|
|
1986
|
+
endpoint,
|
|
1987
|
+
label: browserLabel(endpoint),
|
|
1988
|
+
kind: rungKind(endpoint),
|
|
1989
|
+
available: rungKind(endpoint) !== "local" || localChromium() !== null,
|
|
1990
|
+
headless: browserHeadless(endpoint, cfg)
|
|
1991
|
+
}));
|
|
1992
|
+
}
|
|
1993
|
+
var CHALLENGE = /just a moment|nur einen moment|attention required|verify you are human|human verification|checking your browser|access denied/i;
|
|
1994
|
+
var CONNECT_TIMEOUT_MS = 2e4;
|
|
1995
|
+
var NAV_TIMEOUT_MS = 2e4;
|
|
1996
|
+
var CHALLENGE_WAIT_MS = 8e3;
|
|
1997
|
+
var DOWNLOAD_WAIT_MS = 1e4;
|
|
1998
|
+
var IDLE_CLOSE_MS = 12e4;
|
|
1999
|
+
var ctxs = /* @__PURE__ */ new Map();
|
|
2000
|
+
var idleTimer = null;
|
|
2001
|
+
function profileDir() {
|
|
2002
|
+
const dir = join5(AI4L_LIBRARY_DIR, ".browser");
|
|
2003
|
+
mkdirSync2(join5(dir, "Default"), { recursive: true });
|
|
2004
|
+
const prefs = join5(dir, "Default", "Preferences");
|
|
2005
|
+
let p = {};
|
|
2006
|
+
try {
|
|
2007
|
+
p = JSON.parse(readFileSync4(prefs, "utf8"));
|
|
2008
|
+
} catch {
|
|
2009
|
+
}
|
|
2010
|
+
p.plugins = { ...p.plugins ?? {}, always_open_pdf_externally: true };
|
|
2011
|
+
p.download = { ...p.download ?? {}, prompt_for_download: false };
|
|
2012
|
+
writeFileSync2(prefs, JSON.stringify(p));
|
|
2013
|
+
return dir;
|
|
2014
|
+
}
|
|
2015
|
+
var LAUNCH_ARGS = ["--disable-blink-features=AutomationControlled", "--window-size=1366,900"];
|
|
2016
|
+
var LOCAL_WINDOW_ARGS = [...LAUNCH_ARGS, "--window-position=4000,4000"];
|
|
2017
|
+
var TIMEZONE = Intl.DateTimeFormat().resolvedOptions().timeZone || "Europe/Berlin";
|
|
2018
|
+
var CONTEXT_OPTS = { acceptDownloads: true, viewport: null, locale: "en-US", timezoneId: TIMEZONE };
|
|
2019
|
+
function stateFile(endpoint) {
|
|
2020
|
+
const dir = join5(AI4L_LIBRARY_DIR, ".browser");
|
|
2021
|
+
mkdirSync2(dir, { recursive: true });
|
|
2022
|
+
return join5(dir, `${browserLabel(endpoint).replace(/[^A-Za-z0-9.-]+/g, "_")}.json`);
|
|
2023
|
+
}
|
|
2024
|
+
async function context(endpoint, cfg) {
|
|
2025
|
+
let p = ctxs.get(endpoint);
|
|
2026
|
+
if (!p) {
|
|
2027
|
+
p = (async () => {
|
|
2028
|
+
const headless = browserHeadless(endpoint, cfg);
|
|
2029
|
+
const drop = () => {
|
|
2030
|
+
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
2031
|
+
};
|
|
2032
|
+
if (isCdp(endpoint)) {
|
|
2033
|
+
const browser = await chromium.connectOverCDP(cdpHttp(endpoint), { timeout: CONNECT_TIMEOUT_MS });
|
|
2034
|
+
browser.on("disconnected", drop);
|
|
2035
|
+
const ctx2 = browser.contexts()[0] ?? await browser.newContext(CONTEXT_OPTS);
|
|
2036
|
+
return { ctx: ctx2, browser };
|
|
2037
|
+
}
|
|
2038
|
+
if (isRemote(endpoint)) {
|
|
2039
|
+
const browser = await chromium.connect(endpoint, {
|
|
2040
|
+
timeout: CONNECT_TIMEOUT_MS,
|
|
2041
|
+
headers: { "x-playwright-launch-options": JSON.stringify({ headless, args: LAUNCH_ARGS, ignoreDefaultArgs: ["--enable-automation"] }) }
|
|
2042
|
+
});
|
|
2043
|
+
browser.on("disconnected", drop);
|
|
2044
|
+
const state = stateFile(endpoint);
|
|
2045
|
+
const ctx2 = await browser.newContext({ ...CONTEXT_OPTS, ...existsSync5(state) ? { storageState: state } : {} });
|
|
2046
|
+
return { ctx: ctx2, browser };
|
|
2047
|
+
}
|
|
2048
|
+
if (!localChromium()) throw new Error(`no local Chromium (${INSTALL_HINT})`);
|
|
2049
|
+
const ctx = await chromium.launchPersistentContext(profileDir(), {
|
|
2050
|
+
channel: "chromium",
|
|
2051
|
+
headless,
|
|
2052
|
+
...CONTEXT_OPTS,
|
|
2053
|
+
args: headless ? LAUNCH_ARGS : LOCAL_WINDOW_ARGS,
|
|
2054
|
+
ignoreDefaultArgs: ["--enable-automation"]
|
|
2055
|
+
});
|
|
2056
|
+
ctx.on("close", drop);
|
|
2057
|
+
return { ctx, browser: null };
|
|
2058
|
+
})();
|
|
2059
|
+
ctxs.set(endpoint, p);
|
|
2060
|
+
p.catch(() => {
|
|
2061
|
+
if (ctxs.get(endpoint) === p) ctxs.delete(endpoint);
|
|
2062
|
+
});
|
|
2063
|
+
}
|
|
2064
|
+
return (await p).ctx;
|
|
2065
|
+
}
|
|
2066
|
+
function touchIdle() {
|
|
2067
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
2068
|
+
idleTimer = setTimeout(() => {
|
|
2069
|
+
void closeBrowsers();
|
|
2070
|
+
}, IDLE_CLOSE_MS);
|
|
2071
|
+
idleTimer.unref();
|
|
2072
|
+
}
|
|
2073
|
+
async function closeBrowsers() {
|
|
2074
|
+
const open = [...ctxs.entries()];
|
|
2075
|
+
ctxs.clear();
|
|
2076
|
+
for (const [endpoint, p] of open) {
|
|
2077
|
+
try {
|
|
2078
|
+
const { ctx, browser } = await p;
|
|
2079
|
+
if (isCdp(endpoint)) {
|
|
2080
|
+
await browser?.close().catch(() => void 0);
|
|
2081
|
+
continue;
|
|
2082
|
+
}
|
|
2083
|
+
if (browser) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
2084
|
+
await ctx.close().catch(() => void 0);
|
|
2085
|
+
await browser?.close().catch(() => void 0);
|
|
2086
|
+
} catch {
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
}
|
|
2090
|
+
async function browserFetch(url, endpoint, cfg = browserConfig()) {
|
|
2091
|
+
const ctx = await context(endpoint, cfg);
|
|
2092
|
+
touchIdle();
|
|
2093
|
+
const page = await ctx.newPage();
|
|
2094
|
+
try {
|
|
2095
|
+
let pdfBest = null;
|
|
2096
|
+
const pdfReads = [];
|
|
2097
|
+
let status, contentType = "";
|
|
2098
|
+
page.on("response", (r) => {
|
|
2099
|
+
if (r.request().isNavigationRequest() && r.frame() === page.mainFrame()) {
|
|
2100
|
+
status = r.status();
|
|
2101
|
+
contentType = r.headers()["content-type"] ?? "";
|
|
2102
|
+
}
|
|
2103
|
+
if (r.status() !== 200 || !/application\/pdf/i.test(r.headers()["content-type"] ?? "")) return;
|
|
2104
|
+
pdfReads.push(r.body().then((b) => {
|
|
2105
|
+
if (b.subarray(0, 5).toString("latin1") === "%PDF-" && (!pdfBest || b.length > pdfBest.length)) pdfBest = b;
|
|
2106
|
+
}).catch(() => void 0));
|
|
2107
|
+
});
|
|
2108
|
+
let download2 = null;
|
|
2109
|
+
const dl = page.waitForEvent("download", { timeout: NAV_TIMEOUT_MS + CHALLENGE_WAIT_MS + DOWNLOAD_WAIT_MS }).then((d) => {
|
|
2110
|
+
download2 = d;
|
|
2111
|
+
return d;
|
|
2112
|
+
}).catch(() => null);
|
|
2113
|
+
let navError = null;
|
|
2114
|
+
const nav = page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).then((resp) => {
|
|
2115
|
+
if (status === void 0) {
|
|
2116
|
+
status = resp?.status();
|
|
2117
|
+
contentType = resp?.headers()["content-type"] ?? "";
|
|
2118
|
+
}
|
|
2119
|
+
}).catch((e) => {
|
|
2120
|
+
navError = (e instanceof Error ? e.message : String(e)).split("\n")[0].slice(0, 80);
|
|
2121
|
+
});
|
|
2122
|
+
await Promise.race([nav, dl]);
|
|
2123
|
+
const pause = (ms) => Promise.race([dl, new Promise((r) => setTimeout(r, ms))]);
|
|
2124
|
+
if (download2) {
|
|
2125
|
+
} else if (navError && !/download is starting|net::ERR_ABORTED/i.test(navError)) {
|
|
2126
|
+
await pause(2e3);
|
|
2127
|
+
if (!download2) throw new Error(`navigation: ${navError}`);
|
|
2128
|
+
} else {
|
|
2129
|
+
await pause(2e3);
|
|
2130
|
+
const t0 = Date.now();
|
|
2131
|
+
while (!download2 && Date.now() - t0 < CHALLENGE_WAIT_MS) {
|
|
2132
|
+
const title2 = await page.title().catch(() => "");
|
|
2133
|
+
if (!CHALLENGE.test(title2)) break;
|
|
2134
|
+
await pause(1e3);
|
|
2135
|
+
}
|
|
2136
|
+
if (!download2) await pause(DOWNLOAD_WAIT_MS / 5);
|
|
2137
|
+
}
|
|
2138
|
+
if (download2) {
|
|
2139
|
+
const d = download2;
|
|
2140
|
+
return { bytes: readFileSync4(await d.path()), contentType: "application/pdf", url: d.url(), via: "download" };
|
|
2141
|
+
}
|
|
2142
|
+
const title = await page.title().catch(() => "");
|
|
2143
|
+
if (CHALLENGE.test(title)) throw new Error(`challenge not cleared (${title.slice(0, 40)})`);
|
|
2144
|
+
if (status && status >= 400) throw new Error(`HTTP ${status}`);
|
|
2145
|
+
const docType = await page.evaluate(() => document.contentType).catch(() => "");
|
|
2146
|
+
const inViewer = /application\/pdf/i.test(contentType) || /application\/pdf/i.test(docType);
|
|
2147
|
+
if (inViewer) {
|
|
2148
|
+
const t1 = Date.now();
|
|
2149
|
+
while (Date.now() - t1 < DOWNLOAD_WAIT_MS) {
|
|
2150
|
+
await Promise.all(pdfReads);
|
|
2151
|
+
if (pdfBest && pdfBest.length > 4096) break;
|
|
2152
|
+
if (!pdfReads.length && Date.now() - t1 > 3e3) break;
|
|
2153
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
2154
|
+
}
|
|
2155
|
+
if (pdfBest) return { bytes: pdfBest, contentType: "application/pdf", url: page.url(), via: "response" };
|
|
2156
|
+
}
|
|
2157
|
+
if (inViewer) {
|
|
2158
|
+
const r = await ctx.request.get(page.url(), { timeout: NAV_TIMEOUT_MS });
|
|
2159
|
+
if (!r.ok()) throw new Error(`HTTP ${r.status()} (pdf request)`);
|
|
2160
|
+
return { bytes: Buffer.from(await r.body()), contentType: "application/pdf", url: page.url(), via: "request" };
|
|
2161
|
+
}
|
|
2162
|
+
const html = await page.content();
|
|
2163
|
+
return { bytes: Buffer.from(html, "utf8"), contentType: contentType || "text/html", url: page.url(), via: "page" };
|
|
2164
|
+
} finally {
|
|
2165
|
+
await page.close().catch(() => void 0);
|
|
2166
|
+
if (isRemote(endpoint)) await ctx.storageState({ path: stateFile(endpoint) }).catch(() => void 0);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
// src/unlock.ts
|
|
2171
|
+
var ENDPOINT = "https://api.brightdata.com/request";
|
|
2172
|
+
var TIMEOUT_MS2 = 6e4;
|
|
2173
|
+
var unlockerConfigured = () => !!process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
2174
|
+
async function unlock(url, fetchImpl = fetch) {
|
|
2175
|
+
const token = process.env.BRIGHTDATA_API_TOKEN?.trim();
|
|
2176
|
+
if (!token) throw new Error("unlocker not configured");
|
|
2177
|
+
const zone = process.env.BRIGHTDATA_ZONE?.trim() || "mcp_unlocker";
|
|
2178
|
+
const res = await fetchImpl(ENDPOINT, {
|
|
2179
|
+
method: "POST",
|
|
2180
|
+
headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/json" },
|
|
2181
|
+
body: JSON.stringify({ zone, url, format: "raw" }),
|
|
2182
|
+
signal: AbortSignal.timeout(TIMEOUT_MS2)
|
|
2183
|
+
});
|
|
2184
|
+
if (!res.ok) throw new Error(`unlocker HTTP ${res.status}`);
|
|
2185
|
+
return { bytes: Buffer.from(await res.arrayBuffer()), contentType: res.headers.get("content-type") ?? "" };
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
// src/tiers/tier.ts
|
|
2189
|
+
function oaVersion(v) {
|
|
2190
|
+
if (v === "acceptedVersion") return "accepted_manuscript";
|
|
2191
|
+
if (v === "submittedVersion") return "preprint";
|
|
2192
|
+
return "published";
|
|
2193
|
+
}
|
|
2194
|
+
var miss = (tier, reason) => ({ ok: false, tier, reason });
|
|
2195
|
+
var errMsg = (e) => (e instanceof Error ? e.message : String(e)).replace(/\x1b\[[0-9;]*m/g, "").split("\n")[0].trim().slice(0, 200);
|
|
2196
|
+
|
|
2197
|
+
// src/download.ts
|
|
2198
|
+
var hostOf = (url) => {
|
|
2199
|
+
try {
|
|
2200
|
+
return new URL(url).hostname.replace(/^www\./, "");
|
|
2201
|
+
} catch {
|
|
2202
|
+
return url;
|
|
2203
|
+
}
|
|
2204
|
+
};
|
|
2205
|
+
var NCBI = /^https?:\/\/([a-z0-9.-]+\.)?(ncbi\.nlm\.nih\.gov|pubmed\.gov)\//i;
|
|
2206
|
+
function order(candidates) {
|
|
2207
|
+
const rank = (c) => (c.pdf ? 0 : 2) + (c.host === "publisher" ? 1 : 0);
|
|
2208
|
+
return [...candidates].sort((a, b) => rank(a) - rank(b));
|
|
2209
|
+
}
|
|
2210
|
+
async function download(tier, p, candidates, ctx) {
|
|
2211
|
+
const usable = candidates.filter((c) => !NCBI.test(c.url));
|
|
2212
|
+
if (!usable.length && candidates.length) return miss(tier, "only pmc copies (covered by the pmc tier)");
|
|
2213
|
+
const ordered = order(usable).filter((c) => !ctx.tried.has(c.url));
|
|
2214
|
+
if (!ordered.length) return miss(tier, "only copies already tried");
|
|
2215
|
+
const reasons = [];
|
|
2216
|
+
for (const c of ordered) {
|
|
2217
|
+
ctx.tried.add(c.url);
|
|
2218
|
+
let r = await tryOne(p, c, tier);
|
|
2219
|
+
const target = !r.ok && r.url && r.url !== c.url ? { ...c, url: r.url } : c;
|
|
2220
|
+
if (target !== c) ctx.onTry?.(`${c.url} \u2192 ${target.url}`);
|
|
2221
|
+
ctx.onTry?.(r.ok ? `${target.url} ok fetch` : `${target.url} failed fetch: ${r.reason}`);
|
|
2222
|
+
const why = (reason) => reason.replace(/^not a pdf \((.*)\)$/, "$1");
|
|
2223
|
+
for (const b of browserRungs()) {
|
|
2224
|
+
if (r.ok || c.headers || c.service || !b.available || !/^(HTTP 403|HTTP 429|wall page|no methods\/results|unknown content|browser )/.test(why(r.reason))) break;
|
|
2225
|
+
r = await tryOne(p, target, tier, { browser: b.endpoint, label: b.label });
|
|
2226
|
+
ctx.onTry?.(r.ok ? `${target.url} ok browser ${b.label}` : `${target.url} failed browser ${b.label}: ${bare(r.reason)}`);
|
|
2227
|
+
}
|
|
2228
|
+
if (!r.ok && !c.headers && !c.service && unlockerConfigured() && /^(HTTP 403|wall page|browser )/.test(why(r.reason))) {
|
|
2229
|
+
r = await tryOne(p, target, tier, "unlocker");
|
|
2230
|
+
ctx.onTry?.(r.ok ? `${target.url} ok brightdata` : `${target.url} failed brightdata: ${bare(r.reason)}`);
|
|
2231
|
+
}
|
|
2232
|
+
if (r.ok) return r;
|
|
2233
|
+
reasons.push(`${r.reason}: ${hostOf(c.url)}`);
|
|
2234
|
+
}
|
|
2235
|
+
return miss(tier, reasons.join("; "));
|
|
2236
|
+
}
|
|
2237
|
+
var bare = (reason) => reason.replace(/^(brightdata|browser [^:]+): /, "");
|
|
2238
|
+
async function tryOne(p, c, tier, via = "plain") {
|
|
2239
|
+
let got;
|
|
2240
|
+
try {
|
|
2241
|
+
got = via === "unlocker" ? await unlock(c.url) : typeof via === "object" ? await browserFetch(c.url, via.browser) : await fetchBytes(c.url, { ...DOWNLOAD_HEADERS, ...c.headers ?? {} });
|
|
2242
|
+
} catch (e) {
|
|
2243
|
+
const body = e?.body;
|
|
2244
|
+
if (c.service && body?.trim()) return { ok: false, reason: `${errMsg(e)} \u2014 ${serviceMessage(Buffer.from(body), e.contentType)}`, url: e?.url };
|
|
2245
|
+
return { ok: false, reason: (via === "plain" ? "" : via === "unlocker" ? "brightdata: " : `browser ${via.label}: `) + errMsg(e), url: e?.url };
|
|
2246
|
+
}
|
|
2247
|
+
const { bytes, contentType } = got;
|
|
2248
|
+
if (c.service && !looksLikePdf(bytes) && !isXmlArticle(bytes.toString("utf8"))) return { ok: false, reason: `service: ${serviceMessage(bytes, contentType)}`, url: got.url };
|
|
2249
|
+
if (via === "unlocker" && !bytes.length) return { ok: false, reason: "brightdata: empty response" };
|
|
2250
|
+
const r = await convertBytes(p, bytes, contentType);
|
|
2251
|
+
if (!r.ok) return { ok: false, reason: c.pdf && r.format !== "pdf" ? `not a pdf (${r.reason})` : r.reason, url: got.url };
|
|
2252
|
+
return { ok: true, tier, format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: c.version };
|
|
2253
|
+
}
|
|
2254
|
+
var isXmlArticle = (text3) => /^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && !/<html/i.test(text3.slice(0, 2e3));
|
|
2255
|
+
function serviceMessage(bytes, contentType = "") {
|
|
2256
|
+
const text3 = bytes.toString("utf8").trim();
|
|
2257
|
+
if (!text3) return `empty response (${contentType.split(";")[0] || "no type"})`;
|
|
2258
|
+
try {
|
|
2259
|
+
const j = JSON.parse(text3);
|
|
2260
|
+
const m = j?.message ?? j?.error?.message ?? j?.error ?? j?.detail;
|
|
2261
|
+
if (typeof m === "string" && m.trim()) return m.trim().replace(/\s+/g, " ").slice(0, 160);
|
|
2262
|
+
} catch {
|
|
2263
|
+
}
|
|
2264
|
+
const visible = text3.replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<title>([\s\S]*?)<\/title>/i, "$1. ").replace(/<[^>]+>/g, " ").replace(/ /g, " ").replace(/\s+/g, " ").trim();
|
|
2265
|
+
return (visible || text3).slice(0, 160);
|
|
2266
|
+
}
|
|
2267
|
+
function describe(reason, format, md) {
|
|
2268
|
+
const words2 = md.split(/\s+/).filter(Boolean).length;
|
|
2269
|
+
const headings2 = md.split("\n").filter((l) => /^#{2,6}\s/.test(l)).length;
|
|
2270
|
+
const seen = `${format}, ${words2} words, ${proseWords(md)} in prose, ${headings2} heading${headings2 === 1 ? "" : "s"}`;
|
|
2271
|
+
if (reason === "no methods/results") return `no article body (${seen})`;
|
|
2272
|
+
if (reason === "title missing") return `neither title nor DOI in text (${seen})`;
|
|
2273
|
+
return reason;
|
|
2274
|
+
}
|
|
2275
|
+
async function convertBytes(p, bytes, contentType = "") {
|
|
2276
|
+
let format;
|
|
2277
|
+
let md;
|
|
2278
|
+
if (looksLikePdf(bytes)) {
|
|
2279
|
+
format = "pdf";
|
|
2280
|
+
const stats = {};
|
|
2281
|
+
try {
|
|
2282
|
+
md = await pdfToMarkdown(bytes, p.title, p.headLine, stats);
|
|
2283
|
+
} catch (e) {
|
|
2284
|
+
return { ok: false, reason: `pdf: ${errMsg(e)}` };
|
|
2285
|
+
}
|
|
2286
|
+
if ((stats.pages ?? 0) >= 1 && (stats.chars ?? 0) < 500 * (stats.pages ?? 1)) return { ok: false, reason: "scanned pdf", format };
|
|
2287
|
+
} else {
|
|
2288
|
+
const text3 = bytes.toString("utf8");
|
|
2289
|
+
if (isWallPage(text3)) return { ok: false, reason: "wall page" };
|
|
2290
|
+
if (/^\s*<\?xml|<article\b/.test(text3.slice(0, 2e3)) && /<body\b/.test(text3) && !/<html/i.test(text3.slice(0, 2e3))) {
|
|
2291
|
+
format = "xml";
|
|
2292
|
+
const doc = parseJats(/full-text-retrieval-response/.test(text3.slice(0, 3e3)) ? elsevierToJats(text3) : text3);
|
|
2293
|
+
if (!doc.title) doc.title = p.title;
|
|
2294
|
+
if (!doc.hasBody) return { ok: false, reason: "xml without body" };
|
|
2295
|
+
md = renderFulltextMd(doc, p.headLine);
|
|
2296
|
+
} else if (/text\/html|<html/i.test(contentType + text3.slice(0, 500))) {
|
|
2297
|
+
format = "html";
|
|
2298
|
+
const body = /<body[^>]*>([\s\S]*)<\/body>/i.exec(text3)?.[1] ?? text3;
|
|
2299
|
+
md = `# ${p.title}
|
|
2300
|
+
|
|
2301
|
+
${p.headLine}
|
|
2302
|
+
|
|
2303
|
+
${htmlToMarkdown(body)}`;
|
|
2304
|
+
} else {
|
|
2305
|
+
return { ok: false, reason: `unknown content (${contentType.split(";")[0] || "no type"})` };
|
|
2306
|
+
}
|
|
2307
|
+
}
|
|
2308
|
+
const content = format === "xml" ? md : md.replace(/^# [^\n]*\n/, "");
|
|
2309
|
+
const rawText = format === "html" ? bytes.toString("utf8").replace(/<script[\s\S]*?<\/script>|<style[\s\S]*?<\/style>/gi, " ").replace(/<[^>]+>/g, " ") : void 0;
|
|
2310
|
+
const check = checkFulltext(content, p.title, rawText, p.doi, format === "xml");
|
|
2311
|
+
if (!check.ok) return { ok: false, reason: describe(check.reason, format, content), format };
|
|
2312
|
+
return { ok: true, format, md };
|
|
2313
|
+
}
|
|
2314
|
+
var ELSEVIER_TAGS = [
|
|
2315
|
+
[/ja:article\b/g, "article"],
|
|
2316
|
+
[/ja:body\b/g, "body"],
|
|
2317
|
+
[/ja:head\b/g, "front"],
|
|
2318
|
+
[/ce:sections\b/g, "sec-group"],
|
|
2319
|
+
[/ce:section-title\b/g, "title"],
|
|
2320
|
+
[/ce:section\b/g, "sec"],
|
|
2321
|
+
[/ce:simple-para\b/g, "p"],
|
|
2322
|
+
[/ce:para\b/g, "p"],
|
|
2323
|
+
[/ce:abstract-sec\b/g, "sec"],
|
|
2324
|
+
[/ce:abstract\b/g, "abstract"],
|
|
2325
|
+
[/ce:title\b/g, "article-title"],
|
|
2326
|
+
[/ce:italic\b/g, "italic"],
|
|
2327
|
+
[/ce:bold\b/g, "bold"],
|
|
2328
|
+
[/ce:sup\b/g, "sup"],
|
|
2329
|
+
[/ce:inf\b/g, "sub"],
|
|
2330
|
+
[/ce:cross-refs?\b/g, "xref"],
|
|
2331
|
+
[/ce:list-item\b/g, "list-item"],
|
|
2332
|
+
[/ce:list\b/g, "list"],
|
|
2333
|
+
[/ce:label\b/g, "label"],
|
|
2334
|
+
[/ce:caption\b/g, "caption"],
|
|
2335
|
+
[/ce:table\b/g, "table-wrap"],
|
|
2336
|
+
[/ce:figure\b/g, "fig"],
|
|
2337
|
+
[/ce:bib-reference\b/g, "ref"],
|
|
2338
|
+
[/ce:bibliography\b/g, "ref-list"],
|
|
2339
|
+
[/sb:reference\b/g, "mixed-citation"],
|
|
2340
|
+
[/ce:acknowledgment\b/g, "ack"],
|
|
2341
|
+
[/ce:inter-ref\b/g, "ext-link"]
|
|
2342
|
+
];
|
|
2343
|
+
function elsevierToJats(xml) {
|
|
2344
|
+
let out = xml;
|
|
2345
|
+
for (const [re, to] of ELSEVIER_TAGS) out = out.replace(re, to);
|
|
2346
|
+
return out.replace(/<\/?full-text-retrieval-response[^>]*>/g, "").replace(/<\/?originalText[^>]*>/g, "").replace(/<\/?xocs:doc[^>]*>/g, "").replace(/<\/?xocs:serial-item[^>]*>/g, "");
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
// src/tiers/core.ts
|
|
2350
|
+
var WINDOW_MS = 6e4;
|
|
2351
|
+
var PER_WINDOW = 10;
|
|
2352
|
+
var MAX_WAIT_MS = 65e3;
|
|
2353
|
+
var TIMEOUT_MS3 = 1e4;
|
|
2354
|
+
var calls = [];
|
|
2355
|
+
async function pace() {
|
|
2356
|
+
const now = Date.now();
|
|
2357
|
+
while (calls.length && calls[0] <= now - WINDOW_MS) calls.shift();
|
|
2358
|
+
if (calls.length >= PER_WINDOW) await new Promise((r) => setTimeout(r, calls[0] + WINDOW_MS - now + 200));
|
|
2359
|
+
calls.push(Date.now());
|
|
2360
|
+
}
|
|
2361
|
+
async function coreFetch(url, key) {
|
|
2362
|
+
const headers = { "User-Agent": LIBRARY_UA, Authorization: `Bearer ${key}` };
|
|
2363
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
2364
|
+
await pace();
|
|
2365
|
+
const res = await fetch(url, { headers, signal: AbortSignal.timeout(TIMEOUT_MS3), redirect: "follow" });
|
|
2366
|
+
if (res.ok) return await res.text();
|
|
2367
|
+
const body = await res.text().catch(() => "");
|
|
2368
|
+
if (res.status === 429 && attempt === 0) {
|
|
2369
|
+
const until = Date.parse(res.headers.get("x-ratelimit-retry-after") ?? "");
|
|
2370
|
+
const wait = Number.isFinite(until) ? Math.min(Math.max(until - Date.now(), 1e3), MAX_WAIT_MS) : 1e4;
|
|
2371
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
2372
|
+
continue;
|
|
2373
|
+
}
|
|
2374
|
+
throw new Error(res.status < 500 ? httpReason(res.status, body) : `HTTP ${res.status}`);
|
|
2375
|
+
}
|
|
2376
|
+
throw new Error("HTTP 429");
|
|
2377
|
+
}
|
|
2378
|
+
function coreTier(apiKey) {
|
|
2379
|
+
const key = apiKey?.trim();
|
|
2380
|
+
return {
|
|
2381
|
+
name: "core",
|
|
2382
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2383
|
+
async run(p, ctx) {
|
|
2384
|
+
if (!key) return { ...miss(this.name, "needs api key"), skipped: true };
|
|
2385
|
+
let data;
|
|
2386
|
+
try {
|
|
2387
|
+
const q = encodeURIComponent(`doi:"${p.doi}"`);
|
|
2388
|
+
data = JSON.parse(await coreFetch(`https://api.core.ac.uk/v3/search/works/?q=${q}&limit=5`, key));
|
|
2389
|
+
} catch (e) {
|
|
2390
|
+
return miss(this.name, errMsg(e));
|
|
2391
|
+
}
|
|
2392
|
+
const candidates = coreCandidates(data);
|
|
2393
|
+
if (!candidates.length) return miss(this.name, data.results?.length ? "no full text" : "not in core");
|
|
2394
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2395
|
+
return download(this.name, p, candidates, ctx);
|
|
2396
|
+
}
|
|
2397
|
+
};
|
|
2398
|
+
}
|
|
2399
|
+
function coreCandidates(d) {
|
|
2400
|
+
const out = [];
|
|
2401
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2402
|
+
const add = (url, pdf, host) => {
|
|
2403
|
+
if (!url || seen.has(url)) return;
|
|
2404
|
+
seen.add(url);
|
|
2405
|
+
out.push({ url, version: "published", source: "core", host, pdf });
|
|
2406
|
+
};
|
|
2407
|
+
for (const w of d.results ?? []) {
|
|
2408
|
+
add(w.downloadUrl, true, "core");
|
|
2409
|
+
for (const l of w.links ?? []) if (l.type === "download") add(l.url, /\.pdf($|\?)/i.test(l.url ?? ""), "core");
|
|
2410
|
+
for (const u of w.sourceFulltextUrls ?? []) add(u, /\.pdf($|\?)/i.test(u), "repository");
|
|
2411
|
+
}
|
|
2412
|
+
return out;
|
|
2413
|
+
}
|
|
2414
|
+
|
|
2415
|
+
// src/tiers/europepmc.ts
|
|
2416
|
+
var europepmc = {
|
|
2417
|
+
name: "europepmc",
|
|
2418
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
2419
|
+
async run(p, ctx) {
|
|
2420
|
+
let data;
|
|
2421
|
+
try {
|
|
2422
|
+
const q = encodeURIComponent(`EXT_ID:${p.pmid} AND SRC:MED`);
|
|
2423
|
+
data = JSON.parse(await fetchText(`https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${q}&resultType=core&format=json`, { "User-Agent": LIBRARY_UA }));
|
|
2424
|
+
} catch (e) {
|
|
2425
|
+
return miss(this.name, errMsg(e));
|
|
2426
|
+
}
|
|
2427
|
+
const rec = data.resultList?.result?.[0];
|
|
2428
|
+
if (!rec) return miss(this.name, "not in europe pmc");
|
|
2429
|
+
const candidates = europepmcCandidates(rec, p.pmcid);
|
|
2430
|
+
if (!candidates.length) return miss(this.name, "no free copy");
|
|
2431
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2432
|
+
return download(this.name, p, candidates, ctx);
|
|
2433
|
+
}
|
|
2434
|
+
};
|
|
2435
|
+
function europepmcCandidates(r, pmcid) {
|
|
2436
|
+
const out = [];
|
|
2437
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2438
|
+
const add = (url, pdf, host, version) => {
|
|
2439
|
+
if (!url || seen.has(url)) return;
|
|
2440
|
+
seen.add(url);
|
|
2441
|
+
out.push({ url, version, source: "europepmc", host, pdf });
|
|
2442
|
+
};
|
|
2443
|
+
const id = (r.pmcid ?? pmcid ?? "").toUpperCase() || null;
|
|
2444
|
+
const own = r.isOpenAccess === "Y" ? "published" : "accepted_manuscript";
|
|
2445
|
+
if (id && r.inEPMC === "Y") {
|
|
2446
|
+
if (r.isOpenAccess === "Y") add(`https://www.ebi.ac.uk/europepmc/webservices/rest/${id}/fullTextXML`, false, "europepmc", own);
|
|
2447
|
+
if (r.hasPDF === "Y") add(`https://europepmc.org/articles/${id}?pdf=render`, true, "europepmc", own);
|
|
2448
|
+
}
|
|
2449
|
+
for (const u of r.fullTextUrlList?.fullTextUrl ?? []) {
|
|
2450
|
+
if (!u.url || u.site === "DOI" || !/^(F|OA)$/.test(u.availabilityCode ?? "")) continue;
|
|
2451
|
+
if (u.documentStyle === "abs") continue;
|
|
2452
|
+
if (/^https?:\/\/europepmc\.org\/articles\//i.test(u.url) && u.documentStyle === "html") continue;
|
|
2453
|
+
add(u.url, u.documentStyle === "pdf", u.site?.toLowerCase() ?? "unknown", "published");
|
|
2454
|
+
}
|
|
2455
|
+
return out;
|
|
2456
|
+
}
|
|
2457
|
+
|
|
2458
|
+
// src/tiers/inbox.ts
|
|
2459
|
+
import { existsSync as existsSync6, readdirSync as readdirSync2, readFileSync as readFileSync5, unlinkSync } from "fs";
|
|
2460
|
+
import { join as join6 } from "path";
|
|
2461
|
+
var INBOX_DIR = join6(AI4L_LIBRARY_DIR, "inbox");
|
|
2462
|
+
var EXT = { ".pdf": "application/pdf", ".xml": "text/xml", ".html": "text/html", ".htm": "text/html" };
|
|
2463
|
+
function findInboxFile(dir, pmid) {
|
|
2464
|
+
if (!existsSync6(dir)) return null;
|
|
2465
|
+
const re = new RegExp(`(^|[^0-9])${pmid}([^0-9]|$)`);
|
|
2466
|
+
for (const name of readdirSync2(dir).sort()) {
|
|
2467
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
2468
|
+
if (ext in EXT && re.test(name.slice(0, -ext.length))) return join6(dir, name);
|
|
2469
|
+
}
|
|
2470
|
+
return null;
|
|
2471
|
+
}
|
|
2472
|
+
function inboxTier(dir = INBOX_DIR) {
|
|
2473
|
+
return {
|
|
2474
|
+
name: "inbox",
|
|
2475
|
+
always: true,
|
|
2476
|
+
applies: (p) => p.pmid ? true : "no pmid",
|
|
2477
|
+
async run(p, ctx) {
|
|
2478
|
+
const file = findInboxFile(dir, p.pmid);
|
|
2479
|
+
if (!file) return { ...miss("inbox", "no file"), skipped: true, ephemeral: true };
|
|
2480
|
+
const name = file.slice(file.lastIndexOf("/") + 1);
|
|
2481
|
+
const ext = name.slice(name.lastIndexOf(".")).toLowerCase();
|
|
2482
|
+
const bytes = readFileSync5(file);
|
|
2483
|
+
const r = await convertBytes(p, bytes, EXT[ext]);
|
|
2484
|
+
if (!r.ok) return miss("inbox", `${name}: ${r.reason}`);
|
|
2485
|
+
ctx.onTry?.(`${name} ok, removed from the inbox`);
|
|
2486
|
+
unlinkSync(file);
|
|
2487
|
+
return { ok: true, tier: "inbox", format: r.format, raw: r.format === "pdf" ? bytes : bytes.toString("utf8"), md: r.md, version: "published" };
|
|
2488
|
+
}
|
|
2489
|
+
};
|
|
2490
|
+
}
|
|
2491
|
+
var inbox = inboxTier();
|
|
2492
|
+
|
|
2493
|
+
// src/tiers/openalex.ts
|
|
2494
|
+
var openalex = {
|
|
2495
|
+
name: "openalex",
|
|
2496
|
+
applies: (p) => p.pmid || p.doi ? true : "no pmid or doi",
|
|
2497
|
+
async run(p, ctx) {
|
|
2498
|
+
const mail = contactEmail();
|
|
2499
|
+
const key = p.pmid ? `pmid:${p.pmid}` : `doi:${p.doi}`;
|
|
2500
|
+
let data;
|
|
2501
|
+
try {
|
|
2502
|
+
data = JSON.parse(await fetchText(`https://api.openalex.org/works/${encodeURIComponent(key)}?mailto=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
2503
|
+
} catch (e) {
|
|
2504
|
+
return miss(this.name, errMsg(e));
|
|
2505
|
+
}
|
|
2506
|
+
const candidates = openalexCandidates(data);
|
|
2507
|
+
if (!candidates.length) return miss(this.name, data.open_access?.is_oa === false ? "closed" : "no oa copy");
|
|
2508
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2509
|
+
return download(this.name, p, candidates, ctx);
|
|
2510
|
+
}
|
|
2511
|
+
};
|
|
2512
|
+
function openalexCandidates(d) {
|
|
2513
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.locations ?? []].filter((l) => l.is_oa);
|
|
2514
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2515
|
+
const out = [];
|
|
2516
|
+
for (const l of locs) {
|
|
2517
|
+
for (const [url, pdf] of [[l.pdf_url, true], [l.landing_page_url, false]]) {
|
|
2518
|
+
if (!url || seen.has(url) || /doi\.org\//.test(url)) continue;
|
|
2519
|
+
seen.add(url);
|
|
2520
|
+
out.push({ url, version: oaVersion(l.version), source: "openalex", host: l.source?.display_name ?? "unknown", pdf });
|
|
2521
|
+
}
|
|
2522
|
+
}
|
|
2523
|
+
return out;
|
|
2524
|
+
}
|
|
2525
|
+
|
|
2526
|
+
// src/tiers/pmc.ts
|
|
2527
|
+
function pmcEfetchUrl(pmcid) {
|
|
2528
|
+
const u = new URL("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi");
|
|
2529
|
+
u.searchParams.set("db", "pmc");
|
|
2530
|
+
u.searchParams.set("id", pmcid.replace(/^PMC/i, ""));
|
|
2531
|
+
u.searchParams.set("retmode", "xml");
|
|
2532
|
+
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
2533
|
+
u.searchParams.set("tool", "ai4l-library");
|
|
2534
|
+
u.searchParams.set("email", contactEmail());
|
|
2535
|
+
return u.toString();
|
|
2536
|
+
}
|
|
2537
|
+
var pmc = {
|
|
2538
|
+
name: "pmc",
|
|
2539
|
+
applies: (p) => p.pmcid ? true : "no pmcid",
|
|
2540
|
+
async run(p) {
|
|
2541
|
+
let xml;
|
|
2542
|
+
try {
|
|
2543
|
+
xml = await fetchText(pmcEfetchUrl(p.pmcid));
|
|
2544
|
+
} catch (e) {
|
|
2545
|
+
return miss(this.name, errMsg(e));
|
|
2546
|
+
}
|
|
2547
|
+
const doc = parseJats(xml);
|
|
2548
|
+
if (!doc.hasBody) return miss(this.name, "no body");
|
|
2549
|
+
const md = renderFulltextMd(doc, p.headLine);
|
|
2550
|
+
const check = checkFulltext(md, p.title);
|
|
2551
|
+
if (!check.ok) return miss(this.name, check.reason);
|
|
2552
|
+
return { ok: true, tier: this.name, format: "xml", raw: xml, md, version: "published" };
|
|
2553
|
+
}
|
|
2554
|
+
};
|
|
2555
|
+
|
|
2556
|
+
// src/tiers/tdm.ts
|
|
2557
|
+
var CROSSREF = "https://api.crossref.org/works/";
|
|
2558
|
+
var TOKENED = [
|
|
2559
|
+
{ host: /(^|\.)api\.elsevier\.com$/i, key: "elsevier", env: "ELSEVIER_API_KEY", header: "X-ELS-APIKey" },
|
|
2560
|
+
{ host: /(^|\.)api\.wiley\.com$/i, key: "wiley", env: "WILEY_TDM_TOKEN", header: "Wiley-TDM-Client-Token" }
|
|
2561
|
+
];
|
|
2562
|
+
var hostOf2 = (url) => {
|
|
2563
|
+
try {
|
|
2564
|
+
return new URL(url).hostname;
|
|
2565
|
+
} catch {
|
|
2566
|
+
return "";
|
|
2567
|
+
}
|
|
2568
|
+
};
|
|
2569
|
+
function tdmCandidates(work, tokens = {}, env = process.env) {
|
|
2570
|
+
const links = (work.message?.link ?? []).filter((l) => l["intended-application"] === "text-mining" && l.URL);
|
|
2571
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2572
|
+
const candidates = [];
|
|
2573
|
+
const missingTokens = /* @__PURE__ */ new Set();
|
|
2574
|
+
for (const l of links) {
|
|
2575
|
+
const ct = (l["content-type"] ?? "").toLowerCase();
|
|
2576
|
+
if (seen.has(l.URL)) {
|
|
2577
|
+
const prev = candidates.find((c) => c.url === l.URL);
|
|
2578
|
+
if (prev && /pdf/.test(ct)) prev.pdf = true;
|
|
2579
|
+
continue;
|
|
2580
|
+
}
|
|
2581
|
+
seen.add(l.URL);
|
|
2582
|
+
const rule = TOKENED.find((r) => r.host.test(hostOf2(l.URL)));
|
|
2583
|
+
let headers;
|
|
2584
|
+
if (rule) {
|
|
2585
|
+
const token = tokens[rule.key]?.trim() || env[rule.env]?.trim();
|
|
2586
|
+
if (!token) {
|
|
2587
|
+
missingTokens.add(rule.key);
|
|
2588
|
+
continue;
|
|
2589
|
+
}
|
|
2590
|
+
headers = { [rule.header]: token };
|
|
2591
|
+
}
|
|
2592
|
+
if (/text\/plain/.test(ct)) continue;
|
|
2593
|
+
candidates.push({ url: l.URL, version: "published", source: "tdm", host: work.message?.publisher ?? hostOf2(l.URL), pdf: /pdf/.test(ct) || /\.pdf(\?|$)/i.test(l.URL), headers });
|
|
2594
|
+
}
|
|
2595
|
+
const rank = (c) => /\/xml|text\/xml/.test(c.url) ? 0 : c.pdf ? 1 : 2;
|
|
2596
|
+
candidates.sort((a, b) => rank(a) - rank(b));
|
|
2597
|
+
return { candidates, missingTokens: [...missingTokens] };
|
|
2598
|
+
}
|
|
2599
|
+
function tdmTier(tokens = {}) {
|
|
2600
|
+
return {
|
|
2601
|
+
name: "tdm",
|
|
2602
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2603
|
+
async run(p, ctx) {
|
|
2604
|
+
let work;
|
|
2605
|
+
try {
|
|
2606
|
+
work = JSON.parse(await fetchText(CROSSREF + encodeURIComponent(p.doi), { "User-Agent": `${LIBRARY_UA} (mailto:${contactEmail()})` }));
|
|
2607
|
+
} catch (e) {
|
|
2608
|
+
return miss(this.name, errMsg(e));
|
|
2609
|
+
}
|
|
2610
|
+
const { candidates, missingTokens } = tdmCandidates(work, tokens);
|
|
2611
|
+
if (!candidates.length) return missingTokens.length ? { ...miss(this.name, `needs ${missingTokens.join("/")} token`), skipped: true } : miss(this.name, "no text-mining links");
|
|
2612
|
+
ctx.onTry?.(`${candidates.length} text-mining links${missingTokens.length ? ` (skipped hosts needing ${missingTokens.join("/")} token)` : ""}`);
|
|
2613
|
+
return download(this.name, p, candidates, ctx);
|
|
2614
|
+
}
|
|
2615
|
+
};
|
|
2616
|
+
}
|
|
2617
|
+
var tdm = tdmTier();
|
|
2618
|
+
|
|
2619
|
+
// src/tiers/unpaywall.ts
|
|
2620
|
+
var unpaywall = {
|
|
2621
|
+
name: "unpaywall",
|
|
2622
|
+
applies: (p) => p.doi ? true : "no doi",
|
|
2623
|
+
async run(p, ctx) {
|
|
2624
|
+
const mail = contactEmail();
|
|
2625
|
+
let data;
|
|
2626
|
+
try {
|
|
2627
|
+
data = JSON.parse(await fetchText(`https://api.unpaywall.org/v2/${encodeURIComponent(p.doi)}?email=${encodeURIComponent(mail)}`, { "User-Agent": LIBRARY_UA }));
|
|
2628
|
+
} catch (e) {
|
|
2629
|
+
return miss(this.name, errMsg(e));
|
|
2630
|
+
}
|
|
2631
|
+
const candidates = unpaywallCandidates(data);
|
|
2632
|
+
if (!candidates.length) return miss(this.name, data.is_oa === false ? "closed" : "no oa copy");
|
|
2633
|
+
ctx.onTry?.(`located ${candidates.length} copies`);
|
|
2634
|
+
return download(this.name, p, candidates, ctx);
|
|
2635
|
+
}
|
|
2636
|
+
};
|
|
2637
|
+
function unpaywallCandidates(d) {
|
|
2638
|
+
const locs = [...d.best_oa_location ? [d.best_oa_location] : [], ...d.oa_locations ?? []];
|
|
2639
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2640
|
+
const out = [];
|
|
2641
|
+
for (const l of locs) {
|
|
2642
|
+
for (const [url, pdf] of [[l.url_for_pdf, true], [l.url ?? l.url_for_landing_page, false]]) {
|
|
2643
|
+
if (!url || seen.has(url)) continue;
|
|
2644
|
+
seen.add(url);
|
|
2645
|
+
out.push({ url, version: oaVersion(l.version), source: "unpaywall", host: l.host_type ?? "unknown", pdf });
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
return out;
|
|
2649
|
+
}
|
|
2650
|
+
|
|
2651
|
+
// src/tiers/index.ts
|
|
2652
|
+
var BUILTIN = [
|
|
2653
|
+
{ name: "inbox", make: () => inbox },
|
|
2654
|
+
{ name: "pmc", make: () => pmc },
|
|
2655
|
+
{ name: "europepmc", make: () => europepmc },
|
|
2656
|
+
{ name: "tdm", make: (o) => tdmTier(o.tokens ?? {}) },
|
|
2657
|
+
{ name: "unpaywall", make: () => unpaywall },
|
|
2658
|
+
{ name: "openalex", make: () => openalex },
|
|
2659
|
+
{ name: "core", make: (o) => coreTier(o.api_key) }
|
|
2660
|
+
];
|
|
2661
|
+
|
|
2662
|
+
// src/tiers/url.ts
|
|
2663
|
+
var PLACEHOLDERS = ["doi", "pmid", "pmcid"];
|
|
2664
|
+
function urlTier(name, template, headers) {
|
|
2665
|
+
const used = PLACEHOLDERS.filter((k) => template.includes(`{${k}}`));
|
|
2666
|
+
return {
|
|
2667
|
+
name,
|
|
2668
|
+
applies(p) {
|
|
2669
|
+
const missing = used.filter((k) => !p[k]);
|
|
2670
|
+
return missing.length ? `no ${missing.join("/")}` : true;
|
|
2671
|
+
},
|
|
2672
|
+
async run(p, ctx) {
|
|
2673
|
+
if (!used.length) return miss(name, "template has no placeholder");
|
|
2674
|
+
const url = used.reduce((u, k) => u.replaceAll(`{${k}}`, encodeURIComponent(p[k])), template);
|
|
2675
|
+
return download(name, p, [{ url, version: "published", source: name, host: "user", pdf: /\.pdf(\?|$)/i.test(url), headers, service: true }], ctx);
|
|
2676
|
+
}
|
|
2677
|
+
};
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
// src/config.ts
|
|
2681
|
+
var CONFIG_FILE = process.env.AI4L_LIBRARY_CONFIG?.trim() || join7(AI4L_LIBRARY_DIR, "config.json");
|
|
2682
|
+
function resolveEnv(v, env = process.env) {
|
|
2683
|
+
return v.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, n) => env[n] ?? "");
|
|
2684
|
+
}
|
|
2685
|
+
var resolveMap = (m) => m ? Object.fromEntries(Object.entries(m).map(([k, v]) => [k, resolveEnv(String(v))])) : void 0;
|
|
2686
|
+
function settings(file = CONFIG_FILE) {
|
|
2687
|
+
const out = { retry_after_days: 1, retry_blocked: false, refresh_after_days: 30 };
|
|
2688
|
+
if (!existsSync7(file)) return out;
|
|
2689
|
+
try {
|
|
2690
|
+
const cfg = JSON.parse(readFileSync6(file, "utf8"));
|
|
2691
|
+
if (typeof cfg.retry_after_days === "number" && cfg.retry_after_days >= 0) out.retry_after_days = cfg.retry_after_days;
|
|
2692
|
+
if (typeof cfg.retry_blocked === "boolean") out.retry_blocked = cfg.retry_blocked;
|
|
2693
|
+
if (typeof cfg.refresh_after_days === "number" && cfg.refresh_after_days >= 0) out.refresh_after_days = cfg.refresh_after_days;
|
|
2694
|
+
} catch {
|
|
2695
|
+
}
|
|
2696
|
+
return out;
|
|
2697
|
+
}
|
|
2698
|
+
function resolveTiers(env = process.env.AI4L_LIBRARY_TIERS, file = CONFIG_FILE) {
|
|
2699
|
+
const problems = [];
|
|
2700
|
+
const builtin = new Map(BUILTIN.map((t) => [t.name, t]));
|
|
2701
|
+
const known = /* @__PURE__ */ new Map();
|
|
2702
|
+
let source = "default";
|
|
2703
|
+
for (const t of BUILTIN) known.set(t.name, { tier: t.make({}), enabled: true, kind: "builtin" });
|
|
2704
|
+
if (existsSync7(file)) {
|
|
2705
|
+
source = "file";
|
|
2706
|
+
let cfg = {};
|
|
2707
|
+
try {
|
|
2708
|
+
cfg = JSON.parse(readFileSync6(file, "utf8"));
|
|
2709
|
+
} catch (e) {
|
|
2710
|
+
problems.push(`${file}: ${e instanceof Error ? e.message : String(e)}`);
|
|
2711
|
+
}
|
|
2712
|
+
if (Array.isArray(cfg.tiers)) {
|
|
2713
|
+
for (const k of known.keys()) known.get(k).enabled = false;
|
|
2714
|
+
const ordered = /* @__PURE__ */ new Map();
|
|
2715
|
+
for (const e of cfg.tiers) {
|
|
2716
|
+
if (!e || typeof e.name !== "string" || !e.name.trim()) {
|
|
2717
|
+
problems.push(`${file}: tier entry without name`);
|
|
2718
|
+
continue;
|
|
2719
|
+
}
|
|
2720
|
+
const enabled = e.enabled !== false;
|
|
2721
|
+
const description = typeof e.description === "string" ? e.description : void 0;
|
|
2722
|
+
if (typeof e.url === "string" && e.url.trim()) ordered.set(e.name, { tier: urlTier(e.name, resolveEnv(e.url.trim()), resolveMap(e.headers)), enabled, kind: "url", description });
|
|
2723
|
+
else if (builtin.has(e.name)) ordered.set(e.name, { tier: builtin.get(e.name).make({ tokens: resolveMap(e.tokens), api_key: typeof e.api_key === "string" ? resolveEnv(e.api_key) : void 0 }), enabled, kind: "builtin", description });
|
|
2724
|
+
else problems.push(`${file}: unknown tier "${e.name}" (built-in: ${[...builtin.keys()].join(", ")}; a user tier needs "url")`);
|
|
2725
|
+
}
|
|
2726
|
+
for (const [k, v] of known) if (!ordered.has(k)) ordered.set(k, v);
|
|
2727
|
+
known.clear();
|
|
2728
|
+
for (const [k, v] of ordered) known.set(k, v);
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
if (env && env.trim()) {
|
|
2732
|
+
source = "env";
|
|
2733
|
+
for (const v of known.values()) v.enabled = false;
|
|
2734
|
+
const ordered = /* @__PURE__ */ new Map();
|
|
2735
|
+
for (const name of env.split(",").map((s) => s.trim()).filter(Boolean)) {
|
|
2736
|
+
const v = known.get(name);
|
|
2737
|
+
if (!v) {
|
|
2738
|
+
problems.push(`AI4L_LIBRARY_TIERS: unknown tier "${name}"`);
|
|
2739
|
+
continue;
|
|
2740
|
+
}
|
|
2741
|
+
ordered.set(name, { ...v, enabled: true });
|
|
2742
|
+
}
|
|
2743
|
+
for (const [k, v] of known) if (!ordered.has(k)) ordered.set(k, v);
|
|
2744
|
+
known.clear();
|
|
2745
|
+
for (const [k, v] of ordered) known.set(k, v);
|
|
2746
|
+
}
|
|
2747
|
+
const all = [...known.entries()].map(([name, v]) => ({ name, enabled: v.enabled, description: v.description ?? (v.kind === "url" ? "url tier" : "built-in") }));
|
|
2748
|
+
return { tiers: [...known.values()].filter((v) => v.enabled).map((v) => v.tier), all, source, file, problems };
|
|
2749
|
+
}
|
|
2750
|
+
|
|
2751
|
+
// src/cascade.ts
|
|
2752
|
+
var PERMANENT = /^(no pmcid|no doi|no pmid|no pmid or doi|no body|closed|no oa copy|no free copy|not in core|no full text|xml without body|template has no placeholder|title missing|title not in text|no methods\/results|no article body|scanned pdf|not a pdf|only copies already tried|only pmc copies|nothing located|unknown content|no text-mining links|needs [a-z/]+ token)/i;
|
|
2753
|
+
var BLOCKED = /^(challenge not cleared|empty response)/i;
|
|
2754
|
+
var RUNG = /^(browser [^:]+|brightdata|unlocker): /;
|
|
2755
|
+
function isPermanent(reason, retryBlocked = false) {
|
|
2756
|
+
return reason.split(";").every((r) => {
|
|
2757
|
+
const b = r.trim().replace(RUNG, "");
|
|
2758
|
+
return PERMANENT.test(b) || !retryBlocked && BLOCKED.test(b);
|
|
2759
|
+
});
|
|
2760
|
+
}
|
|
2761
|
+
function parseAttempt(a) {
|
|
2762
|
+
const m = /^(\S+) \((.*)\)(?: (\d{4}-\d{2}-\d{2}))?$/.exec(a.trim());
|
|
2763
|
+
return m ? { tier: m[1], reason: m[2], date: m[3] ?? null } : null;
|
|
2764
|
+
}
|
|
2765
|
+
var attemptLabel = (m, date = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10)) => `${m.tier} (${m.reason}) ${date}`;
|
|
2766
|
+
function shouldSkip(attempts, tier, retryAfterDays, now = /* @__PURE__ */ new Date(), retryBlocked = false) {
|
|
2767
|
+
for (const a of attempts) {
|
|
2768
|
+
const p = parseAttempt(a);
|
|
2769
|
+
if (!p || p.tier !== tier) continue;
|
|
2770
|
+
if (isPermanent(p.reason, retryBlocked)) return true;
|
|
2771
|
+
if (!p.date) return false;
|
|
2772
|
+
const age = (now.getTime() - new Date(p.date).getTime()) / 864e5;
|
|
2773
|
+
return age < retryAfterDays;
|
|
2774
|
+
}
|
|
2775
|
+
return false;
|
|
2776
|
+
}
|
|
2777
|
+
async function fulltextCascade(p, attempts, onEvent, tiers = resolveTiers().tiers, retryAfterDays = settings().retry_after_days, retryBlocked = settings().retry_blocked) {
|
|
2778
|
+
const misses = [];
|
|
2779
|
+
const tried = /* @__PURE__ */ new Set();
|
|
2780
|
+
for (const tier of tiers) {
|
|
2781
|
+
if (!tier.always && shouldSkip(attempts, tier.name, retryAfterDays, /* @__PURE__ */ new Date(), retryBlocked)) continue;
|
|
2782
|
+
const applies = tier.applies(p);
|
|
2783
|
+
const ctx = { tried, onTry: (message) => onEvent?.({ ok: "try", tier: tier.name, message }) };
|
|
2784
|
+
const r = applies === true ? await tier.run(p, ctx) : { ...miss(tier.name, applies), skipped: true };
|
|
2785
|
+
onEvent?.(r);
|
|
2786
|
+
if (r.ok) return { hit: r, misses };
|
|
2787
|
+
misses.push(r);
|
|
2788
|
+
}
|
|
2789
|
+
return { hit: null, misses };
|
|
2790
|
+
}
|
|
2791
|
+
|
|
2792
|
+
// src/ids.ts
|
|
2793
|
+
var SUPPORTED = /* @__PURE__ */ new Set(["pmid", "dailymed", "nct", "doi"]);
|
|
2794
|
+
var KNOWN = /* @__PURE__ */ new Set(["pmid", "pmc", "doi", "nct", "dailymed"]);
|
|
2795
|
+
var UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
2796
|
+
function parseId(raw) {
|
|
2797
|
+
const requested = String(raw ?? "").trim();
|
|
2798
|
+
const bare2 = /^\d+$/.test(requested);
|
|
2799
|
+
const bareNct = /^NCT\d{8}$/i.test(requested);
|
|
2800
|
+
const m = bare2 || bareNct ? null : /^([A-Za-z][A-Za-z0-9_-]*):(.+)$/.exec(requested);
|
|
2801
|
+
if (!bare2 && !bareNct && !m) return { requested, type: null, id: null, error: "invalid id" };
|
|
2802
|
+
const type = bare2 ? "pmid" : bareNct ? "nct" : m[1].toLowerCase();
|
|
2803
|
+
const value = bare2 || bareNct ? requested : m[2].trim();
|
|
2804
|
+
if (!KNOWN.has(type)) return { requested, type, id: value, error: "invalid id" };
|
|
2805
|
+
if (!SUPPORTED.has(type)) return { requested, type, id: value, error: "type not supported yet" };
|
|
2806
|
+
if (type === "doi") {
|
|
2807
|
+
const v = value.replace(/^https?:\/\/(dx\.)?doi\.org\//i, "").replace(/^doi:\s*/i, "").trim().toLowerCase();
|
|
2808
|
+
return /^10\.\d{4,9}\/\S+$/.test(v) ? { requested, type, id: v } : { requested, type, id: value, error: "invalid id" };
|
|
2809
|
+
}
|
|
2810
|
+
if (type === "nct") {
|
|
2811
|
+
const v = value.toUpperCase().replace(/^NCT/, "");
|
|
2812
|
+
return /^\d{8}$/.test(v) ? { requested, type, id: v } : { requested, type, id: value, error: "invalid id" };
|
|
2813
|
+
}
|
|
2814
|
+
if (type === "dailymed") {
|
|
2815
|
+
const v = value.toLowerCase();
|
|
2816
|
+
return UUID.test(v) ? { requested, type, id: v } : { requested, type, id: value, error: "invalid id" };
|
|
2817
|
+
}
|
|
2818
|
+
if (!/^[1-9]\d*$/.test(value)) return { requested, type, id: value, error: "invalid id" };
|
|
2819
|
+
return { requested, type, id: value };
|
|
2820
|
+
}
|
|
2821
|
+
|
|
2822
|
+
// src/info.ts
|
|
2823
|
+
import { existsSync as existsSync9, readdirSync as readdirSync3, readFileSync as readFileSync8, statSync } from "fs";
|
|
2824
|
+
import { dirname as dirname2, join as join8 } from "path";
|
|
2825
|
+
|
|
2826
|
+
// src/verify.ts
|
|
2827
|
+
import { existsSync as existsSync8, readFileSync as readFileSync7 } from "fs";
|
|
2828
|
+
var DEFAULT_MODEL = "pplx-decider-flash-v1.2";
|
|
2829
|
+
var QUESTION = "Does the paper state this, in substance? Ignore section labels and definitions of terms in parentheses. Every fact and number of the statement must be stated in the paper or follow from its numbers; a rounded number counts. If any part of the statement is absent from the paper or contradicted by it, answer false.";
|
|
2830
|
+
function verifyConfig(file = CONFIG_FILE) {
|
|
2831
|
+
if (!existsSync8(file)) return null;
|
|
2832
|
+
try {
|
|
2833
|
+
const v = JSON.parse(readFileSync7(file, "utf8")).verify;
|
|
2834
|
+
if (!v || typeof v.base_url !== "string") return null;
|
|
2835
|
+
const base_url = resolveEnv(v.base_url).trim().replace(/\/+$/, "");
|
|
2836
|
+
if (!base_url) return null;
|
|
2837
|
+
const key = typeof v.api_key === "string" ? resolveEnv(v.api_key).trim() : "";
|
|
2838
|
+
return {
|
|
2839
|
+
base_url,
|
|
2840
|
+
api_key: key || null,
|
|
2841
|
+
model: typeof v.model === "string" && resolveEnv(v.model).trim() ? resolveEnv(v.model).trim() : DEFAULT_MODEL,
|
|
2842
|
+
threshold: typeof v.threshold === "number" && v.threshold > 0 && v.threshold < 1 ? v.threshold : 0.5,
|
|
2843
|
+
// Hosted takes 256k tokens in one request (a 50k-token paper in 2 s); the
|
|
2844
|
+
// Mac app is limited to 32k and slows steeply beyond 25k, so it windows.
|
|
2845
|
+
window_chars: typeof v.window_chars === "number" && v.window_chars >= 1e4 ? v.window_chars : isHosted(base_url) ? 8e5 : 9e4,
|
|
2846
|
+
timeout_seconds: typeof v.timeout_seconds === "number" && v.timeout_seconds > 0 ? v.timeout_seconds : 60
|
|
2847
|
+
};
|
|
2848
|
+
} catch {
|
|
2849
|
+
return null;
|
|
2850
|
+
}
|
|
2851
|
+
}
|
|
2852
|
+
var isHosted = (baseUrl) => {
|
|
2853
|
+
try {
|
|
2854
|
+
return new URL(baseUrl).hostname === "api.perplexity.ai";
|
|
2855
|
+
} catch {
|
|
2856
|
+
return false;
|
|
2857
|
+
}
|
|
2858
|
+
};
|
|
2859
|
+
function isLocal(baseUrl) {
|
|
2860
|
+
try {
|
|
2861
|
+
const h = new URL(baseUrl).hostname;
|
|
2862
|
+
return h === "localhost" || /^127\./.test(h) || h === "::1" || /^10\./.test(h) || /^192\.168\./.test(h) || /^172\.(1[6-9]|2\d|3[01])\./.test(h) || /^100\.(6[4-9]|[7-9]\d|1[01]\d|12[0-7])\./.test(h);
|
|
2863
|
+
} catch {
|
|
2864
|
+
return false;
|
|
2865
|
+
}
|
|
2866
|
+
}
|
|
2867
|
+
function cleanMarkdown(md) {
|
|
2868
|
+
return md.replace(/!\[[^\]]*\]\([^)]*\)/g, "").replace(/\]\([^)]*\)/g, "]").replace(/^\s*#+\s*(see more about|more for you|back to top)\b.*$/gim, "").replace(/\n{3,}/g, "\n\n");
|
|
2869
|
+
}
|
|
2870
|
+
function windows(md, max) {
|
|
2871
|
+
const out = [];
|
|
2872
|
+
let cur = "";
|
|
2873
|
+
for (const s of cleanMarkdown(md).split(/\n(?=#)/)) {
|
|
2874
|
+
if (cur && cur.length + s.length + 1 > max) {
|
|
2875
|
+
out.push(cur);
|
|
2876
|
+
cur = "";
|
|
2877
|
+
}
|
|
2878
|
+
cur += (cur ? "\n" : "") + s.slice(0, max);
|
|
2879
|
+
}
|
|
2880
|
+
if (cur) out.push(cur);
|
|
2881
|
+
return out.length ? out : [""];
|
|
2882
|
+
}
|
|
2883
|
+
function citationGroups(claim) {
|
|
2884
|
+
const one = /[(\[][^()\[\]]*?\b(1[89]\d\d|20\d\d)[a-z]?\b[^()\[\]]*[)\]]/g;
|
|
2885
|
+
const groups = [];
|
|
2886
|
+
let last = -1;
|
|
2887
|
+
for (const m of claim.matchAll(one)) {
|
|
2888
|
+
const between = claim.slice(last, m.index);
|
|
2889
|
+
if (last >= 0 && /^[\s;,]*(and|&)?[\s;,]*$/i.test(between)) {
|
|
2890
|
+
groups[groups.length - 1] += between + m[0];
|
|
2891
|
+
} else groups.push(m[0]);
|
|
2892
|
+
last = m.index + m[0].length;
|
|
2893
|
+
}
|
|
2894
|
+
return groups;
|
|
2895
|
+
}
|
|
2896
|
+
function fragment(claim) {
|
|
2897
|
+
const body = claim.trim().replace(/^[-*+]\s+/, "").replace(/^\*\*[^*]+:\*\*\s*/, "").replace(/^[("'\u201c]+/, "");
|
|
2898
|
+
const first = /^(\S+)/.exec(body)?.[1] ?? "";
|
|
2899
|
+
if (/^(and|or|but|nor|which|whereas|while)$/i.test(first.replace(/[,;:]$/, ""))) return `claim is a fragment: starts with "${first}" \u2014 send a complete statement with its subject`;
|
|
2900
|
+
if (/^[a-z]/.test(body)) return `claim is a fragment: starts with "${first}" in lowercase \u2014 send a complete statement with its subject`;
|
|
2901
|
+
return null;
|
|
2902
|
+
}
|
|
2903
|
+
var SUPERSEDED = "The corrected text replaces figures that the paper first published wrongly. Does the statement give a figure for a quantity the corrected text also gives, but with a different value \u2014 the superseded, uncorrected figure? If the statement's figures match the corrected text, or it concerns something the corrected text does not cover, answer false.";
|
|
2904
|
+
function correctedText(md) {
|
|
2905
|
+
const i = md.indexOf(`
|
|
2906
|
+
${CORRECTED_HEADING}
|
|
2907
|
+
`);
|
|
2908
|
+
if (i < 0) return null;
|
|
2909
|
+
return md.slice(i + CORRECTED_HEADING.length + 2).split("\n").filter((l) => l.trim() && !/^NLM corrected these sentences/.test(l)).join("\n").trim() || null;
|
|
2910
|
+
}
|
|
2911
|
+
function decider(cfg, fetchFn = fetch) {
|
|
2912
|
+
return async (state, questions) => {
|
|
2913
|
+
const headers = { "content-type": "application/json" };
|
|
2914
|
+
if (cfg.api_key) headers.authorization = `Bearer ${cfg.api_key}`;
|
|
2915
|
+
const body = JSON.stringify({ model: cfg.model, state, questions });
|
|
2916
|
+
let r;
|
|
2917
|
+
try {
|
|
2918
|
+
r = await fetchFn(`${cfg.base_url}/v1/decisions`, { method: "POST", headers, body, signal: AbortSignal.timeout(cfg.timeout_seconds * 1e3) });
|
|
2919
|
+
} catch (e) {
|
|
2920
|
+
const name = e instanceof Error ? e.name : "";
|
|
2921
|
+
const cause = e.cause;
|
|
2922
|
+
if (name === "TimeoutError" || name === "AbortError" || cause?.code === "UND_ERR_HEADERS_TIMEOUT") throw new Error(`${cfg.model}: no answer within ${cfg.timeout_seconds}s (${body.length} chars)`);
|
|
2923
|
+
throw new Error(`${cfg.model}: ${cause?.message ?? (e instanceof Error ? e.message : String(e))}`);
|
|
2924
|
+
}
|
|
2925
|
+
const j = await r.json().catch(() => ({}));
|
|
2926
|
+
if (j.answers) return j.answers;
|
|
2927
|
+
const msg = typeof j.error === "string" ? j.error : j.error?.message;
|
|
2928
|
+
throw new Error(msg ? `${cfg.model}: ${msg}` : `${cfg.model}: HTTP ${r.status} without an answer`);
|
|
2929
|
+
};
|
|
2930
|
+
}
|
|
2931
|
+
async function readPaper(raw, libraryDir = AI4L_LIBRARY_DIR) {
|
|
2932
|
+
const p = parseId(raw);
|
|
2933
|
+
if (p.error) return { id: p.requested, error: p.error };
|
|
2934
|
+
const paths = entryPaths(libraryDir, p.type, p.id);
|
|
2935
|
+
if (!entryExists(paths)) return { id: p.requested, error: "not in library" };
|
|
2936
|
+
const full = existsSync8(paths.fulltextMd);
|
|
2937
|
+
if (full) return { id: p.requested, basis: "full_text", text: readFileSync7(paths.fulltextMd, "utf8") };
|
|
2938
|
+
const meta = readMeta(paths);
|
|
2939
|
+
return { id: p.requested, basis: meta.has_abstract ? "abstract" : "title", text: readFileSync7(paths.md, "utf8") };
|
|
2940
|
+
}
|
|
2941
|
+
async function verifyClaim(claim, id, cfg, decide, read = readPaper) {
|
|
2942
|
+
const out = { claim, id, verdict: "error", score: null, basis: null, windows: 0, glyphs_unresolved: 0, superseded: null, model: cfg?.model ?? null };
|
|
2943
|
+
const words2 = claim.trim().split(/\s+/).filter(Boolean).length;
|
|
2944
|
+
if (!words2) return { ...out, error: "claim empty" };
|
|
2945
|
+
if (words2 < 4) return { ...out, error: "claim too short to be a finding" };
|
|
2946
|
+
const frag = fragment(claim);
|
|
2947
|
+
if (frag) return { ...out, error: frag };
|
|
2948
|
+
const groups = citationGroups(claim);
|
|
2949
|
+
if (groups.length) return { ...out, error: groups.length > 1 ? `compound claim: ${groups.length} citation groups ${groups.join(" ")} \u2014 one finding per call, without its sources` : `claim names a source ${groups[0]} \u2014 send the finding without its sources; the paper is the id` };
|
|
2950
|
+
const paper = await read(id);
|
|
2951
|
+
out.id = paper.id;
|
|
2952
|
+
if ("error" in paper) return { ...out, error: paper.error };
|
|
2953
|
+
if (!cfg || !decide) return { ...out, error: "no verify model configured (config.json: verify)" };
|
|
2954
|
+
out.basis = paper.basis;
|
|
2955
|
+
out.glyphs_unresolved = countUnresolved(paper.text);
|
|
2956
|
+
if (paper.basis === "title") return { ...out, verdict: "na" };
|
|
2957
|
+
const ws = windows(paper.text, cfg.window_chars);
|
|
2958
|
+
out.windows = ws.length;
|
|
2959
|
+
try {
|
|
2960
|
+
let score = 0;
|
|
2961
|
+
for (const w of ws) {
|
|
2962
|
+
const a = await decide({ paper: w, statement: claim }, { backed: { type: "noul", instructions: QUESTION } });
|
|
2963
|
+
score = Math.max(score, a.backed?.noul ?? 0);
|
|
2964
|
+
}
|
|
2965
|
+
out.score = score;
|
|
2966
|
+
const corrected = correctedText(paper.text);
|
|
2967
|
+
if (corrected) {
|
|
2968
|
+
const a = await decide({ corrected, statement: claim }, { superseded: { type: "noul", instructions: SUPERSEDED } });
|
|
2969
|
+
out.superseded = (a.superseded?.noul ?? 0) >= 0.5;
|
|
2970
|
+
}
|
|
2971
|
+
} catch (e) {
|
|
2972
|
+
return { ...out, error: e instanceof Error ? e.message : String(e) };
|
|
2973
|
+
}
|
|
2974
|
+
if (out.score >= cfg.threshold) return { ...out, verdict: "pass" };
|
|
2975
|
+
return { ...out, verdict: paper.basis === "full_text" ? "fail" : "na" };
|
|
2976
|
+
}
|
|
2977
|
+
|
|
2978
|
+
// src/info.ts
|
|
2979
|
+
function* metaFiles(dir) {
|
|
2980
|
+
if (!existsSync9(dir)) return;
|
|
2981
|
+
for (const name of readdirSync3(dir)) {
|
|
2982
|
+
const p = join8(dir, name);
|
|
2983
|
+
if (statSync(p).isDirectory()) yield* metaFiles(p);
|
|
2984
|
+
else if (name.endsWith("-meta.md")) yield p;
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
2987
|
+
var human = (n) => {
|
|
2988
|
+
const u = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
2989
|
+
let i = 0;
|
|
2990
|
+
while (n >= 1024 && i < u.length - 1) {
|
|
2991
|
+
n /= 1024;
|
|
2992
|
+
i++;
|
|
2993
|
+
}
|
|
2994
|
+
return i ? `${n.toFixed(1)} ${u[i]}` : `${n} B`;
|
|
2995
|
+
};
|
|
2996
|
+
function field(md, key) {
|
|
2997
|
+
const m = new RegExp(`^${key}:\\s*(.*)$`, "m").exec(md);
|
|
2998
|
+
return m ? m[1].trim() : "";
|
|
2999
|
+
}
|
|
3000
|
+
function libraryInfo(libraryDir, host, withIds, tiers = resolveTiers()) {
|
|
3001
|
+
const info = {
|
|
3002
|
+
library_dir: libraryDir,
|
|
3003
|
+
host,
|
|
3004
|
+
tiers: tiers.all,
|
|
3005
|
+
tiers_config: { source: tiers.source, file: tiers.file, problems: tiers.problems, retry_after_days: settings().retry_after_days, retry_blocked: settings().retry_blocked },
|
|
3006
|
+
download: downloadInfo(),
|
|
3007
|
+
verify: verifyInfo(),
|
|
3008
|
+
entries: 0,
|
|
3009
|
+
size_bytes: 0,
|
|
3010
|
+
size: "",
|
|
3011
|
+
by_type: {},
|
|
3012
|
+
by_level: {},
|
|
3013
|
+
types: {},
|
|
3014
|
+
retracted: 0,
|
|
3015
|
+
not_citable: 0,
|
|
3016
|
+
errata: 0,
|
|
3017
|
+
concerns: 0,
|
|
3018
|
+
updates: 0
|
|
3019
|
+
};
|
|
3020
|
+
const ids = [];
|
|
3021
|
+
for (const type of existsSync9(libraryDir) ? readdirSync3(libraryDir) : []) {
|
|
3022
|
+
const typeDir = join8(libraryDir, type);
|
|
3023
|
+
if (!statSync(typeDir).isDirectory()) continue;
|
|
3024
|
+
for (const meta of metaFiles(typeDir)) {
|
|
3025
|
+
const md = readFileSync8(meta, "utf8");
|
|
3026
|
+
const raw = field(md, "level") || "unknown";
|
|
3027
|
+
const level = raw === "abstract" && field(md, "has_abstract") === "no" ? "title" : raw;
|
|
3028
|
+
info.entries++;
|
|
3029
|
+
info.by_type[type] = (info.by_type[type] ?? 0) + 1;
|
|
3030
|
+
info.by_level[level] = (info.by_level[level] ?? 0) + 1;
|
|
3031
|
+
const t = info.types[type] ??= { entries: 0, full_text: 0, abstract: 0, title: 0, retracted: 0 };
|
|
3032
|
+
t.entries++;
|
|
3033
|
+
if (level === "full_text" || level === "abstract" || level === "title") t[level]++;
|
|
3034
|
+
const dir = dirname2(meta);
|
|
3035
|
+
for (const f of readdirSync3(dir)) {
|
|
3036
|
+
try {
|
|
3037
|
+
const st = statSync(join8(dir, f));
|
|
3038
|
+
if (st.isFile()) info.size_bytes += st.size;
|
|
3039
|
+
} catch {
|
|
3040
|
+
}
|
|
3041
|
+
}
|
|
3042
|
+
const types = noticeTypes(md, meta);
|
|
3043
|
+
if (field(md, "retracted") === "yes" || types.has("retraction_in") || types.has("partial_retraction_in") || types.has("retracted_and_republished_in")) {
|
|
3044
|
+
info.retracted++;
|
|
3045
|
+
info.not_citable++;
|
|
3046
|
+
t.retracted++;
|
|
3047
|
+
}
|
|
3048
|
+
if (types.has("erratum_in")) info.errata++;
|
|
3049
|
+
if (types.has("expression_of_concern_in")) info.concerns++;
|
|
3050
|
+
if (types.has("update_in")) info.updates++;
|
|
3051
|
+
if (withIds) ids.push(`${type}:${field(md, type) || meta.split("/").slice(-2, -1)[0]}`);
|
|
3052
|
+
}
|
|
3053
|
+
}
|
|
3054
|
+
if (withIds) info.ids = ids.sort();
|
|
3055
|
+
info.size = human(info.size_bytes);
|
|
3056
|
+
return info;
|
|
3057
|
+
}
|
|
3058
|
+
var XML_TYPES = { ErratumIn: "erratum_in", RetractionIn: "retraction_in", PartialRetractionIn: "partial_retraction_in", RetractedandRepublishedIn: "retracted_and_republished_in", ExpressionOfConcernIn: "expression_of_concern_in", UpdateIn: "update_in" };
|
|
3059
|
+
function noticeTypes(md, metaPath) {
|
|
3060
|
+
const out = /* @__PURE__ */ new Set();
|
|
3061
|
+
if (/^pubmed_checked:/m.test(md)) {
|
|
3062
|
+
for (const m of md.matchAll(/^corrections:\s*(\S+)/gm)) out.add(m[1]);
|
|
3063
|
+
return out;
|
|
3064
|
+
}
|
|
3065
|
+
try {
|
|
3066
|
+
const xml = readFileSync8(metaPath.replace(/-meta\.md$/, "-abstract.xml"), "utf8");
|
|
3067
|
+
for (const m of xml.matchAll(/<CommentsCorrections RefType="([A-Za-z]+)"/g)) if (XML_TYPES[m[1]]) out.add(XML_TYPES[m[1]]);
|
|
3068
|
+
} catch {
|
|
3069
|
+
}
|
|
3070
|
+
return out;
|
|
3071
|
+
}
|
|
3072
|
+
function verifyInfo() {
|
|
3073
|
+
const v = verifyConfig();
|
|
3074
|
+
return v ? { model: v.model, base_url: v.base_url, local: isLocal(v.base_url) } : null;
|
|
3075
|
+
}
|
|
3076
|
+
function downloadInfo() {
|
|
3077
|
+
const browsers = browserRungs().map((b) => b.available ? b : { ...b, note: `not installed: ${INSTALL_HINT}` });
|
|
3078
|
+
return { fetch: true, browsers, unlocker: unlockerConfigured() };
|
|
3079
|
+
}
|
|
3080
|
+
|
|
3081
|
+
// src/read.ts
|
|
3082
|
+
function sections(md) {
|
|
3083
|
+
const lines = md.split("\n");
|
|
3084
|
+
const out = [];
|
|
3085
|
+
lines.forEach((l, i) => {
|
|
3086
|
+
const m = /^(#{1,6})\s+(.*\S)\s*$/.exec(l);
|
|
3087
|
+
if (m) out.push({ level: m[1].length, heading: m[2], start: i, end: lines.length, words: 0 });
|
|
3088
|
+
});
|
|
3089
|
+
for (let i = 0; i < out.length; i++) {
|
|
3090
|
+
for (let j = i + 1; j < out.length; j++) {
|
|
3091
|
+
if (out[j].level <= out[i].level) {
|
|
3092
|
+
out[i].end = out[j].start;
|
|
3093
|
+
break;
|
|
3094
|
+
}
|
|
3095
|
+
}
|
|
3096
|
+
out[i].words = countWords(lines.slice(out[i].start + 1, out[i].end).join("\n"));
|
|
3097
|
+
}
|
|
3098
|
+
return out;
|
|
3099
|
+
}
|
|
3100
|
+
var countWords = (s) => (s.replace(/^#{1,6}\s+/gm, "").match(/\S+/g) ?? []).length;
|
|
3101
|
+
function outline(md, text3) {
|
|
3102
|
+
return { text: text3, words: countWords(md), sections: sections(md).filter((s) => s.level > 1).map(({ level, heading, words: words2 }) => ({ level, heading, words: words2 })) };
|
|
3103
|
+
}
|
|
3104
|
+
function readSection(md, name, maxChars) {
|
|
3105
|
+
const q = name.trim().toLowerCase().replace(/^#+\s*/, "");
|
|
3106
|
+
if (!q) return null;
|
|
3107
|
+
const hits = sections(md).filter((s2) => s2.level > 1 && s2.heading.toLowerCase().includes(q)).sort((a, b) => a.heading.length - b.heading.length);
|
|
3108
|
+
const s = hits[0];
|
|
3109
|
+
if (!s) return null;
|
|
3110
|
+
const lines = md.split("\n");
|
|
3111
|
+
const body = lines.slice(s.start + 1, s.end).join("\n").trim();
|
|
3112
|
+
const truncated = body.length > maxChars;
|
|
3113
|
+
return { heading: s.heading, words: countWords(body), text: truncated ? body.slice(0, maxChars) + "\n\n[truncated]" : body, truncated };
|
|
3114
|
+
}
|
|
3115
|
+
function search(md, query, maxHits) {
|
|
3116
|
+
const terms = query.toLowerCase().match(/[\p{L}\p{N}][\p{L}\p{N}\-'.%]*/gu) ?? [];
|
|
3117
|
+
if (!terms.length) return [];
|
|
3118
|
+
const secs = sections(md);
|
|
3119
|
+
const lines = md.split("\n");
|
|
3120
|
+
const sectionAt = (i) => {
|
|
3121
|
+
let h = "";
|
|
3122
|
+
for (const s of secs) {
|
|
3123
|
+
if (s.start <= i) h = s.heading;
|
|
3124
|
+
else break;
|
|
3125
|
+
}
|
|
3126
|
+
return h;
|
|
3127
|
+
};
|
|
3128
|
+
const out = [];
|
|
3129
|
+
let para2 = [];
|
|
3130
|
+
let paraStart = 0;
|
|
3131
|
+
const flush = (endIdx) => {
|
|
3132
|
+
const text3 = para2.join("\n").trim();
|
|
3133
|
+
para2 = [];
|
|
3134
|
+
if (!text3 || /^#{1,6}\s/.test(text3) || /^\|?\s*-{3,}/.test(text3)) return;
|
|
3135
|
+
const low = text3.toLowerCase();
|
|
3136
|
+
let hits = 0;
|
|
3137
|
+
for (const t of terms) {
|
|
3138
|
+
const re = new RegExp(`(^|[^\\p{L}\\p{N}])${escape(t)}(?=$|[^\\p{L}\\p{N}])`, "giu");
|
|
3139
|
+
const n = (low.match(re) ?? []).length;
|
|
3140
|
+
if (!n) return;
|
|
3141
|
+
hits += n;
|
|
3142
|
+
}
|
|
3143
|
+
out.push({ section: sectionAt(paraStart), text: text3, hits });
|
|
3144
|
+
void endIdx;
|
|
3145
|
+
};
|
|
3146
|
+
lines.forEach((l, i) => {
|
|
3147
|
+
if (!l.trim()) {
|
|
3148
|
+
flush(i);
|
|
3149
|
+
paraStart = i + 1;
|
|
3150
|
+
return;
|
|
3151
|
+
}
|
|
3152
|
+
if (/^#{1,6}\s/.test(l)) {
|
|
3153
|
+
flush(i);
|
|
3154
|
+
paraStart = i + 1;
|
|
3155
|
+
return;
|
|
3156
|
+
}
|
|
3157
|
+
if (!para2.length) paraStart = i;
|
|
3158
|
+
para2.push(l);
|
|
3159
|
+
});
|
|
3160
|
+
flush(lines.length);
|
|
3161
|
+
return out.sort((a, b) => b.hits - a.hits).slice(0, maxHits);
|
|
3162
|
+
}
|
|
3163
|
+
var escape = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3164
|
+
|
|
3165
|
+
// src/log.ts
|
|
3166
|
+
import { appendFileSync, mkdirSync as mkdirSync3, renameSync as renameSync2, statSync as statSync2 } from "fs";
|
|
3167
|
+
import { hostname } from "os";
|
|
3168
|
+
import { join as join9 } from "path";
|
|
3169
|
+
var HOST = hostname();
|
|
3170
|
+
var LOG_FILE = join9(AI4L_LIBRARY_DIR, "library.log");
|
|
3171
|
+
function stamp() {
|
|
3172
|
+
const d = /* @__PURE__ */ new Date();
|
|
3173
|
+
const p = (n) => String(n).padStart(2, "0");
|
|
3174
|
+
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}:${p(d.getSeconds())}`;
|
|
3175
|
+
}
|
|
3176
|
+
var LOG_MAX_BYTES = 50 * 1024 * 1024;
|
|
3177
|
+
function rotateIfLarge(file, max = LOG_MAX_BYTES) {
|
|
3178
|
+
let size = 0;
|
|
3179
|
+
try {
|
|
3180
|
+
size = statSync2(file).size;
|
|
3181
|
+
} catch {
|
|
3182
|
+
return false;
|
|
3183
|
+
}
|
|
3184
|
+
if (size <= max) return false;
|
|
3185
|
+
try {
|
|
3186
|
+
renameSync2(file, `${file}.1`);
|
|
3187
|
+
return true;
|
|
3188
|
+
} catch {
|
|
3189
|
+
return false;
|
|
3190
|
+
}
|
|
3191
|
+
}
|
|
3192
|
+
function log(tool, id, message) {
|
|
3193
|
+
const clean = message.replace(/\x1b\[[0-9;]*m/g, "").replace(/\s*\n\s*/g, " | ");
|
|
3194
|
+
const line = `${stamp()} ${HOST} ${tool.replace(/^library_/, "")} ${id ?? "-"} ${clean}
|
|
3195
|
+
`;
|
|
3196
|
+
try {
|
|
3197
|
+
mkdirSync3(AI4L_LIBRARY_DIR, { recursive: true });
|
|
3198
|
+
rotateIfLarge(LOG_FILE);
|
|
3199
|
+
appendFileSync(LOG_FILE, line);
|
|
3200
|
+
} catch {
|
|
3201
|
+
}
|
|
3202
|
+
}
|
|
3203
|
+
|
|
3204
|
+
// src/doi.ts
|
|
3205
|
+
var NCBI_ESEARCH = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi";
|
|
3206
|
+
async function pmidForDoi(doi) {
|
|
3207
|
+
const u = new URL(NCBI_ESEARCH);
|
|
3208
|
+
u.searchParams.set("db", "pubmed");
|
|
3209
|
+
u.searchParams.set("term", `${doi}[doi]`);
|
|
3210
|
+
u.searchParams.set("retmode", "json");
|
|
3211
|
+
if (process.env.NCBI_API_KEY) u.searchParams.set("api_key", process.env.NCBI_API_KEY);
|
|
3212
|
+
u.searchParams.set("tool", "ai4l-library");
|
|
3213
|
+
u.searchParams.set("email", contactEmail());
|
|
3214
|
+
const j = JSON.parse(await fetchText(u.toString()));
|
|
3215
|
+
const ids = j.esearchresult?.idlist ?? [];
|
|
3216
|
+
return ids.length === 1 ? ids[0] : null;
|
|
3217
|
+
}
|
|
3218
|
+
async function fetchCrossref(doi) {
|
|
3219
|
+
try {
|
|
3220
|
+
return await fetchText(`https://api.crossref.org/works/${encodeURIComponent(doi)}?mailto=${encodeURIComponent(contactEmail())}`, { "User-Agent": LIBRARY_UA });
|
|
3221
|
+
} catch (e) {
|
|
3222
|
+
if (/HTTP 404/.test(e instanceof Error ? e.message : String(e))) return null;
|
|
3223
|
+
throw new Error(`crossref fetch failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3224
|
+
}
|
|
3225
|
+
}
|
|
3226
|
+
var TYPES = {
|
|
3227
|
+
"journal-article": "Journal Article",
|
|
3228
|
+
"proceedings-article": "Conference Paper",
|
|
3229
|
+
"posted-content": "Preprint",
|
|
3230
|
+
"book-chapter": "Book Chapter",
|
|
3231
|
+
"book": "Book",
|
|
3232
|
+
"monograph": "Book",
|
|
3233
|
+
"report": "Report",
|
|
3234
|
+
"dissertation": "Dissertation",
|
|
3235
|
+
"peer-review": "Peer Review",
|
|
3236
|
+
"dataset": "Dataset",
|
|
3237
|
+
"other": "Other"
|
|
3238
|
+
};
|
|
3239
|
+
var UPDATES = {
|
|
3240
|
+
retraction: "retraction_in",
|
|
3241
|
+
withdrawal: "retraction_in",
|
|
3242
|
+
removal: "retraction_in",
|
|
3243
|
+
"partial-retraction": "partial_retraction_in",
|
|
3244
|
+
correction: "erratum_in",
|
|
3245
|
+
erratum: "erratum_in",
|
|
3246
|
+
corrigendum: "erratum_in",
|
|
3247
|
+
clarification: "erratum_in",
|
|
3248
|
+
addendum: "erratum_in",
|
|
3249
|
+
"expression-of-concern": "expression_of_concern_in",
|
|
3250
|
+
"new-version": "update_in"
|
|
3251
|
+
};
|
|
3252
|
+
var decode = (s) => s.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&");
|
|
3253
|
+
var text2 = (s) => decode(s.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
|
|
3254
|
+
function parseCrossref(json2) {
|
|
3255
|
+
let m;
|
|
3256
|
+
try {
|
|
3257
|
+
m = JSON.parse(json2)?.message;
|
|
3258
|
+
} catch {
|
|
3259
|
+
return null;
|
|
3260
|
+
}
|
|
3261
|
+
if (!m?.DOI) return null;
|
|
3262
|
+
const authors = (m.author ?? []).map((a) => a.family ? [a.family, String(a.given ?? "").split(/[\s-]+/).filter(Boolean).map((g) => g[0]).join("")].filter(Boolean).join(" ") : a.name ?? "").filter(Boolean);
|
|
3263
|
+
const first = (m.author ?? [])[0];
|
|
3264
|
+
const date = m.issued?.["date-parts"]?.[0] ?? m.published?.["date-parts"]?.[0] ?? m.created?.["date-parts"]?.[0];
|
|
3265
|
+
const abstract = [];
|
|
3266
|
+
const raw = String(m.abstract ?? "");
|
|
3267
|
+
const secs = [...raw.matchAll(/<jats:sec>\s*(?:<jats:title>([\s\S]*?)<\/jats:title>)?([\s\S]*?)<\/jats:sec>/g)];
|
|
3268
|
+
if (secs.length) for (const s of secs) {
|
|
3269
|
+
const t = text2(s[2]);
|
|
3270
|
+
if (t) abstract.push({ label: s[1] ? text2(s[1]).replace(/[:.]$/, "") : null, text: t });
|
|
3271
|
+
}
|
|
3272
|
+
else {
|
|
3273
|
+
const t = text2(raw.replace(/<jats:title>[\s\S]*?<\/jats:title>/, ""));
|
|
3274
|
+
if (t) abstract.push({ label: null, text: t });
|
|
3275
|
+
}
|
|
3276
|
+
const corrections = (m["updated-by"] ?? []).flatMap((u) => {
|
|
3277
|
+
const type = UPDATES[String(u.type ?? "").toLowerCase()];
|
|
3278
|
+
return type ? [{ type, pmid: null, source: `${u.label ?? u.type}: doi:${u.DOI}${u.updated?.["date-time"] ? ` (${String(u.updated["date-time"]).slice(0, 10)})` : ""}` }] : [];
|
|
3279
|
+
});
|
|
3280
|
+
const funding = [];
|
|
3281
|
+
for (const f of m.funder ?? []) {
|
|
3282
|
+
const n = text2(String(f.name ?? ""));
|
|
3283
|
+
if (n && !funding.includes(n)) funding.push(n);
|
|
3284
|
+
}
|
|
3285
|
+
return {
|
|
3286
|
+
pmid: "",
|
|
3287
|
+
pmcid: null,
|
|
3288
|
+
doi: String(m.DOI).toLowerCase(),
|
|
3289
|
+
title: text2(String((m.title ?? [])[0] ?? "")) || String(m.DOI),
|
|
3290
|
+
authors,
|
|
3291
|
+
firstAuthor: first ? first.family ?? first.name ?? null : null,
|
|
3292
|
+
journal: text2(String((m["container-title"] ?? [])[0] ?? "")) || m.publisher || null,
|
|
3293
|
+
year: date?.[0] ? String(date[0]) : null,
|
|
3294
|
+
publicationTypes: [TYPES[m.type] ?? "Other"],
|
|
3295
|
+
abstract,
|
|
3296
|
+
corrections,
|
|
3297
|
+
coi: null,
|
|
3298
|
+
funding
|
|
3299
|
+
};
|
|
3300
|
+
}
|
|
3301
|
+
function fromInvertedIndex(index) {
|
|
3302
|
+
if (!index) return "";
|
|
3303
|
+
const words2 = [];
|
|
3304
|
+
for (const [w, ps] of Object.entries(index)) for (const p of ps) words2[p] = w;
|
|
3305
|
+
return words2.filter((w) => w !== void 0).join(" ").replace(/\s+([.,;:!?)])/g, "$1").replace(/\(\s+/g, "(").trim();
|
|
3306
|
+
}
|
|
3307
|
+
async function openAlexAbstract(doi) {
|
|
3308
|
+
try {
|
|
3309
|
+
const j = JSON.parse(await fetchText(`https://api.openalex.org/works/https://doi.org/${encodeURIComponent(doi)}?mailto=${encodeURIComponent(contactEmail())}&select=abstract_inverted_index`, { "User-Agent": LIBRARY_UA }));
|
|
3310
|
+
return fromInvertedIndex(j.abstract_inverted_index);
|
|
3311
|
+
} catch {
|
|
3312
|
+
return "";
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
async function fetchCsl(doi) {
|
|
3316
|
+
try {
|
|
3317
|
+
const r = await fetch(`https://doi.org/${encodeURIComponent(doi)}`, { headers: { Accept: "application/vnd.citationstyles.csl+json", "User-Agent": LIBRARY_UA }, redirect: "follow", signal: AbortSignal.timeout(3e4) });
|
|
3318
|
+
if (!r.ok) return null;
|
|
3319
|
+
const t = await r.text();
|
|
3320
|
+
return /^\s*\{/.test(t) ? t : null;
|
|
3321
|
+
} catch {
|
|
3322
|
+
return null;
|
|
3323
|
+
}
|
|
3324
|
+
}
|
|
3325
|
+
var CSL_TYPES = { "article-journal": "Journal Article", "paper-conference": "Conference Paper", chapter: "Book Chapter", book: "Book", report: "Report", thesis: "Dissertation", "posted-content": "Preprint", article: "Preprint", dataset: "Dataset" };
|
|
3326
|
+
function parseCsl(json2) {
|
|
3327
|
+
let m;
|
|
3328
|
+
try {
|
|
3329
|
+
m = JSON.parse(json2);
|
|
3330
|
+
} catch {
|
|
3331
|
+
return null;
|
|
3332
|
+
}
|
|
3333
|
+
if (!m?.DOI && !m?.title) return null;
|
|
3334
|
+
const people = m.author ?? m.editor ?? [];
|
|
3335
|
+
const authors = people.map((a) => a.family ? [a.family, String(a.given ?? "").split(/[\s.-]+/).filter(Boolean).map((g) => g[0]).join("")].filter(Boolean).join(" ") : a.literal ?? a.name ?? "").filter(Boolean);
|
|
3336
|
+
const date = m.issued?.["date-parts"]?.[0] ?? m.published?.["date-parts"]?.[0];
|
|
3337
|
+
const abs = text2(String(m.abstract ?? ""));
|
|
3338
|
+
return {
|
|
3339
|
+
pmid: "",
|
|
3340
|
+
pmcid: null,
|
|
3341
|
+
doi: String(m.DOI ?? "").toLowerCase(),
|
|
3342
|
+
title: text2(String(Array.isArray(m.title) ? m.title[0] : m.title ?? "")) || String(m.DOI),
|
|
3343
|
+
authors,
|
|
3344
|
+
firstAuthor: people[0] ? people[0].family ?? people[0].literal ?? null : null,
|
|
3345
|
+
journal: text2(String(Array.isArray(m["container-title"]) ? m["container-title"][0] ?? "" : m["container-title"] ?? "")) || m.publisher || null,
|
|
3346
|
+
year: date?.[0] ? String(date[0]) : null,
|
|
3347
|
+
publicationTypes: [CSL_TYPES[m.type] ?? "Other"],
|
|
3348
|
+
abstract: abs ? [{ label: null, text: abs }] : [],
|
|
3349
|
+
corrections: [],
|
|
3350
|
+
coi: null,
|
|
3351
|
+
funding: []
|
|
3352
|
+
};
|
|
3353
|
+
}
|
|
3354
|
+
|
|
3355
|
+
// src/tools.ts
|
|
3356
|
+
var pkg = (() => {
|
|
3357
|
+
try {
|
|
3358
|
+
return JSON.parse(readFileSync9(new URL("../package.json", import.meta.url), "utf8"));
|
|
3359
|
+
} catch {
|
|
3360
|
+
return { name: "ai4l-library", version: "0.0.0" };
|
|
3361
|
+
}
|
|
3362
|
+
})();
|
|
3363
|
+
var INSTRUCTIONS = `ai4l-library builds a local library of the scientific papers, drug labels and trial records cited by evidence reviews. Given an ID ("type:value": a paper pmid:30955454, bare digits count as pmid; a drug label dailymed:<set ID>; a trial record nct:05173896, the eight digits \u2014 nct:NCT05173896 and a bare NCT05173896 are accepted) it fetches the record and stores it as files under the library directory. Nothing flows through your context: the server writes to disk itself and returns only a small JSON status per call. One ID per call \u2014 no batches.
|
|
3364
|
+
|
|
3365
|
+
Tools:
|
|
3366
|
+
- library_add(id) \u2014 build or complete the entry for one ID and return its status. Idempotent: a full_text entry is returned as is; an abstract entry retries only tiers whose earlier failure was transient (network, rate limit, wall page) and older than retry_after_days \u2014 permanent reasons (no PMCID, closed, no body) are never retried. An existing entry whose PubMed record was last read more than refresh_after_days ago (default 30) gets the record re-read for new correction notices and retractions; the stored abstract and full text are not touched.
|
|
3367
|
+
- library_status(id) \u2014 read-only: the entry's status, or level "missing".
|
|
3368
|
+
- library_read(id, section?, query?, max_chars?, max_hits?) \u2014 one small piece of a stored paper's Markdown, never the whole paper. No section/query: the outline (headings with word counts, and whether it is the full text or the abstract only). section: a heading from the outline (case-insensitive substring) \u2192 that section's text, capped at max_chars. query: search terms \u2192 the paragraphs containing all of them, each with its section, best first, at most max_hits; with no hit the answer lists the headings instead. Every full text converted from PMC/JATS XML has a "Declarations" section \u2014 funding, conflicts of interest, author notes \u2014 so section "Declarations" answers who paid and who declares what in one call; an empty answer there means the paper states nothing. Reads are read-only and may run in parallel.
|
|
3369
|
+
- library_verify(claim, id) \u2014 is one finding of a review stated by one source it cites? claim: the finding alone as a complete statement (subject, intervention, outcome, number), without its citations and without the review's framing or inference; id: one source the review cites for it \u2014 a paper, a drug label (dailymed:\u2026) or a trial record (nct:\u2026), each judged on its stored full text the same way. A finding cited to three papers is three calls with the same claim; a claim that names a source, joins two findings or is a fragment is refused. The server reads the paper (full text, else abstract) and asks a decision model; nothing of the paper enters your context. Result: verdict "pass" (score is the probability), "fail" (the full text does not state it), "na" (only the abstract is in the library and does not state it, or only the title) or "error" (claim empty, too short, a fragment, naming a source or compound; paper not in the library; no model; model failure); basis full_text, abstract or title; glyphs_unresolved, the symbols the paper's PDF font gave no character for (above 0: a fail may be the font, not the paper); superseded, true when the claim uses a figure a stored correction replaced (the verdict is kept), null without a stored correction. Call library_add on the paper first.
|
|
3370
|
+
- library_info(ids?) \u2014 library-wide, everything the TUI shows: the library directory, the decision model and whether it answers, entries and size on disk, per type (pmid, doi, dailymed, nct) the entries with their levels (full_text, abstract, title) and retractions; entry counts by type and level, counts of retracted and not-citable entries and of entries with errata, expressions of concern and updates, the configured tiers (enabled/disabled, in order), the verify model with its endpoint and whether it answers right now (ok, ms, error); ids=true lists every entry ID.
|
|
3371
|
+
- get_version() \u2014 package name, version and the resolved library directory.
|
|
3372
|
+
|
|
3373
|
+
Every action is appended to <library>/library.log, one line per action, prefixed with the host the server runs on.
|
|
3374
|
+
|
|
3375
|
+
Status JSON fields: requested, type, id, entry, entry_dir, library_dir, pmid, setid (a drug label's DailyMed set ID), nctid (a trial record's ClinicalTrials.gov ID), publications (a trial record's results publications, pmid:\u2026), pmcid, doi, title, first_author, year, publication_types, retracted, corrections, coi (the paper's conflict-of-interest statement: PubMed's, else the paper's own from its full text; null when neither states one), funding (who paid: PubMed's grant agencies, else the paper's own funding statement; [] when neither states any), citable, comment, has_abstract (false for title-only records: letters, editorials, old papers \u2014 not citable evidence unless a full text was retrieved), level (what the entry holds: "full_text", "abstract", or "title" when there is neither a full text nor abstract text; "missing" from library_status when there is no entry), version, retrieved_by, retrieved_date, pubmed_checked, attempts, files, glyphs_unresolved (control characters left in the full text where a PDF font gave no symbol \u2014 a minus, \xB1, \u2264 \u2014 so a figure there may not match; null without full text), and error when the call could not produce an entry ("invalid id", "type not supported yet", "pmid not found", or a fetch failure).
|
|
3376
|
+
|
|
3377
|
+
citable is the one flag to check before citing a paper as evidence: false when PubMed records it as retracted (later also when a correction changed its data). comment says why, or \u2014 when citable \u2014 names what to be aware of: an erratum (not assessed; read it via library_add on its pmid when it has one), an expression of concern, or a newer version that supersedes this one (cite that instead). corrections lists the notices behind it, {type, pmid, source}: type is erratum_in, retraction_in, partial_retraction_in, retracted_and_republished_in, expression_of_concern_in or update_in; pmid is the notice's own PMID (null when PubMed has none); source its citation. Notices are read from PubMed at ingest and re-read once the record is older than refresh_after_days (pubmed_checked); the stored abstract and full text are never altered by a notice.
|
|
3378
|
+
|
|
3379
|
+
v0.2: pmid:, doi:, dailymed: and nct: IDs (other types return error "type not supported yet"). A paper PubMed does not index \u2014 doi:10.\u2026, from the review's doi.org link \u2014 gets its record from Crossref: title, authors, journal, year, the abstract (Crossref's, else OpenAlex's), funders, and Crossref's update notices (corrections, retractions) as corrections; status pmid null, the head line says "not in PubMed"; the full-text cascade then runs on the DOI (unpaywall, openalex, tdm, core, url tiers). library_add on a DOI PubMed does index builds that paper's pmid: entry and stores a copy of it under doi:, so the DOI resolves on disk: library_status, library_read and library_verify answer from the doi: entry without any lookup; the copy is refreshed from the pmid: entry on every library_add of the DOI. A trial record \u2014 nct:<8 digits>, from the review's ClinicalTrials.gov link \u2014 is fetched from the ClinicalTrials.gov API as the current record and stored as a full text: one section per module (identification with secondary IDs, status with every date and its actual/estimated type, sponsor and collaborators, description, conditions, design with phase, allocation, masking and the enrollment with its type, arms and interventions with doses, primary and secondary outcomes with time frames, eligibility, locations, references) and, when results are posted, the participant flow, baseline, every outcome measure with its group values and the adverse events as tables. Values are in plain words: "Phase 2", "Active, not recruiting", "100 (estimated)". Its status has nctid, publications (the results publications as pmid:\u2026 from the record's references), title the brief title, first_author the lead sponsor, year the first-posted year, version the last-update-posted date, publication_types ["Trial Registration"]; library_read and library_verify work on it like on a paper. An entry older than refresh_after_days is re-fetched and rewritten when the record was updated since \u2014 a stale status in a review then fails, which is right. The entry holds the API JSON and the Markdown as nct-<digits>-fulltext.json and -fulltext.md under nct/<digits 1\u20132>/<digits 1\u20134>/<digits>/; its log line names the update date and the status. A drug label \u2014 dailymed:<set ID>, the set ID being the fixed identifier in the label's DailyMed URL (\u2026/drugInfo.cfm?setid=<set ID>) \u2014 is fetched from DailyMed as the current version of the FDA prescribing information and stored as a full text: a "Highlights" section with the header lines (product line, "Initial U.S. Approval: <year>"), then every text section in order, the boxed warning and recent major changes named as such (indications, dosage, contraindications, warnings, adverse reactions with their tables, drug interactions, clinical pharmacology, clinical studies), no abstract. Its status has setid, title "<brand> (<generic>) <form> \u2014 prescribing information", first_author the labeler, year the effective year, version "spl <n>", publication_types ["Drug Label"]; library_read and library_verify work on it like on a paper. An entry older than refresh_after_days is re-fetched, and rewritten when the label's version changed. The entry holds the SPL XML and the Markdown as dailymed-<set ID>-fulltext.xml and -fulltext.md under dailymed/<first 2>/<first 4>/<set ID>/; its log line names the version and effective date. Tiers: pubmed (metadata + abstract, always), then the full-text cascade: inbox (a file you downloaded yourself, dropped into <library>/inbox/ with the PMID in its name: PDF, XML or HTML), pmc (JATS XML), europepmc (Europe PMC: its own XML/PDF and the free links it lists), tdm (the publishers' text-and-data-mining links from Crossref, with ELSEVIER_API_KEY / WILEY_TDM_TOKEN), unpaywall and openalex (locate open-access copies and download them: PDF, XML or HTML \u2192 Markdown), core (CORE repository copies, with CORE_API_KEY), plus user url tiers (a URL template with {doi}/{pmid}/{pmcid}, e.g. an institutional proxy) from <library>/config.json. Each result is checked (title present, methods and results present, no wall page) before it counts. Level "full_text" when a tier succeeded, else "abstract" with the failed tiers in attempts.`;
|
|
3380
|
+
var idSchema = z.string().describe('ID as "type:value": a paper "pmid:30955454" (bare digits count as pmid) or "doi:10.5812/asjsm.110753" (a paper PubMed does not index; a DOI PubMed indexes is taken as its PMID), a drug label "dailymed:<set ID>" (the set ID from its DailyMed page URL), or a trial record "nct:05173896" (the eight digits; nct:NCT05173896 and a bare NCT05173896 are accepted).');
|
|
3381
|
+
var levelWord = (m) => m.level === "abstract" && !m.has_abstract ? "title" : m.level;
|
|
3382
|
+
var json = (s) => ({ content: [{ type: "text", text: JSON.stringify(s, null, 2) }] });
|
|
3383
|
+
function noticeTag(m) {
|
|
3384
|
+
const n = (t) => m.corrections.filter((c) => c.type === t).length;
|
|
3385
|
+
const parts = [
|
|
3386
|
+
m.retracted ? "retracted" : "",
|
|
3387
|
+
n("erratum_in") ? `erratum=${n("erratum_in")}` : "",
|
|
3388
|
+
n("expression_of_concern_in") ? `concern=${n("expression_of_concern_in")}` : "",
|
|
3389
|
+
n("update_in") ? `update=${n("update_in")}` : ""
|
|
3390
|
+
].filter(Boolean);
|
|
3391
|
+
return parts.length ? " " + parts.join(" ") : "";
|
|
3392
|
+
}
|
|
3393
|
+
function registerTools(server2) {
|
|
3394
|
+
async function completeFulltext(paths, meta, requested) {
|
|
3395
|
+
const paper = { title: meta.title, pmid: meta.pmid, pmcid: meta.pmcid, doi: meta.doi, headLine: readHeadLine(paths) };
|
|
3396
|
+
const tried = /* @__PURE__ */ new Set();
|
|
3397
|
+
let ran = false;
|
|
3398
|
+
const { hit } = await fulltextCascade(paper, meta.attempts, (e) => {
|
|
3399
|
+
if (e.ok === "try") tried.add(e.tier);
|
|
3400
|
+
if (e.ok === false && e.ephemeral) return;
|
|
3401
|
+
ran = true;
|
|
3402
|
+
if (e.ok === false) meta = recordAttempts(paths, meta, [attemptLabel(e)]);
|
|
3403
|
+
log(
|
|
3404
|
+
"library_add",
|
|
3405
|
+
requested,
|
|
3406
|
+
e.ok === true ? `tier ${e.tier} ok (${e.format}, ${e.version})` : e.ok === "try" ? `tier ${e.tier} ${e.message}` : e.skipped ? `tier ${e.tier} skipped: ${e.reason}` : tried.has(e.tier) ? `tier ${e.tier} failed` : `tier ${e.tier} failed: ${e.reason}`
|
|
3407
|
+
);
|
|
3408
|
+
});
|
|
3409
|
+
if (hit) meta = applyFulltext(paths, meta, hit, `${pkg.name} ${pkg.version}`);
|
|
3410
|
+
return { meta, ran };
|
|
3411
|
+
}
|
|
3412
|
+
async function addLabel(requested, setid, paths, status) {
|
|
3413
|
+
if (entryExists(paths)) {
|
|
3414
|
+
const meta = readMeta(paths);
|
|
3415
|
+
if (daysSince(meta.pubmed_checked) < settings().refresh_after_days) {
|
|
3416
|
+
log("library_add", requested, `exists level=full_text retrieved_by=dailymed ${meta.version ?? ""} ${meta.year ?? ""}`);
|
|
3417
|
+
return statusFromMeta(paths, meta, status);
|
|
3418
|
+
}
|
|
3419
|
+
try {
|
|
3420
|
+
const xml = await fetchSpl(setid);
|
|
3421
|
+
const s = parseSpl(xml);
|
|
3422
|
+
if (!s) throw new Error("dailymed: not an SPL document");
|
|
3423
|
+
const version = s.version ? `spl ${s.version}` : null;
|
|
3424
|
+
if (version === meta.version) {
|
|
3425
|
+
const next2 = { ...meta, pubmed_checked: today() };
|
|
3426
|
+
writeMeta(paths, next2);
|
|
3427
|
+
log("library_add", requested, `exists level=full_text retrieved_by=dailymed ${meta.version ?? ""} checked`);
|
|
3428
|
+
return statusFromMeta(paths, next2, status);
|
|
3429
|
+
}
|
|
3430
|
+
const next = writeLabelEntry(paths, s, xml, `${pkg.name} ${pkg.version}`);
|
|
3431
|
+
log("library_add", requested, `refreshed level=full_text retrieved_by=dailymed ${meta.version ?? "?"} \u2192 ${next.version} effective=${s.effective}`);
|
|
3432
|
+
return statusFromMeta(paths, next, status);
|
|
3433
|
+
} catch (e) {
|
|
3434
|
+
log("library_add", requested, `refresh failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3435
|
+
return statusFromMeta(paths, meta, status);
|
|
3436
|
+
}
|
|
3437
|
+
}
|
|
3438
|
+
try {
|
|
3439
|
+
const xml = await fetchSpl(setid);
|
|
3440
|
+
const s = parseSpl(xml);
|
|
3441
|
+
if (!s) {
|
|
3442
|
+
log("library_add", requested, "dailymed failed: not an SPL document");
|
|
3443
|
+
return { ...status, error: "dailymed: not an SPL document" };
|
|
3444
|
+
}
|
|
3445
|
+
const meta = writeLabelEntry(paths, s, xml, `${pkg.name} ${pkg.version}`);
|
|
3446
|
+
log("library_add", requested, `dailymed fetched level=full_text ${meta.version ?? ""} effective=${s.effective} ${s.brand ?? ""} (${s.generic ?? ""}) entry=${paths.entry}`);
|
|
3447
|
+
return statusFromMeta(paths, meta, status);
|
|
3448
|
+
} catch (e) {
|
|
3449
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
3450
|
+
log("library_add", requested, `dailymed failed: ${msg}`);
|
|
3451
|
+
return { ...status, error: msg };
|
|
3452
|
+
}
|
|
3453
|
+
}
|
|
3454
|
+
async function addStudy(requested, nctid, paths, status) {
|
|
3455
|
+
if (entryExists(paths)) {
|
|
3456
|
+
const meta = readMeta(paths);
|
|
3457
|
+
if (daysSince(meta.pubmed_checked) < settings().refresh_after_days) {
|
|
3458
|
+
log("library_add", requested, `exists level=full_text retrieved_by=ctgov updated ${meta.version ?? ""}`);
|
|
3459
|
+
return statusFromMeta(paths, meta, status);
|
|
3460
|
+
}
|
|
3461
|
+
try {
|
|
3462
|
+
const json2 = await fetchStudy(nctid);
|
|
3463
|
+
const s = parseStudy(json2);
|
|
3464
|
+
if (!s) throw new Error("ctgov: not a study record");
|
|
3465
|
+
if (s.lastUpdate === meta.version) {
|
|
3466
|
+
const next2 = { ...meta, pubmed_checked: today() };
|
|
3467
|
+
writeMeta(paths, next2);
|
|
3468
|
+
log("library_add", requested, `exists level=full_text retrieved_by=ctgov updated ${meta.version ?? ""} checked`);
|
|
3469
|
+
return statusFromMeta(paths, next2, status);
|
|
3470
|
+
}
|
|
3471
|
+
const next = writeStudyEntry(paths, s, json2, `${pkg.name} ${pkg.version}`);
|
|
3472
|
+
log("library_add", requested, `refreshed level=full_text retrieved_by=ctgov updated ${meta.version ?? "?"} \u2192 ${next.version} ${s.status ?? ""}${s.hasResults ? " results" : ""}`);
|
|
3473
|
+
return statusFromMeta(paths, next, status);
|
|
3474
|
+
} catch (e) {
|
|
3475
|
+
log("library_add", requested, `refresh failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3476
|
+
return statusFromMeta(paths, meta, status);
|
|
3477
|
+
}
|
|
3478
|
+
}
|
|
3479
|
+
try {
|
|
3480
|
+
const json2 = await fetchStudy(nctid);
|
|
3481
|
+
const s = parseStudy(json2);
|
|
3482
|
+
if (!s) {
|
|
3483
|
+
log("library_add", requested, "ctgov failed: not a study record");
|
|
3484
|
+
return { ...status, error: "ctgov: not a study record" };
|
|
3485
|
+
}
|
|
3486
|
+
const meta = writeStudyEntry(paths, s, json2, `${pkg.name} ${pkg.version}`);
|
|
3487
|
+
log("library_add", requested, `ctgov fetched level=full_text updated ${meta.version ?? ""} ${s.status ?? ""}${s.hasResults ? " results" : ""} ${s.acronym ?? ""} entry=${paths.entry}`);
|
|
3488
|
+
return statusFromMeta(paths, meta, status);
|
|
3489
|
+
} catch (e) {
|
|
3490
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
3491
|
+
log("library_add", requested, `ctgov failed: ${msg}`);
|
|
3492
|
+
return { ...status, error: msg };
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
server2.tool(
|
|
3496
|
+
"library_add",
|
|
3497
|
+
"Build or complete the library entry for one ID and return its status JSON. A drug label (dailymed:<set ID>) is fetched from DailyMed as the current prescribing information, a trial record (nct:<8 digits>, the NCT number's digits; nct:NCT\u2026 and a bare NCT\u2026 are accepted) from ClinicalTrials.gov as the current record with its posted results; both stored as a full text. A paper is a PMID, or a DOI (doi:10.\u2026): a DOI PubMed indexes is added as its PMID's entry plus a copy under doi: (refreshed on every add of the DOI), one PubMed does not index gets its metadata, funders and update notices from Crossref and its abstract from Crossref, else OpenAlex. For a paper, pubmed gives metadata, abstract and the correction notices (errata, retractions, expressions of concern, updates \u2192 citable, comment, corrections); the full-text cascade (pmc, tdm, unpaywall, openalex, user url tiers) is then tried. Idempotent: a full_text entry is returned as is; an abstract entry retries only tiers whose earlier failure was transient and past the cooldown. An entry whose PubMed record is older than refresh_after_days gets the record re-read for new notices; abstract and full text are never touched.",
|
|
3498
|
+
{ id: idSchema },
|
|
3499
|
+
async ({ id }) => {
|
|
3500
|
+
const t0 = Date.now();
|
|
3501
|
+
let parsed = parseId(id);
|
|
3502
|
+
if (parsed.error) {
|
|
3503
|
+
log("library_add", parsed.requested, `error ${parsed.error}`);
|
|
3504
|
+
return json({ ...emptyStatus(parsed.requested, parsed.type, parsed.id, AI4L_LIBRARY_DIR), error: parsed.error });
|
|
3505
|
+
}
|
|
3506
|
+
let copyTo = null;
|
|
3507
|
+
if (parsed.type === "doi") {
|
|
3508
|
+
const doiPaths = entryPaths(AI4L_LIBRARY_DIR, "doi", parsed.id);
|
|
3509
|
+
let pmid = entryExists(doiPaths) ? readMeta(doiPaths).pmid || null : null;
|
|
3510
|
+
if (!pmid && !entryExists(doiPaths)) {
|
|
3511
|
+
try {
|
|
3512
|
+
pmid = await pmidForDoi(parsed.id);
|
|
3513
|
+
} catch (e) {
|
|
3514
|
+
log("library_add", parsed.requested, `pubmed lookup failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3515
|
+
}
|
|
3516
|
+
if (pmid && !entryExists(entryPaths(AI4L_LIBRARY_DIR, "pmid", pmid))) {
|
|
3517
|
+
try {
|
|
3518
|
+
if (!parsePubmedXml(await fetchPubmedXml(pmid))) {
|
|
3519
|
+
log("library_add", parsed.requested, `pmid:${pmid} has no readable PubMed record \u2014 kept as doi:`);
|
|
3520
|
+
pmid = null;
|
|
3521
|
+
}
|
|
3522
|
+
} catch {
|
|
3523
|
+
}
|
|
3524
|
+
}
|
|
3525
|
+
if (pmid) log("library_add", parsed.requested, `indexed in PubMed as pmid:${pmid}`);
|
|
3526
|
+
}
|
|
3527
|
+
if (pmid) {
|
|
3528
|
+
copyTo = doiPaths;
|
|
3529
|
+
parsed = { requested: parsed.requested, type: "pmid", id: pmid };
|
|
3530
|
+
}
|
|
3531
|
+
}
|
|
3532
|
+
const status = emptyStatus(parsed.requested, parsed.type, parsed.id, AI4L_LIBRARY_DIR);
|
|
3533
|
+
const isDoi = parsed.type === "doi";
|
|
3534
|
+
const paths = entryPaths(AI4L_LIBRARY_DIR, parsed.type, parsed.id);
|
|
3535
|
+
const finish = (m) => {
|
|
3536
|
+
if (!copyTo) return json(statusFromMeta(paths, m, status));
|
|
3537
|
+
const copy = copyEntry(paths, copyTo);
|
|
3538
|
+
return json(statusFromMeta(copyTo, copy, emptyStatus(parsed.requested, "doi", parseId(parsed.requested).id, AI4L_LIBRARY_DIR)));
|
|
3539
|
+
};
|
|
3540
|
+
if (parsed.type === "dailymed") return json(await addLabel(`dailymed:${parsed.id}`, parsed.id, paths, status));
|
|
3541
|
+
if (parsed.type === "nct") return json(await addStudy(`nct:${parsed.id}`, `NCT${parsed.id}`, paths, status));
|
|
3542
|
+
let meta, ran = false;
|
|
3543
|
+
const entryExistedBefore = entryExists(paths);
|
|
3544
|
+
if (entryExistedBefore) {
|
|
3545
|
+
meta = readMeta(paths);
|
|
3546
|
+
if (isDoi && daysSince(meta.pubmed_checked) >= settings().refresh_after_days) {
|
|
3547
|
+
try {
|
|
3548
|
+
const r = parseCrossref(await fetchCrossref(parsed.id) ?? "");
|
|
3549
|
+
if (r) {
|
|
3550
|
+
const before = noticeTag(meta);
|
|
3551
|
+
meta = { ...meta, corrections: r.corrections, retracted: r.corrections.some(isRetractionNotice), funding: meta.funding.length ? meta.funding : r.funding, pubmed_checked: today() };
|
|
3552
|
+
writeMeta(paths, meta);
|
|
3553
|
+
log("library_add", parsed.requested, `refreshed crossref${noticeTag(meta) !== before ? ` changed: ${before.trim() || "none"} \u2192 ${noticeTag(meta).trim() || "none"}` : ""}`);
|
|
3554
|
+
} else log("library_add", parsed.requested, "refresh failed: doi not found");
|
|
3555
|
+
} catch (e) {
|
|
3556
|
+
log("library_add", parsed.requested, `refresh failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3557
|
+
}
|
|
3558
|
+
} else if (daysSince(meta.pubmed_checked) >= settings().refresh_after_days) {
|
|
3559
|
+
try {
|
|
3560
|
+
const xml = await fetchPubmedXml(parsed.id);
|
|
3561
|
+
const record = parsePubmedXml(xml);
|
|
3562
|
+
if (record) {
|
|
3563
|
+
const before = noticeTag(meta);
|
|
3564
|
+
meta = refreshRecord(paths, meta, record, xml);
|
|
3565
|
+
log("library_add", parsed.requested, `refreshed pubmed_checked=${meta.pubmed_checked}${noticeTag(meta) !== before ? ` changed: ${before.trim() || "none"} \u2192 ${noticeTag(meta).trim() || "none"}` : ""}`);
|
|
3566
|
+
} else log("library_add", parsed.requested, "refresh failed: pmid not found");
|
|
3567
|
+
} catch (e) {
|
|
3568
|
+
log("library_add", parsed.requested, `refresh failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
3569
|
+
}
|
|
3570
|
+
}
|
|
3571
|
+
if (meta.level === "full_text") {
|
|
3572
|
+
log("library_add", parsed.requested, `exists level=full_text retrieved_by=${meta.retrieved_by}${noticeTag(meta)} ${meta.doi ? `doi:${meta.doi}` : "no-doi"}`);
|
|
3573
|
+
return finish(meta);
|
|
3574
|
+
}
|
|
3575
|
+
log("library_add", parsed.requested, `exists level=${levelWord(meta)}${noticeTag(meta)} ${meta.doi ? `doi:${meta.doi}` : "no-doi"}`);
|
|
3576
|
+
} else if (isDoi) {
|
|
3577
|
+
let json_;
|
|
3578
|
+
try {
|
|
3579
|
+
json_ = await fetchCrossref(parsed.id);
|
|
3580
|
+
} catch (e) {
|
|
3581
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
3582
|
+
log("library_add", parsed.requested, `crossref failed: ${msg}`);
|
|
3583
|
+
return json({ ...status, error: msg });
|
|
3584
|
+
}
|
|
3585
|
+
let record = json_ ? parseCrossref(json_) : null;
|
|
3586
|
+
let recordFrom = "crossref";
|
|
3587
|
+
if (!record) {
|
|
3588
|
+
const csl = await fetchCsl(parsed.id);
|
|
3589
|
+
record = csl ? parseCsl(csl) : null;
|
|
3590
|
+
if (record) {
|
|
3591
|
+
json_ = csl;
|
|
3592
|
+
recordFrom = "doi.org";
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
if (!record) {
|
|
3596
|
+
log("library_add", parsed.requested, "crossref failed: doi not found at Crossref or doi.org");
|
|
3597
|
+
return json({ ...status, error: "doi not found" });
|
|
3598
|
+
}
|
|
3599
|
+
if (!record.doi) record.doi = parsed.id;
|
|
3600
|
+
let abstractFrom = recordFrom;
|
|
3601
|
+
if (!record.abstract.length) {
|
|
3602
|
+
const t = await openAlexAbstract(parsed.id);
|
|
3603
|
+
if (t) {
|
|
3604
|
+
record.abstract = [{ label: null, text: t }];
|
|
3605
|
+
abstractFrom = "openalex";
|
|
3606
|
+
}
|
|
3607
|
+
}
|
|
3608
|
+
meta = writeDoiEntry(paths, record, json_, `${pkg.name} ${pkg.version}`, abstractFrom, recordFrom);
|
|
3609
|
+
log("library_add", parsed.requested, `${recordFrom === "crossref" ? "crossref" : "doi.org"} fetched level=${levelWord(meta)} \u2014 not in PubMed${noticeTag(meta)}${meta.has_abstract ? ` abstract from ${abstractFrom}` : ""} entry=${paths.entry}`);
|
|
3610
|
+
} else {
|
|
3611
|
+
let xml;
|
|
3612
|
+
try {
|
|
3613
|
+
xml = await fetchPubmedXml(parsed.id);
|
|
3614
|
+
} catch (e) {
|
|
3615
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
3616
|
+
log("library_add", parsed.requested, `pubmed failed: ${msg}`);
|
|
3617
|
+
return json({ ...status, error: msg });
|
|
3618
|
+
}
|
|
3619
|
+
const record = parsePubmedXml(xml);
|
|
3620
|
+
if (!record) {
|
|
3621
|
+
log("library_add", parsed.requested, "pubmed failed: pmid not found");
|
|
3622
|
+
return json({ ...status, error: "pmid not found" });
|
|
3623
|
+
}
|
|
3624
|
+
meta = writeEntry(paths, record, xml, `${pkg.name} ${pkg.version}`);
|
|
3625
|
+
log("library_add", parsed.requested, `pubmed fetched level=${levelWord(meta)}${noticeTag(meta)} ${meta.doi ? `doi:${meta.doi}` : "no-doi"}${meta.pmcid ? ` pmcid:${meta.pmcid}` : ""} entry=${paths.entry}`);
|
|
3626
|
+
}
|
|
3627
|
+
({ meta, ran } = await completeFulltext(paths, meta, parsed.requested));
|
|
3628
|
+
if (entryExistedBefore && !ran) return finish(meta);
|
|
3629
|
+
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
3630
|
+
log("library_add", parsed.requested, `done ${secs} level=${levelWord(meta)} retrieved_by=${meta.retrieved_by}`);
|
|
3631
|
+
return finish(meta);
|
|
3632
|
+
}
|
|
3633
|
+
);
|
|
3634
|
+
server2.tool(
|
|
3635
|
+
"library_status",
|
|
3636
|
+
'Read-only status of the library entry for one paper ID: the same status JSON as library_add, or level "missing" when there is no entry. Never fetches.',
|
|
3637
|
+
{ id: idSchema },
|
|
3638
|
+
async ({ id }) => {
|
|
3639
|
+
const parsed = parseId(id);
|
|
3640
|
+
const status = emptyStatus(parsed.requested, parsed.type, parsed.id, AI4L_LIBRARY_DIR);
|
|
3641
|
+
if (parsed.error) {
|
|
3642
|
+
log("library_status", parsed.requested, `error ${parsed.error}`);
|
|
3643
|
+
return json({ ...status, error: parsed.error });
|
|
3644
|
+
}
|
|
3645
|
+
const paths = entryPaths(AI4L_LIBRARY_DIR, parsed.type, parsed.id);
|
|
3646
|
+
if (entryExists(paths)) {
|
|
3647
|
+
const s = readStatus(paths, status);
|
|
3648
|
+
log("library_status", parsed.requested, `level=${s.level}`);
|
|
3649
|
+
return json(s);
|
|
3650
|
+
}
|
|
3651
|
+
log("library_status", parsed.requested, "level=missing");
|
|
3652
|
+
return json({ ...status, entry: paths.entry, entry_dir: paths.entryDir, level: "missing" });
|
|
3653
|
+
}
|
|
3654
|
+
);
|
|
3655
|
+
server2.tool(
|
|
3656
|
+
"library_read",
|
|
3657
|
+
`Read one small piece of a stored paper's Markdown \u2014 never the whole paper. Without section/query: the outline (headings with word counts; says whether the full text or only the abstract is stored). section: a heading from the outline (case-insensitive substring, e.g. "results") \u2192 that section's text, capped at max_chars (default 6000). query: search terms \u2192 paragraphs containing all terms, each with its section, best first, at most max_hits (default 5); with no hit, the headings. Full texts converted from JATS XML have a "Declarations" section \u2014 funding, conflicts of interest, author notes \u2014 so section "Declarations" answers who paid and who declares what in one call; a paper without that section states nothing. For abstract-only entries the status carries coi and funding from PubMed. Call without section/query first, then pick. Reads may run in parallel.`,
|
|
3658
|
+
{
|
|
3659
|
+
id: idSchema,
|
|
3660
|
+
section: z.string().optional().describe("Heading from the outline (case-insensitive substring)."),
|
|
3661
|
+
query: z.string().optional().describe("Search terms; paragraphs containing all of them are returned."),
|
|
3662
|
+
max_chars: z.number().int().positive().optional().describe("Cap for section text (default 6000)."),
|
|
3663
|
+
max_hits: z.number().int().positive().optional().describe("Cap for query passages (default 5).")
|
|
3664
|
+
},
|
|
3665
|
+
async ({ id, section: section2, query, max_chars, max_hits }) => {
|
|
3666
|
+
const parsed = parseId(id);
|
|
3667
|
+
const out = (o2) => ({ content: [{ type: "text", text: JSON.stringify(o2, null, 2) }] });
|
|
3668
|
+
if (parsed.error) {
|
|
3669
|
+
log("library_read", parsed.requested, `error ${parsed.error}`);
|
|
3670
|
+
return out({ requested: parsed.requested, error: parsed.error });
|
|
3671
|
+
}
|
|
3672
|
+
const paths = entryPaths(AI4L_LIBRARY_DIR, parsed.type, parsed.id);
|
|
3673
|
+
if (!entryExists(paths)) {
|
|
3674
|
+
log("library_read", parsed.requested, "error missing");
|
|
3675
|
+
return out({ requested: parsed.requested, entry: paths.entry, error: "missing \u2014 call library_add first" });
|
|
3676
|
+
}
|
|
3677
|
+
const full = existsSync10(paths.fulltextMd);
|
|
3678
|
+
const md = readFileSync9(full ? paths.fulltextMd : paths.md, "utf8");
|
|
3679
|
+
const text3 = full ? "full_text" : "abstract";
|
|
3680
|
+
const base = { requested: parsed.requested, entry: paths.entry, text: text3 };
|
|
3681
|
+
if (section2) {
|
|
3682
|
+
const sec = readSection(md, section2, max_chars ?? 6e3);
|
|
3683
|
+
log("library_read", parsed.requested, `section "${section2}" ${sec ? `\u2192 "${sec.heading}" ${sec.words} words${sec.truncated ? " truncated" : ""}` : "not found"}`);
|
|
3684
|
+
if (!sec) return out({ ...base, error: `no section matching "${section2}"`, outline: outline(md, text3).sections });
|
|
3685
|
+
return out({ ...base, ...sec });
|
|
3686
|
+
}
|
|
3687
|
+
if (query) {
|
|
3688
|
+
const passages = search(md, query, max_hits ?? 5);
|
|
3689
|
+
log("library_read", parsed.requested, `query "${query}" \u2192 ${passages.length} passages`);
|
|
3690
|
+
if (!passages.length) return out({ ...base, query, passages, sections: outline(md, text3).sections.map((s) => s.heading) });
|
|
3691
|
+
return out({ ...base, query, passages });
|
|
3692
|
+
}
|
|
3693
|
+
const o = outline(md, text3);
|
|
3694
|
+
log("library_read", parsed.requested, `outline ${o.sections.length} sections ${o.words} words`);
|
|
3695
|
+
return out({ ...base, ...o });
|
|
3696
|
+
}
|
|
3697
|
+
);
|
|
3698
|
+
server2.tool(
|
|
3699
|
+
"library_verify",
|
|
3700
|
+
`Is one finding of a review stated by one source it cites \u2014 a paper, a drug label (dailymed:\u2026) or a trial record (nct:\u2026)? claim: the finding alone, verbatim from the review, without its citations ("IPSS fell 2.3 points more than placebo in men with ED"); id: one of the papers the review cites for it, which must be in the library (library_add first). A finding cited to three papers is three calls with the same claim. Build the claim as a complete statement: subject, intervention, outcome, number \u2014 when splitting a sentence leaves a fragment, restore the subject from the sentence; cut what the review adds around the finding (where it was published, who funded it, "in two trials", "pooled data", the review's own inference or advice after "so"); one number set per claim, so split at semicolons too. Statements about the literature as a whole ("no trial has tested"), recommendations and history are not claims. A claim that names a source, joins two findings, or is a fragment is refused. The server reads the paper and asks a decision model; nothing of the paper enters your context. verdict: "pass" (the paper states it; score is the probability), "fail" (its full text was read and does not state it), "na" (only its abstract is in the library and that does not state it, or only its title), or "error" (claim empty, too short, a fragment, naming a source or compound; paper not in the library; no model configured; model failure). basis: full_text, abstract or title. glyphs_unresolved: symbols the paper's PDF font gave no character for (a minus, \xB1, \u2264 \u2026) \u2014 when above 0, a fail may be the font, not the paper. superseded: true when the paper carries a stored correction ("Corrected in PubMed abstract") and the claim uses a figure that correction replaced; the verdict is kept, the figure is out of date; null for a paper without a stored correction.`,
|
|
3701
|
+
{
|
|
3702
|
+
claim: z.string().describe("One finding, verbatim from the review, without its citations."),
|
|
3703
|
+
id: idSchema
|
|
3704
|
+
},
|
|
3705
|
+
async ({ claim, id }) => {
|
|
3706
|
+
const t0 = Date.now();
|
|
3707
|
+
const cfg = verifyConfig();
|
|
3708
|
+
const r = await verifyClaim(claim, id, cfg, cfg ? decider(cfg) : null);
|
|
3709
|
+
const secs = `${((Date.now() - t0) / 1e3).toFixed(1).padStart(4, "0")}s`;
|
|
3710
|
+
log("library_verify", r.id, r.error ? `${secs} verdict=${"error".padEnd(5)} ${r.error} | ${claim}` : `${secs} verdict=${r.verdict.padEnd(5)} basis=${r.basis.padEnd(9)} score=${r.score === null ? "- " : r.score.toFixed(3)}${r.windows > 1 ? ` windows=${r.windows}` : ""}${r.superseded ? " superseded" : ""}${r.glyphs_unresolved ? ` glyphs=${r.glyphs_unresolved}` : ""} | ${claim}`);
|
|
3711
|
+
return { content: [{ type: "text", text: JSON.stringify(r, null, 2) }] };
|
|
3712
|
+
}
|
|
3713
|
+
);
|
|
3714
|
+
server2.tool(
|
|
3715
|
+
"library_info",
|
|
3716
|
+
"Library-wide summary \u2014 everything the TUI shows: the library directory and host; the decision model with its endpoint and whether it answers; the entry count and the size on disk (size, size_bytes); per entry type (types) the entries and their levels (full_text, abstract, title) and retractions; entry counts by type and level, counts of retracted and not-citable entries and of entries with errata, expressions of concern and updates, the configured retrieval tiers (enabled/disabled, in order, and where that configuration came from), the decision model behind library_verify (model, endpoint, whether it runs on this machine, and whether it answers right now: ok with the answer time, or the error), the library directory and host. ids=true also lists every entry ID.",
|
|
3717
|
+
{ ids: z.boolean().optional().describe("Also list every entry ID (type:value).") },
|
|
3718
|
+
async ({ ids }) => {
|
|
3719
|
+
const info = libraryInfo(AI4L_LIBRARY_DIR, HOST, ids === true);
|
|
3720
|
+
const cfg = verifyConfig();
|
|
3721
|
+
if (info.verify && cfg) {
|
|
3722
|
+
const t0 = Date.now();
|
|
3723
|
+
try {
|
|
3724
|
+
const a = await decider(cfg)("The sky is blue.", { q: { type: "noul", instructions: "Is the sky blue?" } });
|
|
3725
|
+
info.verify = { ...info.verify, ok: typeof a.q?.noul === "number", ms: Date.now() - t0, error: null };
|
|
3726
|
+
} catch (e) {
|
|
3727
|
+
info.verify = { ...info.verify, ok: false, ms: Date.now() - t0, error: e instanceof Error ? e.message : String(e) };
|
|
3728
|
+
}
|
|
3729
|
+
}
|
|
3730
|
+
log("library_info", null, `entries=${info.entries}${info.verify ? ` verify ${info.verify.model} ${info.verify.ok ? `ok ${info.verify.ms} ms` : `down: ${info.verify.error}`}` : " verify not configured"}`);
|
|
3731
|
+
return { content: [{ type: "text", text: JSON.stringify(info, null, 2) }] };
|
|
3732
|
+
}
|
|
3733
|
+
);
|
|
3734
|
+
server2.tool(
|
|
3735
|
+
"get_version",
|
|
3736
|
+
"Get the running ai4l-library MCP server's package name and version, and the resolved library directory. Useful to confirm which build is loaded.",
|
|
3737
|
+
{},
|
|
3738
|
+
async () => {
|
|
3739
|
+
log("get_version", null, `${pkg.name} ${pkg.version}`);
|
|
3740
|
+
return { content: [{ type: "text", text: `${pkg.name} v${pkg.version} \xB7 host: ${HOST} \xB7 library_dir: ${AI4L_LIBRARY_DIR} \xB7 log: ${LOG_FILE}` }] };
|
|
317
3741
|
}
|
|
318
|
-
);
|
|
319
|
-
server2.tool(
|
|
320
|
-
"get_version",
|
|
321
|
-
"Get the running ai4l-library MCP server's package name and version, and the resolved library directory. Useful to confirm which build is loaded.",
|
|
322
|
-
{},
|
|
323
|
-
async () => ({ content: [{ type: "text", text: `${pkg.name} v${pkg.version} \xB7 library_dir: ${LIBRARY_DIR}` }] })
|
|
324
3742
|
);
|
|
325
3743
|
}
|
|
326
3744
|
|
|
327
3745
|
// src/index.ts
|
|
3746
|
+
loadDotenv();
|
|
328
3747
|
var server = new McpServer(
|
|
329
3748
|
{ name: pkg.name, version: pkg.version },
|
|
330
3749
|
{ instructions: INSTRUCTIONS }
|