@extantape/onephylogeny-acquire 0.0.0-stage → 0.1.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +75 -0
- package/README.md +30 -2
- package/dist/index.d.ts +474 -0
- package/dist/index.js +792 -0
- package/package.json +43 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import { parseGenBank } from "@extantape/onephylogeny-core";
|
|
3
|
+
|
|
4
|
+
// src/papers.ts
|
|
5
|
+
import "@extantape/onephylogeny-core";
|
|
6
|
+
var EUROPE_PMC = "https://www.ebi.ac.uk/europepmc/webservices/rest/search";
|
|
7
|
+
var NOTICE = /retract|correct|erratum|expression of concern/i;
|
|
8
|
+
var normalTitle = (t) => t.toLowerCase().replace(/<[^>]+>/g, "").replace(/[^a-z0-9]+/g, " ").trim();
|
|
9
|
+
var cleanTitle = (t) => t.replace(/<[^>]+>/g, "").replace(/\.$/, "").trim();
|
|
10
|
+
var firstAuthor = (authors) => authors?.split(/[,;]| and /)[0]?.trim().split(/\s+/)[0] ?? "";
|
|
11
|
+
var shortLabel = (authors, year, title) => [firstAuthor(authors) || title.split(/\s+/).slice(0, 3).join(" "), year].filter(Boolean).join(" ");
|
|
12
|
+
var blankPaper = (id, title) => ({ id, label: "", colorIndex: 0, title, sources: [], linked: [], linkLookup: "not-done" });
|
|
13
|
+
var withLabel = (p) => ({ ...p, label: shortLabel(p.authors, p.year, p.title) });
|
|
14
|
+
async function searchEuropePmc(fetch, query, pageSize = 100) {
|
|
15
|
+
const url = `${EUROPE_PMC}?${new URLSearchParams({ query, format: "json", resultType: "core", pageSize: String(pageSize) })}`;
|
|
16
|
+
const res = await fetch(url);
|
|
17
|
+
if (!res.ok) throw new AcquireError(`Europe PMC answered HTTP ${res.status}`);
|
|
18
|
+
const body = JSON.parse(await res.text());
|
|
19
|
+
const results = body.resultList?.result ?? [];
|
|
20
|
+
const papers = results.filter((r) => r.title).map((r) => {
|
|
21
|
+
const types = r.pubTypeList?.pubType ?? [];
|
|
22
|
+
const notices = types.filter((t) => NOTICE.test(t));
|
|
23
|
+
return withLabel({
|
|
24
|
+
...blankPaper(r.pmid ? `pmid:${r.pmid}` : r.doi ? `doi:${String(r.doi).toLowerCase()}` : `epmc:${r.id}`, cleanTitle(r.title)),
|
|
25
|
+
sources: ["europepmc"],
|
|
26
|
+
...r.authorString ? { authors: r.authorString } : {},
|
|
27
|
+
...r.journalInfo?.journal?.title ? { journal: r.journalInfo.journal.title } : {},
|
|
28
|
+
...r.pubYear ? { year: Number(r.pubYear) } : {},
|
|
29
|
+
...r.doi ? { doi: String(r.doi).toLowerCase() } : {},
|
|
30
|
+
...r.pmid ? { pmid: String(r.pmid) } : {},
|
|
31
|
+
openAccess: r.isOpenAccess === "Y",
|
|
32
|
+
...notices.length ? { notices } : {}
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
return { papers, total: Number(body.hitCount ?? papers.length) };
|
|
36
|
+
}
|
|
37
|
+
async function searchPubMed(client, query, retmax = 100) {
|
|
38
|
+
const search = JSON.parse(await client.request("esearch.fcgi", { db: "pubmed", term: query, retmax: String(retmax), retmode: "json" }));
|
|
39
|
+
const ids = search.esearchresult?.idlist ?? [];
|
|
40
|
+
const total = Number(search.esearchresult?.count ?? ids.length);
|
|
41
|
+
if (!ids.length) return { papers: [], total };
|
|
42
|
+
const summary = JSON.parse(await client.request("esummary.fcgi", { db: "pubmed", id: ids.join(","), retmode: "json" })).result ?? {};
|
|
43
|
+
const papers = ids.filter((id) => summary[id]?.title).map((id) => {
|
|
44
|
+
const r = summary[id];
|
|
45
|
+
const authors = (r.authors ?? []).map((a) => a.name).join(", ");
|
|
46
|
+
const doi = (r.articleids ?? []).find((a) => a.idtype === "doi")?.value;
|
|
47
|
+
const year = Number(String(r.pubdate ?? "").slice(0, 4));
|
|
48
|
+
const notices = (r.pubtype ?? []).filter((t) => NOTICE.test(t));
|
|
49
|
+
return withLabel({
|
|
50
|
+
...blankPaper(`pmid:${id}`, cleanTitle(r.title)),
|
|
51
|
+
sources: ["pubmed"],
|
|
52
|
+
pmid: id,
|
|
53
|
+
...authors ? { authors } : {},
|
|
54
|
+
...r.fulljournalname ? { journal: r.fulljournalname } : {},
|
|
55
|
+
...year ? { year } : {},
|
|
56
|
+
...doi ? { doi: String(doi).toLowerCase() } : {},
|
|
57
|
+
...notices.length ? { notices } : {}
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
return { papers, total };
|
|
61
|
+
}
|
|
62
|
+
function studiesFromRecords(records, describe) {
|
|
63
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
64
|
+
for (const r of records) {
|
|
65
|
+
const pub = r.references.find((p) => p.title && p.title !== "Direct Submission");
|
|
66
|
+
const key = pub?.pubmedId ? `pmid:${pub.pubmedId}` : pub ? `genbank:${normalTitle(pub.title)}` : "genbank:no-publication";
|
|
67
|
+
let entry = byKey.get(key);
|
|
68
|
+
if (!entry) {
|
|
69
|
+
const year = Number((pub?.journal ?? "").match(/\b(?:19|20)\d{2}\b/g)?.at(-1));
|
|
70
|
+
const title = pub ? cleanTitle(pub.title) : "Records with no publication listed in GenBank";
|
|
71
|
+
entry = { linked: [], paper: withLabel({
|
|
72
|
+
...blankPaper(key, title),
|
|
73
|
+
sources: ["genbank"],
|
|
74
|
+
...pub?.authors ? { authors: pub.authors } : {},
|
|
75
|
+
...pub?.journal ? { journal: pub.journal } : {},
|
|
76
|
+
...year ? { year } : {},
|
|
77
|
+
...pub?.pubmedId ? { pmid: pub.pubmedId } : {},
|
|
78
|
+
...pub ? {} : { unpublished: true }
|
|
79
|
+
}) };
|
|
80
|
+
if (!pub) entry.paper.label = "No publication";
|
|
81
|
+
byKey.set(key, entry);
|
|
82
|
+
}
|
|
83
|
+
entry.linked.push(describe(r));
|
|
84
|
+
}
|
|
85
|
+
return [...byKey.values()].map(({ paper, linked }) => ({ ...paper, linked, linkLookup: "done" }));
|
|
86
|
+
}
|
|
87
|
+
function mergePapers(lists) {
|
|
88
|
+
const out = [];
|
|
89
|
+
for (const paper of lists.flat()) {
|
|
90
|
+
const same = out.find((p) => paper.pmid && p.pmid === paper.pmid || paper.doi && p.doi === paper.doi);
|
|
91
|
+
if (!same) {
|
|
92
|
+
out.push(paper);
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
same.sources = [.../* @__PURE__ */ new Set([...same.sources, ...paper.sources])];
|
|
96
|
+
same.linked = same.linked.length ? same.linked : paper.linked;
|
|
97
|
+
if (same.linkLookup !== "done") same.linkLookup = paper.linkLookup;
|
|
98
|
+
same.authors ??= paper.authors;
|
|
99
|
+
same.doi ??= paper.doi;
|
|
100
|
+
same.pmid ??= paper.pmid;
|
|
101
|
+
same.journal ??= paper.journal;
|
|
102
|
+
same.year ??= paper.year;
|
|
103
|
+
if (paper.notices) same.notices = [.../* @__PURE__ */ new Set([...same.notices ?? [], ...paper.notices])];
|
|
104
|
+
if (paper.openAccess) same.openAccess = true;
|
|
105
|
+
}
|
|
106
|
+
for (const p of out) {
|
|
107
|
+
const twins = out.filter((q) => q !== p && !p.unpublished && !q.unpublished && normalTitle(q.title) === normalTitle(p.title)).map((q) => q.id);
|
|
108
|
+
if (twins.length) p.possibleSameAs = twins;
|
|
109
|
+
}
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
async function searchPapers(client, fetch, query, records = []) {
|
|
113
|
+
const failed = [];
|
|
114
|
+
const attempt = async (name, work) => {
|
|
115
|
+
try {
|
|
116
|
+
return await work();
|
|
117
|
+
} catch (e) {
|
|
118
|
+
failed.push(`${name}: ${e instanceof Error ? e.message : String(e)}`);
|
|
119
|
+
return { papers: [], total: 0 };
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
const empty = !query.trim();
|
|
123
|
+
const pubmed = empty ? { papers: [], total: 0 } : await attempt("PubMed", () => searchPubMed(client, query));
|
|
124
|
+
const europe = empty ? { papers: [], total: 0 } : await attempt("Europe PMC", () => searchEuropePmc(fetch, query));
|
|
125
|
+
return {
|
|
126
|
+
papers: mergePapers([records, pubmed.papers, europe.papers]),
|
|
127
|
+
counts: { PubMed: pubmed.papers.length, "Europe PMC": europe.papers.length, "GenBank studies": records.length },
|
|
128
|
+
totals: { PubMed: pubmed.total, "Europe PMC": europe.total, "GenBank studies": records.length },
|
|
129
|
+
sources: [...empty ? [] : ["PubMed", "Europe PMC"], ...records.length ? ["GenBank"] : []],
|
|
130
|
+
failed
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
async function linkedAccessions(client, pmid) {
|
|
134
|
+
const r = JSON.parse(await client.request("elink.fcgi", { dbfrom: "pubmed", db: "nuccore", linkname: "pubmed_nuccore", id: pmid, retmode: "json" }));
|
|
135
|
+
const ids = r.linksets?.[0]?.linksetdbs?.[0]?.links ?? [];
|
|
136
|
+
if (!ids.length) return [];
|
|
137
|
+
const out = [];
|
|
138
|
+
for (let i = 0; i < ids.length; i += 200) {
|
|
139
|
+
const s = JSON.parse(await client.request("esummary.fcgi", { db: "nuccore", id: ids.slice(i, i + 200).join(","), retmode: "json" })).result ?? {};
|
|
140
|
+
for (const id of s.uids ?? []) if (s[id]?.accessionversion) out.push(s[id].accessionversion);
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
var taxaTerm = (selection) => selection.taxa.filter((t) => t.source === "ncbi-taxonomy").map((t) => `txid${t.id}[Organism:exp]`).join(" OR ");
|
|
145
|
+
async function classifyAccessions(client, accessions, selection) {
|
|
146
|
+
const inTarget = /* @__PURE__ */ new Set();
|
|
147
|
+
const hasMarker = /* @__PURE__ */ new Set();
|
|
148
|
+
const genomeTerm = genomesHoldingMarkerQuery(selection) ? " OR (mitochondrion[Filter] AND 10000:25000[SLEN])" : "";
|
|
149
|
+
for (let i = 0; i < accessions.length; i += 150) {
|
|
150
|
+
const chunk = accessions.slice(i, i + 150);
|
|
151
|
+
const list = `(${chunk.map((a) => `${a}[Accession]`).join(" OR ")})`;
|
|
152
|
+
for (const a of await client.searchAccessions(`${list} AND (${taxaTerm(selection)})`)) inTarget.add(a);
|
|
153
|
+
for (const a of await client.searchAccessions(`${list} AND ((${markerTerm(selection.marker)})${genomeTerm})`)) hasMarker.add(a);
|
|
154
|
+
}
|
|
155
|
+
return { inTarget, hasMarker };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// src/lookup.ts
|
|
159
|
+
var CROSSREF = "https://api.crossref.org/works";
|
|
160
|
+
var EUROPE_PMC2 = "https://www.ebi.ac.uk/europepmc/webservices/rest";
|
|
161
|
+
var LOOKUP_USER_AGENT = "OnePhylogeny (https://github.com/ExtantApe/OnePhylogeny)";
|
|
162
|
+
var words = (t) => new Set(t.toLowerCase().replace(/<[^>]+>/g, "").replace(/[^a-z0-9]+/g, " ").trim().split(" ").filter(Boolean));
|
|
163
|
+
function titleMatch(a, b) {
|
|
164
|
+
const x = words(a);
|
|
165
|
+
const y = words(b);
|
|
166
|
+
const shared = [...x].filter((w) => y.has(w)).length;
|
|
167
|
+
const all = (/* @__PURE__ */ new Set([...x, ...y])).size;
|
|
168
|
+
return { score: all ? Math.round(shared / all * 1e3) / 1e3 : 0, exact: all > 0 && shared === all };
|
|
169
|
+
}
|
|
170
|
+
function paced(fetch, gapMs = 1e3, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now()) {
|
|
171
|
+
let last = -Infinity;
|
|
172
|
+
let queue = Promise.resolve();
|
|
173
|
+
return (url, init) => {
|
|
174
|
+
const run = queue.then(async () => {
|
|
175
|
+
const wait = last + gapMs - now();
|
|
176
|
+
if (wait > 0) await sleep(wait);
|
|
177
|
+
last = now();
|
|
178
|
+
return fetch(url, { ...init, headers: { "User-Agent": LOOKUP_USER_AGENT, ...init?.headers } });
|
|
179
|
+
});
|
|
180
|
+
queue = run.then(() => void 0, () => void 0);
|
|
181
|
+
return run;
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
async function getJson(fetch, url, service) {
|
|
185
|
+
const res = await fetch(url);
|
|
186
|
+
if (!res.ok) throw new AcquireError(`${service} answered HTTP ${res.status}`);
|
|
187
|
+
return JSON.parse(await res.text());
|
|
188
|
+
}
|
|
189
|
+
async function lookupCrossref(fetch, paper, at, rows = 5) {
|
|
190
|
+
const params = new URLSearchParams({ "query.bibliographic": paper.title, rows: String(rows), select: "DOI,title,issued,container-title" });
|
|
191
|
+
if (paper.authors) params.set("query.author", paper.authors.split(/[,;]| and /)[0].trim());
|
|
192
|
+
const url = `${CROSSREF}?${params}`;
|
|
193
|
+
const body = await getJson(fetch, url, "Crossref");
|
|
194
|
+
return (body.message?.items ?? []).filter((i) => i.title?.[0] && i.DOI).map((i) => {
|
|
195
|
+
const m = titleMatch(paper.title, i.title[0]);
|
|
196
|
+
const year = i.issued?.["date-parts"]?.[0]?.[0];
|
|
197
|
+
return {
|
|
198
|
+
service: "crossref",
|
|
199
|
+
title: i.title[0],
|
|
200
|
+
doi: String(i.DOI).toLowerCase(),
|
|
201
|
+
titleMatch: m.score,
|
|
202
|
+
exactTitle: m.exact,
|
|
203
|
+
...year ? { year: Number(year) } : {},
|
|
204
|
+
...i["container-title"]?.[0] ? { journal: i["container-title"][0] } : {},
|
|
205
|
+
source: { kind: "publication", citation: `Crossref ${url}`, retrievedAt: at }
|
|
206
|
+
};
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
async function lookupEuropePmc(fetch, paper, at, pageSize = 5) {
|
|
210
|
+
const title = paper.title.replace(/["\\]/g, " ");
|
|
211
|
+
const url = `${EUROPE_PMC2}/search?${new URLSearchParams({ query: `TITLE:"${title}"`, format: "json", resultType: "lite", pageSize: String(pageSize) })}`;
|
|
212
|
+
const body = await getJson(fetch, url, "Europe PMC");
|
|
213
|
+
return (body.resultList?.result ?? []).filter((r) => r.title).map((r) => {
|
|
214
|
+
const m = titleMatch(paper.title, r.title);
|
|
215
|
+
const open = r.isOpenAccess === "Y" && r.pmcid;
|
|
216
|
+
return {
|
|
217
|
+
service: "europepmc",
|
|
218
|
+
title: String(r.title).replace(/\.$/, ""),
|
|
219
|
+
titleMatch: m.score,
|
|
220
|
+
exactTitle: m.exact,
|
|
221
|
+
...r.pubYear ? { year: Number(r.pubYear) } : {},
|
|
222
|
+
...r.journalTitle ? { journal: r.journalTitle } : {},
|
|
223
|
+
...r.doi ? { doi: String(r.doi).toLowerCase() } : {},
|
|
224
|
+
...r.pmid ? { pmid: String(r.pmid) } : {},
|
|
225
|
+
...r.pmcid ? { pmcid: String(r.pmcid) } : {},
|
|
226
|
+
...open ? { supplementsUrl: `${EUROPE_PMC2}/${r.pmcid}/supplementaryFiles` } : {},
|
|
227
|
+
source: { kind: "publication", citation: `Europe PMC ${url}`, retrievedAt: at }
|
|
228
|
+
};
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
async function lookupPaper(fetch, paper, options = {}) {
|
|
232
|
+
const at = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
233
|
+
const failed = [];
|
|
234
|
+
const attempt = async (name, work) => {
|
|
235
|
+
try {
|
|
236
|
+
return await work();
|
|
237
|
+
} catch (e) {
|
|
238
|
+
failed.push(`${name}: ${e.message}`);
|
|
239
|
+
return [];
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
const all = [...await attempt("Crossref", () => lookupCrossref(fetch, paper, at)), ...await attempt("Europe PMC", () => lookupEuropePmc(fetch, paper, at))];
|
|
243
|
+
const proposals = all.filter((p) => p.titleMatch >= (options.minMatch ?? 0.5)).sort((a, b) => Number(b.exactTitle) - Number(a.exactTitle) || b.titleMatch - a.titleMatch || a.service.localeCompare(b.service));
|
|
244
|
+
return { proposals, failed };
|
|
245
|
+
}
|
|
246
|
+
function approveLink(paper, proposal, meta) {
|
|
247
|
+
if (paper.doi && proposal.doi && paper.doi !== proposal.doi) throw new AcquireError(`This study already has DOI ${paper.doi}; remove it first to replace it`);
|
|
248
|
+
if (paper.pmid && proposal.pmid && paper.pmid !== proposal.pmid) throw new AcquireError(`This study already has PMID ${paper.pmid}; remove it first to replace it`);
|
|
249
|
+
return {
|
|
250
|
+
...paper,
|
|
251
|
+
...proposal.doi ? { doi: proposal.doi } : {},
|
|
252
|
+
...proposal.pmid ? { pmid: proposal.pmid } : {},
|
|
253
|
+
linkSource: { ...proposal.doi ? { doi: proposal.doi } : {}, ...proposal.pmid ? { pmid: proposal.pmid } : {}, source: { ...proposal.source, ...meta.by ? { approvedBy: meta.by } : {}, approvedAt: meta.at } }
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// src/sources.ts
|
|
258
|
+
var EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/";
|
|
259
|
+
var DATASETS = "https://api.ncbi.nlm.nih.gov/datasets/v2/";
|
|
260
|
+
var GBIF = "https://api.gbif.org/v1/";
|
|
261
|
+
var IDIGBIO = "https://search.idigbio.org/v2/";
|
|
262
|
+
var BOLD = "https://portal.boldsystems.org/api/";
|
|
263
|
+
var pacedFor = (fetch, o) => paced(fetch, Math.max(1e3, o.gapMs ?? 1e3), o.sleep, o.now);
|
|
264
|
+
async function getJson2(fetch, url, service, requests) {
|
|
265
|
+
requests.push(url);
|
|
266
|
+
const res = await fetch(url);
|
|
267
|
+
if (res.status === 403 || res.status === 429) throw new AcquireError(`${service} refused the request (HTTP ${res.status}). Not retried; try again later`);
|
|
268
|
+
if (!res.ok) throw new AcquireError(`${service} answered HTTP ${res.status}`);
|
|
269
|
+
const text2 = await res.text();
|
|
270
|
+
try {
|
|
271
|
+
return JSON.parse(text2);
|
|
272
|
+
} catch {
|
|
273
|
+
throw new AcquireError(`${service} did not return JSON`);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
var eutilsUrl = (path, params) => `${EUTILS}${path}?${new URLSearchParams({ ...params, tool: "onephylogeny" })}`;
|
|
277
|
+
async function eutils(client, path, params, requests) {
|
|
278
|
+
requests.push(eutilsUrl(path, params));
|
|
279
|
+
const text2 = await client.request(path, { ...params, retmode: "json" });
|
|
280
|
+
try {
|
|
281
|
+
return JSON.parse(text2);
|
|
282
|
+
} catch {
|
|
283
|
+
throw new AcquireError(`${path} did not return JSON`);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
function ncbiTaxonTerm(taxon) {
|
|
287
|
+
const t = taxon.trim();
|
|
288
|
+
if (!t) throw new AcquireError("Name a taxon");
|
|
289
|
+
return /^\d+$/.test(t) ? `txid${t}[Organism:exp]` : `"${t.replace(/"/g, "")}"[Organism:exp]`;
|
|
290
|
+
}
|
|
291
|
+
var decode = (s) => s.replace(/</g, "<").replace(/>/g, ">").replace(/"/g, '"').replace(/'/g, "'").replace(/&/g, "&").trim();
|
|
292
|
+
var attr = (xml, tag, name) => {
|
|
293
|
+
const el = new RegExp(`<${tag}\\b[^>]*>`).exec(xml)?.[0];
|
|
294
|
+
const v = el && new RegExp(`\\b${name}="([^"]*)"`).exec(el)?.[1];
|
|
295
|
+
return v ? decode(v) : void 0;
|
|
296
|
+
};
|
|
297
|
+
var text = (xml, tag) => {
|
|
298
|
+
const v = new RegExp(`<${tag}\\b[^>]*>([^<]*)</${tag}>`).exec(xml)?.[1];
|
|
299
|
+
return v ? decode(v) : void 0;
|
|
300
|
+
};
|
|
301
|
+
var num = (v) => v && /^\d+$/.test(v) ? Number(v) : void 0;
|
|
302
|
+
function parseSraSummary(doc) {
|
|
303
|
+
const x = doc.expxml ?? "";
|
|
304
|
+
const layout = /<LIBRARY_LAYOUT>\s*<(PAIRED|SINGLE)\b/.exec(x)?.[1];
|
|
305
|
+
const shared = {
|
|
306
|
+
...attr(x, "Experiment", "acc") ? { experiment: attr(x, "Experiment", "acc") } : {},
|
|
307
|
+
...attr(x, "Study", "acc") ? { study: attr(x, "Study", "acc") } : {},
|
|
308
|
+
...text(x, "Bioproject") ? { bioProject: text(x, "Bioproject") } : {},
|
|
309
|
+
...text(x, "Biosample") ? { bioSample: text(x, "Biosample") } : {},
|
|
310
|
+
...attr(x, "Organism", "ScientificName") ? { organism: attr(x, "Organism", "ScientificName") } : {},
|
|
311
|
+
...attr(x, "Organism", "taxid") ? { taxid: attr(x, "Organism", "taxid") } : {},
|
|
312
|
+
...text(x, "LIBRARY_STRATEGY") ? { libraryStrategy: text(x, "LIBRARY_STRATEGY") } : {},
|
|
313
|
+
...text(x, "LIBRARY_SOURCE") ? { librarySource: text(x, "LIBRARY_SOURCE") } : {},
|
|
314
|
+
...text(x, "LIBRARY_SELECTION") ? { librarySelection: text(x, "LIBRARY_SELECTION") } : {},
|
|
315
|
+
...layout ? { layout } : {},
|
|
316
|
+
...text(x, "Platform") ? { platform: text(x, "Platform") } : {},
|
|
317
|
+
...attr(x, "Platform", "instrument_model") ? { instrument: attr(x, "Platform", "instrument_model") } : {},
|
|
318
|
+
...doc.createdate ? { published: doc.createdate } : {}
|
|
319
|
+
};
|
|
320
|
+
return [...(doc.runs ?? "").matchAll(/<Run\b[^>]*>/g)].flatMap(([el]) => {
|
|
321
|
+
const run = /\bacc="([^"]+)"/.exec(el)?.[1];
|
|
322
|
+
if (!run) return [];
|
|
323
|
+
const bases = num(/\btotal_bases="([^"]*)"/.exec(el)?.[1]);
|
|
324
|
+
const spots = num(/\btotal_spots="([^"]*)"/.exec(el)?.[1]);
|
|
325
|
+
return [{ run, ...shared, ...bases !== void 0 ? { bases } : {}, ...spots !== void 0 ? { spots } : {} }];
|
|
326
|
+
});
|
|
327
|
+
}
|
|
328
|
+
async function listSraRuns(client, taxon, options = {}) {
|
|
329
|
+
const requests = [];
|
|
330
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
331
|
+
const term = ncbiTaxonTerm(taxon);
|
|
332
|
+
const search = await eutils(client, "esearch.fcgi", { db: "sra", term, retmax: String(options.limit ?? 500) }, requests);
|
|
333
|
+
const res = search.esearchresult;
|
|
334
|
+
if (!res || res.ERROR) throw new AcquireError(`SRA search failed: ${String(res?.ERROR ?? "no result")}`);
|
|
335
|
+
const ids = res.idlist ?? [];
|
|
336
|
+
const runs = [];
|
|
337
|
+
for (let i = 0; i < ids.length; i += 200) {
|
|
338
|
+
const r = (await eutils(client, "esummary.fcgi", { db: "sra", id: ids.slice(i, i + 200).join(",") }, requests)).result ?? {};
|
|
339
|
+
for (const id of ids.slice(i, i + 200)) if (r[id]) runs.push(...parseSraSummary(r[id]));
|
|
340
|
+
}
|
|
341
|
+
return { source: "NCBI SRA", retrieved, query: requests[0], requests, taxon, experiments: { total: Number(res.count ?? ids.length), listed: ids.length }, runs };
|
|
342
|
+
}
|
|
343
|
+
var BIOSAMPLE_ATTRIBUTES = ["isolate", "breed", "sex", "dev_stage", "tissue", "sample_type", "specimen_voucher", "geo_loc_name", "country", "collection_date", "isolation_source", "host"];
|
|
344
|
+
var COORDINATE_NAME = /(^|[_\s])(lat|long?|latitude|longitude|coords?|coordinates?|gps|geo_?point|utm|easting|northing)([_\s]|$)/i;
|
|
345
|
+
var COORDINATE_VALUE = /-?\d{1,3}\.\d+\s*°?\s*[NSns]?\s*[,;\s]\s*-?\d{1,3}\.\d+\s*°?\s*[EWew]?|\d{1,3}\.\d+\s*°?\s*[NSns]\b|\d{1,3}\.\d+\s*°?\s*[EWew]\b|\d{1,3}\s*°\s*\d{1,2}|\d{1,3}\s*[°d]\s*\d{1,2}\s*['′m]/;
|
|
346
|
+
var PLACEHOLDER = /^(missing|not (collected|applicable|provided|available)|unknown|n\/?a|none|restricted access)$/i;
|
|
347
|
+
function looksLikeCoordinates(value) {
|
|
348
|
+
return COORDINATE_VALUE.test(value);
|
|
349
|
+
}
|
|
350
|
+
function parseBioSample(doc) {
|
|
351
|
+
const attributes = {};
|
|
352
|
+
const withheld = [];
|
|
353
|
+
const allowed = new Set(BIOSAMPLE_ATTRIBUTES);
|
|
354
|
+
for (const m of String(doc.sampledata ?? "").matchAll(/<Attribute\b([^>]*)>([^<]*)<\/Attribute>/g)) {
|
|
355
|
+
const name = /\bharmonized_name="([^"]+)"/.exec(m[1])?.[1] ?? /\battribute_name="([^"]+)"/.exec(m[1])?.[1];
|
|
356
|
+
if (!name) continue;
|
|
357
|
+
if (COORDINATE_NAME.test(name)) {
|
|
358
|
+
withheld.push(name);
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
if (!allowed.has(name)) continue;
|
|
362
|
+
const value = decode(m[2]);
|
|
363
|
+
if (looksLikeCoordinates(value)) {
|
|
364
|
+
withheld.push(name);
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
if (value && !PLACEHOLDER.test(value)) attributes[name] = value;
|
|
368
|
+
}
|
|
369
|
+
return {
|
|
370
|
+
accession: String(doc.accession),
|
|
371
|
+
...doc.organism ? { organism: String(doc.organism) } : {},
|
|
372
|
+
...doc.taxonomy ? { taxid: String(doc.taxonomy) } : {},
|
|
373
|
+
...doc.publicationdate ? { published: String(doc.publicationdate) } : {},
|
|
374
|
+
...doc.package ? { package: String(doc.package) } : {},
|
|
375
|
+
attributes,
|
|
376
|
+
withheld: [...new Set(withheld)].sort()
|
|
377
|
+
};
|
|
378
|
+
}
|
|
379
|
+
async function biosampleFor(client, accessions, options = {}) {
|
|
380
|
+
const requests = [];
|
|
381
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
382
|
+
const wanted = [...new Set(accessions.map((a) => a.trim()).filter(Boolean))];
|
|
383
|
+
for (const a of wanted) if (!/^SAM[NED][A-Z]?\d+$/.test(a)) throw new AcquireError(`${a} isn't a BioSample accession`);
|
|
384
|
+
const samples = [];
|
|
385
|
+
for (let i = 0; i < wanted.length; i += 100) {
|
|
386
|
+
const chunk = wanted.slice(i, i + 100);
|
|
387
|
+
const search = await eutils(client, "esearch.fcgi", { db: "biosample", term: chunk.map((a) => `${a}[accn]`).join(" OR "), retmax: String(chunk.length * 2) }, requests);
|
|
388
|
+
const ids = search.esearchresult?.idlist ?? [];
|
|
389
|
+
if (!ids.length) continue;
|
|
390
|
+
const r = (await eutils(client, "esummary.fcgi", { db: "biosample", id: ids.join(",") }, requests)).result ?? {};
|
|
391
|
+
for (const id of ids) if (r[id]?.accession) samples.push(parseBioSample(r[id]));
|
|
392
|
+
}
|
|
393
|
+
const got = new Set(samples.map((s) => s.accession));
|
|
394
|
+
return { source: "NCBI BioSample", retrieved, query: requests[0] ?? eutilsUrl("esearch.fcgi", { db: "biosample" }), requests, samples, missing: wanted.filter((a) => !got.has(a)) };
|
|
395
|
+
}
|
|
396
|
+
function parseGenomeReport(r) {
|
|
397
|
+
const info = r.assembly_info ?? {};
|
|
398
|
+
const organelles = [...new Set((r.organelle_info ?? []).map((o) => String(o.description ?? "")).filter(Boolean))];
|
|
399
|
+
const db = String(r.source_database ?? "");
|
|
400
|
+
const length = Number(r.assembly_stats?.total_sequence_length);
|
|
401
|
+
return {
|
|
402
|
+
accession: String(r.accession),
|
|
403
|
+
database: /REFSEQ/i.test(db) ? "refseq" : /GENBANK/i.test(db) ? "genbank" : "unknown",
|
|
404
|
+
...r.paired_accession ? { pairedAccession: String(r.paired_accession) } : info.paired_assembly?.accession ? { pairedAccession: String(info.paired_assembly.accession) } : {},
|
|
405
|
+
...r.organism?.organism_name ? { organism: String(r.organism.organism_name) } : {},
|
|
406
|
+
...r.organism?.tax_id ? { taxid: String(r.organism.tax_id) } : {},
|
|
407
|
+
...info.assembly_name ? { assemblyName: String(info.assembly_name) } : {},
|
|
408
|
+
...info.assembly_level ? { assemblyLevel: String(info.assembly_level) } : {},
|
|
409
|
+
...info.assembly_status ? { status: String(info.assembly_status) } : {},
|
|
410
|
+
...Number.isFinite(length) && length > 0 ? { length } : {},
|
|
411
|
+
...info.release_date ? { released: String(info.release_date) } : {},
|
|
412
|
+
organelles,
|
|
413
|
+
mitogenome: organelles.some((o) => /mitochondri/i.test(o)) ? "listed" : "not listed"
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
async function datasetsGenomes(fetch, taxon, options = {}) {
|
|
417
|
+
const t = taxon.trim();
|
|
418
|
+
if (!t) throw new AcquireError("Name a taxon");
|
|
419
|
+
const f = pacedFor(fetch, options);
|
|
420
|
+
const requests = [];
|
|
421
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
422
|
+
const genomes = [];
|
|
423
|
+
let total = 0;
|
|
424
|
+
let token;
|
|
425
|
+
for (let page = 0; page < (options.maxPages ?? 10); page++) {
|
|
426
|
+
const params = new URLSearchParams({ page_size: String(options.pageSize ?? 100), ...token ? { page_token: token } : {} });
|
|
427
|
+
const body = await getJson2(f, `${DATASETS}genome/taxon/${encodeURIComponent(t)}/dataset_report?${params}`, "NCBI Datasets", requests);
|
|
428
|
+
total = Number(body.total_count ?? 0);
|
|
429
|
+
genomes.push(...(body.reports ?? []).filter((r) => r.accession).map(parseGenomeReport));
|
|
430
|
+
token = body.next_page_token;
|
|
431
|
+
if (!token) break;
|
|
432
|
+
}
|
|
433
|
+
return { source: "NCBI Datasets", retrieved, query: requests[0], requests, taxon: t, total, genomes };
|
|
434
|
+
}
|
|
435
|
+
var facet = (facets, field) => ((facets ?? []).find((f) => f.field === field)?.counts ?? []).map((c) => ({ name: String(c.name), count: Number(c.count) })).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
436
|
+
var sum = (counts) => counts.reduce((n, c) => n + c.count, 0);
|
|
437
|
+
async function gbifCoverage(fetch, name, options = {}) {
|
|
438
|
+
const n = name.trim();
|
|
439
|
+
if (!n) throw new AcquireError("Name a taxon");
|
|
440
|
+
const f = pacedFor(fetch, options);
|
|
441
|
+
const requests = [];
|
|
442
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
443
|
+
const m = await getJson2(f, `${GBIF}species/match?${new URLSearchParams({ name: n })}`, "GBIF", requests);
|
|
444
|
+
if (!m.usageKey || m.matchType === "NONE") throw new AcquireError(`GBIF has no match for ${n}`);
|
|
445
|
+
if (m.matchType === "HIGHERRANK") throw new AcquireError(`GBIF matched ${n} only at a higher rank (${m.rank} ${m.canonicalName ?? m.scientificName}). Not counted`);
|
|
446
|
+
const key = String(m.usageKey);
|
|
447
|
+
const params = new URLSearchParams({ taxonKey: key, limit: "0" });
|
|
448
|
+
for (const fct of ["country", "basisOfRecord", "license", "datasetKey"]) params.append("facet", fct);
|
|
449
|
+
params.set("facetLimit", "300");
|
|
450
|
+
params.set("datasetKey.facetLimit", "1000");
|
|
451
|
+
const url = `${GBIF}occurrence/search?${params}`;
|
|
452
|
+
const body = await getJson2(f, url, "GBIF", requests);
|
|
453
|
+
const total = Number(body.count ?? 0);
|
|
454
|
+
const byCountry = facet(body.facets, "COUNTRY");
|
|
455
|
+
const datasets = facet(body.facets, "DATASET_KEY").length;
|
|
456
|
+
const notes = ["Counts include synonyms and subtaxa GBIF files under this name.", "LIVING_SPECIMEN is an animal in a zoo or collection, not a wild record."];
|
|
457
|
+
if (m.matchType !== "EXACT") notes.unshift(`GBIF matched "${n}" to ${m.scientificName} (${m.matchType}). Check it's the taxon you meant.`);
|
|
458
|
+
if (datasets >= 1e3) notes.push("At least 1000 datasets; the count stops there.");
|
|
459
|
+
return {
|
|
460
|
+
source: "GBIF",
|
|
461
|
+
retrieved,
|
|
462
|
+
query: url,
|
|
463
|
+
requests,
|
|
464
|
+
matched: { name: String(m.scientificName ?? n), key, ...m.rank ? { rank: String(m.rank) } : {}, ...m.matchType ? { matchType: String(m.matchType) } : {} },
|
|
465
|
+
total,
|
|
466
|
+
byCountry,
|
|
467
|
+
noCountry: Math.max(0, total - sum(byCountry)),
|
|
468
|
+
byBasisOfRecord: facet(body.facets, "BASIS_OF_RECORD"),
|
|
469
|
+
licences: facet(body.facets, "LICENSE"),
|
|
470
|
+
datasets,
|
|
471
|
+
notes
|
|
472
|
+
};
|
|
473
|
+
}
|
|
474
|
+
var topCounts = (o) => Object.entries(o ?? {}).filter(([k, v]) => k !== "itemCount" && v && typeof v === "object").map(([k, v]) => ({ name: k, count: Number(v.itemCount ?? 0) })).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
475
|
+
async function idigbioCoverage(fetch, name, options = {}) {
|
|
476
|
+
const n = name.trim();
|
|
477
|
+
if (!n) throw new AcquireError("Name a taxon");
|
|
478
|
+
const f = pacedFor(fetch, options);
|
|
479
|
+
const requests = [];
|
|
480
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
481
|
+
const rq = JSON.stringify({ scientificname: n.toLowerCase() });
|
|
482
|
+
const top = (field, count) => `${IDIGBIO}summary/top/records/?${new URLSearchParams({ rq, top_fields: JSON.stringify([field]), count: String(count) })}`;
|
|
483
|
+
const countries = await getJson2(f, top("country", 300), "iDigBio", requests);
|
|
484
|
+
const basis = await getJson2(f, top("basisofrecord", 50), "iDigBio", requests);
|
|
485
|
+
const total = Number(countries.itemCount ?? 0);
|
|
486
|
+
const byCountry = topCounts(countries.country);
|
|
487
|
+
return {
|
|
488
|
+
source: "iDigBio",
|
|
489
|
+
retrieved,
|
|
490
|
+
query: requests[0],
|
|
491
|
+
requests,
|
|
492
|
+
matched: { name: n.toLowerCase(), matchType: "scientificname" },
|
|
493
|
+
total,
|
|
494
|
+
byCountry,
|
|
495
|
+
noCountry: Math.max(0, total - sum(byCountry)),
|
|
496
|
+
byBasisOfRecord: topCounts(basis.basisofrecord),
|
|
497
|
+
licences: null,
|
|
498
|
+
notes: [
|
|
499
|
+
"Countries are as the records spell them, so one country can appear under several names.",
|
|
500
|
+
"Matched on the scientific name as written on records; synonyms are not included.",
|
|
501
|
+
"iDigBio doesn't index a licence per record. Each recordset states its own rights."
|
|
502
|
+
]
|
|
503
|
+
};
|
|
504
|
+
}
|
|
505
|
+
async function occurrenceCoverage(fetch, name, options = {}) {
|
|
506
|
+
const failed = [];
|
|
507
|
+
const attempt = async (service, work) => {
|
|
508
|
+
try {
|
|
509
|
+
return await work();
|
|
510
|
+
} catch (e) {
|
|
511
|
+
failed.push(`${service}: ${e.message}`);
|
|
512
|
+
return null;
|
|
513
|
+
}
|
|
514
|
+
};
|
|
515
|
+
const at = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
516
|
+
const gbif = await attempt("GBIF", () => gbifCoverage(fetch, name, { ...options, at }));
|
|
517
|
+
const idigbio = options.idigbio === false ? null : await attempt("iDigBio", () => idigbioCoverage(fetch, name, { ...options, at }));
|
|
518
|
+
return { taxon: name.trim(), gbif, idigbio, failed };
|
|
519
|
+
}
|
|
520
|
+
async function boldCoverage(fetch, name, options = {}) {
|
|
521
|
+
const n = name.trim().replace(/[;:]/g, " ");
|
|
522
|
+
if (!n.trim()) throw new AcquireError("Name a taxon");
|
|
523
|
+
const f = pacedFor(fetch, options);
|
|
524
|
+
const requests = [];
|
|
525
|
+
const retrieved = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
526
|
+
const pre = await getJson2(f, `${BOLD}query/preprocessor?${new URLSearchParams({ query: `tax:${n}` })}`, "BOLD", requests);
|
|
527
|
+
const matched = (pre.successful_terms ?? []).map((t) => t.matched).find((t) => typeof t === "string" && t.startsWith("tax:"));
|
|
528
|
+
if (!matched) throw new AcquireError(`BOLD has no taxon named ${n}`);
|
|
529
|
+
const url = `${BOLD}summary?${new URLSearchParams({ query: matched, fields: "country/ocean,marker_code" })}`;
|
|
530
|
+
const body = await getJson2(f, url, "BOLD", requests);
|
|
531
|
+
const counts = (o) => Object.entries(o ?? {}).map(([k, v]) => ({ name: k, count: Number(v) })).sort((a, b) => b.count - a.count || a.name.localeCompare(b.name));
|
|
532
|
+
return {
|
|
533
|
+
source: "BOLD",
|
|
534
|
+
retrieved,
|
|
535
|
+
query: url,
|
|
536
|
+
requests,
|
|
537
|
+
matched,
|
|
538
|
+
byCountry: counts(body["country/ocean"]),
|
|
539
|
+
byMarker: counts(body.marker_code),
|
|
540
|
+
notes: ["BOLD's summary gives no total or licence, so neither is reported. Check BOLD's terms before reusing its records."]
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// src/listings.ts
|
|
545
|
+
var LISTING_KINDS = ["sra", "genomes", "occurrences", "bold"];
|
|
546
|
+
async function publicListings(services, taxon, options = {}) {
|
|
547
|
+
const t = taxon.trim();
|
|
548
|
+
if (!t) throw new AcquireError("Name a taxon");
|
|
549
|
+
const asked = LISTING_KINDS.filter((k) => !options.only?.length || options.only.includes(k));
|
|
550
|
+
const at = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
|
|
551
|
+
const o = { ...options, at };
|
|
552
|
+
const out = { taxon: t, name: null, nameFrom: null, asked, retrieved: at, sra: null, genomes: null, gbif: null, idigbio: null, bold: null, failed: [] };
|
|
553
|
+
const attempt = async (service, work) => {
|
|
554
|
+
try {
|
|
555
|
+
return await work();
|
|
556
|
+
} catch (e) {
|
|
557
|
+
out.failed.push({ service, message: e instanceof Error ? e.message : String(e) });
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
if (asked.includes("sra")) out.sra = await attempt("NCBI SRA", () => listSraRuns(services.client, t, { at, ...options.sraLimit ? { limit: options.sraLimit } : {} }));
|
|
562
|
+
if (asked.includes("genomes")) out.genomes = await attempt("NCBI Datasets", () => datasetsGenomes(services.fetch, t, o));
|
|
563
|
+
const byName = [
|
|
564
|
+
...asked.includes("occurrences") ? [{ kind: "occurrences", service: "GBIF" }, { kind: "occurrences", service: "iDigBio" }] : [],
|
|
565
|
+
...asked.includes("bold") ? [{ kind: "bold", service: "BOLD" }] : []
|
|
566
|
+
];
|
|
567
|
+
if (!byName.length) return out;
|
|
568
|
+
if (/^\d+$/.test(t)) {
|
|
569
|
+
try {
|
|
570
|
+
out.name = (await services.client.lineage(t)).scientificName;
|
|
571
|
+
out.nameFrom = "ncbi-taxonomy";
|
|
572
|
+
} catch (e) {
|
|
573
|
+
const why = `needs a scientific name, and NCBI Taxonomy couldn't name taxon ${t}: ${e instanceof Error ? e.message : String(e)}`;
|
|
574
|
+
for (const { service } of byName) out.failed.push({ service, message: `${service} ${why}` });
|
|
575
|
+
return out;
|
|
576
|
+
}
|
|
577
|
+
} else {
|
|
578
|
+
out.name = t;
|
|
579
|
+
out.nameFrom = "given";
|
|
580
|
+
}
|
|
581
|
+
const name = out.name;
|
|
582
|
+
if (asked.includes("occurrences")) {
|
|
583
|
+
out.gbif = await attempt("GBIF", () => gbifCoverage(services.fetch, name, o));
|
|
584
|
+
out.idigbio = await attempt("iDigBio", () => idigbioCoverage(services.fetch, name, o));
|
|
585
|
+
}
|
|
586
|
+
if (asked.includes("bold")) out.bold = await attempt("BOLD", () => boldCoverage(services.fetch, name, o));
|
|
587
|
+
return out;
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
// src/index.ts
|
|
591
|
+
var AcquireError = class extends Error {
|
|
592
|
+
};
|
|
593
|
+
var BASE = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/";
|
|
594
|
+
var NcbiClient = class {
|
|
595
|
+
o;
|
|
596
|
+
constructor(options) {
|
|
597
|
+
this.o = { delayMs: options.apiKey ? 120 : 380, retries: 4, sleep: (ms) => new Promise((r) => setTimeout(r, ms)), ...options };
|
|
598
|
+
}
|
|
599
|
+
async request(path, params) {
|
|
600
|
+
const body = new URLSearchParams({ ...params, tool: this.o.tool ?? "onephylogeny", ...this.o.email ? { email: this.o.email } : {}, ...this.o.apiKey ? { api_key: this.o.apiKey } : {} }).toString();
|
|
601
|
+
let lastError = "";
|
|
602
|
+
for (let attempt = 0; attempt <= this.o.retries; attempt++) {
|
|
603
|
+
if (attempt) await this.o.sleep(Math.min(3e4, 1e3 * 2 ** attempt));
|
|
604
|
+
await this.o.sleep(this.o.delayMs);
|
|
605
|
+
try {
|
|
606
|
+
const res = await this.o.fetch(BASE + path, { method: "POST", headers: { "content-type": "application/x-www-form-urlencoded" }, body });
|
|
607
|
+
if (res.ok) return await res.text();
|
|
608
|
+
lastError = `HTTP ${res.status}`;
|
|
609
|
+
if (res.status < 500 && res.status !== 429) break;
|
|
610
|
+
} catch (e) {
|
|
611
|
+
lastError = e instanceof Error ? e.message : String(e);
|
|
612
|
+
}
|
|
613
|
+
this.o.log?.("warning", `${path} failed (${lastError}), attempt ${attempt + 1} of ${this.o.retries + 1}`);
|
|
614
|
+
}
|
|
615
|
+
throw new AcquireError(`${path} failed: ${lastError}`);
|
|
616
|
+
}
|
|
617
|
+
async json(path, params) {
|
|
618
|
+
const text2 = await this.request(path, { ...params, retmode: "json" });
|
|
619
|
+
try {
|
|
620
|
+
return JSON.parse(text2);
|
|
621
|
+
} catch {
|
|
622
|
+
throw new AcquireError(`${path} did not return JSON: ${text2.slice(0, 120)}`);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
/** Accession versions for a search, every page of them. A short read is an error. */
|
|
626
|
+
async searchAccessions(term, page = 5e3) {
|
|
627
|
+
const out = [];
|
|
628
|
+
let total = 0;
|
|
629
|
+
do {
|
|
630
|
+
const r = await this.json("esearch.fcgi", { db: "nuccore", term, idtype: "acc", retmax: String(page), retstart: String(out.length) });
|
|
631
|
+
const res = r.esearchresult;
|
|
632
|
+
if (!res || res.ERROR) throw new AcquireError(`Search failed: ${String(res?.ERROR ?? "no result")}`);
|
|
633
|
+
total = Number(res.count);
|
|
634
|
+
const ids = res.idlist ?? [];
|
|
635
|
+
if (!ids.length && out.length < total) throw new AcquireError(`Search stopped at ${out.length} of ${total} records`);
|
|
636
|
+
out.push(...ids);
|
|
637
|
+
} while (out.length < total);
|
|
638
|
+
return [...new Set(out)];
|
|
639
|
+
}
|
|
640
|
+
async count(term) {
|
|
641
|
+
const r = await this.json("esearch.fcgi", { db: "nuccore", term, retmax: "0" });
|
|
642
|
+
return Number(r.esearchresult?.count ?? 0);
|
|
643
|
+
}
|
|
644
|
+
/** Raw GenBank text for these accessions, checked to contain exactly what was asked for. */
|
|
645
|
+
async fetchGenBankText(ids) {
|
|
646
|
+
const text2 = await this.request("efetch.fcgi", { db: "nuccore", id: ids.join(","), rettype: "gbwithparts", retmode: "text" });
|
|
647
|
+
const records = parseGenBank(text2);
|
|
648
|
+
const got = new Set(records.flatMap((r) => [r.version, r.accession]));
|
|
649
|
+
const missing = ids.filter((id) => !got.has(id) && !got.has(id.split(".")[0]));
|
|
650
|
+
if (missing.length) throw new AcquireError(`NCBI did not return ${missing.length} of ${ids.length} records, including ${missing.slice(0, 3).join(", ")}`);
|
|
651
|
+
return { text: text2, records };
|
|
652
|
+
}
|
|
653
|
+
async summaries(term, retmax = 300) {
|
|
654
|
+
const search = await this.json("esearch.fcgi", { db: "nuccore", term, retmax: String(retmax) });
|
|
655
|
+
const ids = search.esearchresult?.idlist ?? [];
|
|
656
|
+
if (!ids.length) return [];
|
|
657
|
+
const r = await this.json("esummary.fcgi", { db: "nuccore", id: ids.join(",") });
|
|
658
|
+
return ids.map((id) => r.result?.[id]).filter(Boolean).map((s) => ({ accession: s.accessionversion, organism: s.organism, length: Number(s.slen), title: s.title, taxid: s.taxid ? String(s.taxid) : void 0 }));
|
|
659
|
+
}
|
|
660
|
+
/** Finds a taxon by scientific name. More than one result is returned for a person to choose from. */
|
|
661
|
+
async findTaxon(name) {
|
|
662
|
+
const r = await this.json("esearch.fcgi", { db: "taxonomy", term: `${quote(name)}[Scientific Name]`, retmax: "5" });
|
|
663
|
+
const ids = r.esearchresult?.idlist ?? [];
|
|
664
|
+
const out = [];
|
|
665
|
+
for (const id of ids) out.push({ id, scientificName: (await this.lineage(id)).scientificName });
|
|
666
|
+
return out;
|
|
667
|
+
}
|
|
668
|
+
/** A taxon's ancestors, nearest first. */
|
|
669
|
+
async lineage(taxid) {
|
|
670
|
+
const xml = await this.request("efetch.fcgi", { db: "taxonomy", id: taxid, retmode: "xml" });
|
|
671
|
+
const name = /<ScientificName>([^<]+)<\/ScientificName>/.exec(xml)?.[1];
|
|
672
|
+
if (!name) throw new AcquireError(`Taxon ${taxid} was not found`);
|
|
673
|
+
const lineage = /<LineageEx>([\s\S]*?)<\/LineageEx>/.exec(xml)?.[1] ?? "";
|
|
674
|
+
const ancestors = [...lineage.matchAll(/<Taxon>\s*<TaxId>(\d+)<\/TaxId>\s*<ScientificName>([^<]+)<\/ScientificName>\s*<Rank>([^<]*)<\/Rank>/g)].map((m) => ({ id: m[1], name: m[2], rank: m[3] })).reverse();
|
|
675
|
+
const mgc = /<MitoGeneticCode>\s*<MGCId>(\d+)<\/MGCId>/.exec(xml.replace(/<LineageEx>[\s\S]*?<\/LineageEx>/, ""))?.[1];
|
|
676
|
+
return { scientificName: name, ancestors, ...mgc ? { mitoGeneticCode: Number(mgc) } : {} };
|
|
677
|
+
}
|
|
678
|
+
};
|
|
679
|
+
var quote = (s) => `"${s.replace(/"/g, "")}"`;
|
|
680
|
+
function markerTerm(marker) {
|
|
681
|
+
if (marker.kind === "mitogenome") return '(mitochondrion[Filter] AND ("complete genome"[Title] OR "partial genome"[Title]))';
|
|
682
|
+
if (marker.kind === "snp") throw new AcquireError("SNP data has no NCBI nucleotide search. Import a prepared dataset instead");
|
|
683
|
+
const names = [marker.gene, ...marker.synonyms ?? []].filter(Boolean);
|
|
684
|
+
const fields = [...names.map((n) => `${quote(n)}[Gene]`), ...(marker.products ?? []).map((p) => `${quote(p)}[Protein Name]`)];
|
|
685
|
+
if (!fields.length) throw new AcquireError("A gene marker must name its gene");
|
|
686
|
+
const organelle = marker.organelle === "mitochondrion" ? "gene_in_mitochondrion[PROP] AND " : marker.organelle === "plastid" ? "gene_in_plastid[PROP] AND " : "";
|
|
687
|
+
return `${organelle}(${fields.join(" OR ")})`;
|
|
688
|
+
}
|
|
689
|
+
var ncbiTaxa = (taxa) => {
|
|
690
|
+
const ids = taxa.filter((t) => t.source === "ncbi-taxonomy").map((t) => t.id);
|
|
691
|
+
if (!ids.length) throw new AcquireError("NCBI searches need taxa from the ncbi-taxonomy source");
|
|
692
|
+
return ids;
|
|
693
|
+
};
|
|
694
|
+
function genomesHoldingMarkerQuery(selection) {
|
|
695
|
+
const { marker } = selection;
|
|
696
|
+
if (marker.kind !== "gene" || marker.organelle !== "mitochondrion") return null;
|
|
697
|
+
const taxa = ncbiTaxa(selection.taxa).map((id) => `txid${id}[Organism:exp]`).join(" OR ");
|
|
698
|
+
return `(${taxa}) AND mitochondrion[Filter] AND 10000:25000[SLEN] NOT refseq[Filter]`;
|
|
699
|
+
}
|
|
700
|
+
function selectionQuery(selection) {
|
|
701
|
+
const taxa = ncbiTaxa(selection.taxa).map((id) => `txid${id}[Organism:exp]`).join(" OR ");
|
|
702
|
+
return `(${taxa}) AND ${markerTerm(selection.marker)} NOT refseq[Filter]`;
|
|
703
|
+
}
|
|
704
|
+
async function suggestOutgroups(client, selection, options = {}) {
|
|
705
|
+
const { minSpecies = 2, maxSteps = 8, minLength = 0, perSpecies = 1 } = options;
|
|
706
|
+
const taxid = ncbiTaxa(selection.taxa)[0];
|
|
707
|
+
const { scientificName, ancestors } = await client.lineage(taxid);
|
|
708
|
+
const marker = markerTerm(selection.marker);
|
|
709
|
+
const steps = [];
|
|
710
|
+
let suggested = null;
|
|
711
|
+
let inner = taxid;
|
|
712
|
+
for (const taxon of ancestors.slice(0, maxSteps)) {
|
|
713
|
+
const term = `txid${taxon.id}[Organism:exp] NOT txid${inner}[Organism:exp] AND ${marker} NOT refseq[Filter]`;
|
|
714
|
+
const records = await client.count(term);
|
|
715
|
+
const summaries = records ? await client.summaries(term) : [];
|
|
716
|
+
const bySpecies = /* @__PURE__ */ new Map();
|
|
717
|
+
for (const s of summaries) {
|
|
718
|
+
if (s.length < minLength) continue;
|
|
719
|
+
const list = bySpecies.get(s.organism);
|
|
720
|
+
if (list) list.push(s);
|
|
721
|
+
else bySpecies.set(s.organism, [s]);
|
|
722
|
+
}
|
|
723
|
+
const candidates = [...bySpecies.values()].flatMap((list) => list.sort((a, b) => b.length - a.length).slice(0, perSpecies));
|
|
724
|
+
const step = { taxon, records, species: [...bySpecies.keys()].sort(), candidates: candidates.sort((a, b) => b.length - a.length), sufficient: bySpecies.size >= minSpecies };
|
|
725
|
+
steps.push(step);
|
|
726
|
+
inner = taxon.id;
|
|
727
|
+
if (step.sufficient) {
|
|
728
|
+
suggested = step;
|
|
729
|
+
break;
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
return { ingroup: scientificName, steps, suggested, caveat: "Taxonomic rank is not evidence of ancestry. Check that the ingroup is monophyletic and these outgroups fall outside it in the finished tree." };
|
|
733
|
+
}
|
|
734
|
+
var CODE_LABELS = { 1: "Standard (1)", 2: "Vertebrate mitochondrial (2)" };
|
|
735
|
+
async function suggestReference(client, selection, options = {}) {
|
|
736
|
+
const notes = [];
|
|
737
|
+
const mito = selection.marker.kind === "mitogenome" || selection.marker.kind === "gene" && selection.marker.organelle === "mitochondrion";
|
|
738
|
+
const taxid = selection.taxa.find((t) => t.source === "ncbi-taxonomy")?.id;
|
|
739
|
+
if (!taxid) return { from: null, candidates: [], geneticCode: { supported: false, reason: "Choose an NCBI taxon first" }, notes };
|
|
740
|
+
const lin = await client.lineage(taxid);
|
|
741
|
+
const code = lin.mitoGeneticCode;
|
|
742
|
+
const geneticCode = !mito ? { supported: false, reason: "Only mitochondrial markers get a suggested genetic code" } : code && CODE_LABELS[code] ? { code, supported: true, label: CODE_LABELS[code] } : { ...code ? { code } : {}, supported: false, reason: code ? `NCBI gives mitochondrial code ${code}, which the frame check doesn't support yet. Leave it unchecked.` : "NCBI gives no mitochondrial code for this taxon" };
|
|
743
|
+
if (!mito) return { from: null, candidates: [], geneticCode, notes: ["Only mitochondrial markers get a suggested reference so far. Enter one by hand."] };
|
|
744
|
+
const levels = [{ id: taxid, name: lin.scientificName, rank: "taxon" }, ...lin.ancestors.slice(0, options.maxSteps ?? 3)];
|
|
745
|
+
for (const [i, level] of levels.entries()) {
|
|
746
|
+
const found = await client.summaries(`txid${level.id}[Organism:exp] AND mitochondrion[Filter] AND refseq[Filter] AND 10000:25000[SLEN]`, 20);
|
|
747
|
+
if (!found.length) continue;
|
|
748
|
+
const candidates = found.map(({ accession, organism, length, title }) => ({ accession, organism, length, title })).sort((a, b) => Number(b.organism === lin.scientificName) - Number(a.organism === lin.scientificName) || b.length - a.length || a.accession.localeCompare(b.accession));
|
|
749
|
+
if (i > 0) notes.push(`No RefSeq mitochondrial genome for ${lin.scientificName} itself; these are from the ${level.rank} ${level.name}. Records are anchored to the reference's coordinates, so a relative works, but check that it holds the marker.`);
|
|
750
|
+
notes.push("A RefSeq genome is a convention, not proof it is typical of the species. Check that its marker is complete and annotated.");
|
|
751
|
+
return { from: { ...level, sameTaxon: i === 0 }, candidates, geneticCode, notes };
|
|
752
|
+
}
|
|
753
|
+
return { from: null, candidates: [], geneticCode, notes: [`No RefSeq mitochondrial genome within ${levels.length - 1} levels above ${lin.scientificName}. Enter a reference by hand, such as the longest complete record of the marker.`] };
|
|
754
|
+
}
|
|
755
|
+
export {
|
|
756
|
+
AcquireError,
|
|
757
|
+
BIOSAMPLE_ATTRIBUTES,
|
|
758
|
+
LISTING_KINDS,
|
|
759
|
+
LOOKUP_USER_AGENT,
|
|
760
|
+
NcbiClient,
|
|
761
|
+
approveLink,
|
|
762
|
+
biosampleFor,
|
|
763
|
+
boldCoverage,
|
|
764
|
+
classifyAccessions,
|
|
765
|
+
datasetsGenomes,
|
|
766
|
+
gbifCoverage,
|
|
767
|
+
genomesHoldingMarkerQuery,
|
|
768
|
+
idigbioCoverage,
|
|
769
|
+
linkedAccessions,
|
|
770
|
+
listSraRuns,
|
|
771
|
+
looksLikeCoordinates,
|
|
772
|
+
lookupCrossref,
|
|
773
|
+
lookupEuropePmc,
|
|
774
|
+
lookupPaper,
|
|
775
|
+
markerTerm,
|
|
776
|
+
mergePapers,
|
|
777
|
+
ncbiTaxonTerm,
|
|
778
|
+
occurrenceCoverage,
|
|
779
|
+
paced,
|
|
780
|
+
parseBioSample,
|
|
781
|
+
parseGenomeReport,
|
|
782
|
+
parseSraSummary,
|
|
783
|
+
publicListings,
|
|
784
|
+
searchEuropePmc,
|
|
785
|
+
searchPapers,
|
|
786
|
+
searchPubMed,
|
|
787
|
+
selectionQuery,
|
|
788
|
+
studiesFromRecords,
|
|
789
|
+
suggestOutgroups,
|
|
790
|
+
suggestReference,
|
|
791
|
+
titleMatch
|
|
792
|
+
};
|