@captiveorigins/onephylogeny-acquire 0.1.0-alpha.5 → 0.1.0-alpha.6

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { StudyPaper, Selection, GenBankRecord, LinkedRecord, Marker } from '@captiveorigins/onephylogeny-core';
+import { StudyPaper, Selection, GenBankRecord, LinkedRecord, FieldSource, Marker } from '@captiveorigins/onephylogeny-core';
 
 /** A paper as a search found it. Nothing joins a Study until someone adds it. */
 type FoundPaper = StudyPaper & {
@@ -35,6 +35,63 @@ declare function classifyAccessions(client: NcbiClient, accessions: string[], se
     hasMarker: Set<string>;
 }>;
 
+/**
+ * Paper lookup (roadmap stage 3): for a study known only by its GenBank title, find the published paper's DOI and
+ * PubMed IDs in Crossref and Europe PMC, and the paper's open supplements. Every result is a proposal with its source
+ * and how well the title matched; nothing is attached until a person approves it, and the approval records who and when.
+ *
+ * Both services are free and keyless. We keep to their published etiquette: one request at a time, at least a second
+ * apart by default, a User-Agent naming this tool and its repository (no personal email), and no retries on refusal.
+ */
+type LinkProposal = {
+    service: 'crossref' | 'europepmc';
+    title: string;
+    year?: number;
+    journal?: string;
+    doi?: string;
+    pmid?: string;
+    pmcid?: string;
+    /** Shared title words over all title words (0–1), and whether the titles are the same after normalising. */
+    titleMatch: number;
+    exactTitle: boolean;
+    /** Europe PMC's list of a paper's supplementary files, for open-access papers with a PMC ID. A link, not a download. */
+    supplementsUrl?: string;
+    source: FieldSource;
+};
+type ApprovedLink = Pick<LinkProposal, 'doi' | 'pmid'> & {
+    source: FieldSource & {
+        approvedBy?: string;
+        approvedAt: string;
+    };
+};
+declare const LOOKUP_USER_AGENT = "OnePhylogeny (https://github.com/CaptiveOrigins/onephylogeny)";
+declare function titleMatch(a: string, b: string): {
+    score: number;
+    exact: boolean;
+};
+/** Sends requests one at a time, at least `gapMs` apart. */
+declare function paced(fetch: FetchLike, gapMs?: number, sleep?: (ms: number) => Promise<void>, now?: () => number): FetchLike;
+declare function lookupCrossref(fetch: FetchLike, paper: Pick<StudyPaper, 'title' | 'authors'>, at: string, rows?: number): Promise<LinkProposal[]>;
+declare function lookupEuropePmc(fetch: FetchLike, paper: Pick<StudyPaper, 'title'>, at: string, pageSize?: number): Promise<LinkProposal[]>;
+/**
+ * Both services, best title match first. A service that fails is listed in `failed`, so a short list never looks
+ * complete. Proposals below `minMatch` are dropped as noise.
+ */
+declare function lookupPaper(fetch: FetchLike, paper: Pick<StudyPaper, 'title' | 'authors'>, options?: {
+    at?: string;
+    minMatch?: number;
+}): Promise<{
+    proposals: LinkProposal[];
+    failed: string[];
+}>;
+/** A person's approval of one proposal: the paper gains its DOI and PMID, each with where it came from, who and when. */
+declare function approveLink(paper: StudyPaper, proposal: LinkProposal, meta: {
+    by?: string;
+    at: string;
+}): StudyPaper & {
+    linkSource: ApprovedLink;
+};
+
 type FetchLike = (url: string, init?: {
     method?: string;
     headers?: Record<string, string>;
@@ -88,6 +145,7 @@ declare class NcbiClient {
     lineage(taxid: string): Promise<{
         scientificName: string;
         ancestors: TaxonStep[];
+        mitoGeneticCode?: number;
     }>;
 }
 type TaxonStep = {
@@ -134,4 +192,40 @@ declare function suggestOutgroups(client: NcbiClient, selection: Selection, opti
     perSpecies?: number;
 }): Promise<OutgroupLadder>;
 
-export { AcquireError, type FetchLike, type FoundPaper, NcbiClient, type NcbiOptions, type OutgroupCandidate, type OutgroupLadder, type OutgroupStep, type PaperSearch, type TaxonStep, classifyAccessions, genomesHoldingMarkerQuery, linkedAccessions, markerTerm, mergePapers, searchEuropePmc, searchPapers, searchPubMed, selectionQuery, studiesFromRecords, suggestOutgroups };
+type ReferenceCandidate = {
+    accession: string;
+    organism: string;
+    length: number;
+    title: string;
+};
+type ReferenceSuggestion = {
+    /** The nearest level with a RefSeq mitochondrial genome: the taxon itself, or an ancestor. */
+    from: {
+        id: string;
+        name: string;
+        rank: string;
+        sameTaxon: boolean;
+    } | null;
+    candidates: ReferenceCandidate[];
+    /** From NCBI Taxonomy. Only codes the frame check supports are offered; others say why not. */
+    geneticCode: {
+        code: number;
+        supported: true;
+        label: string;
+    } | {
+        code?: number;
+        supported: false;
+        reason: string;
+    };
+    notes: string[];
+};
+/**
+ * Guided setup (roadmap stage 3): a reference a newcomer can start from. The nearest RefSeq complete mitochondrial genome,
+ * from the taxon itself or up to `maxSteps` ancestors, and the taxon's mitochondrial genetic code. Suggestions only: a
+ * person chooses, and can change both.
+ */
+declare function suggestReference(client: NcbiClient, selection: Selection, options?: {
+    maxSteps?: number;
+}): Promise<ReferenceSuggestion>;
+
+export { AcquireError, type ApprovedLink, type FetchLike, type FoundPaper, LOOKUP_USER_AGENT, type LinkProposal, NcbiClient, type NcbiOptions, type OutgroupCandidate, type OutgroupLadder, type OutgroupStep, type PaperSearch, type ReferenceCandidate, type ReferenceSuggestion, type TaxonStep, approveLink, classifyAccessions, genomesHoldingMarkerQuery, linkedAccessions, lookupCrossref, lookupEuropePmc, lookupPaper, markerTerm, mergePapers, paced, searchEuropePmc, searchPapers, searchPubMed, selectionQuery, studiesFromRecords, suggestOutgroups, suggestReference, titleMatch };

package/dist/index.js

@@ -155,6 +155,105 @@ async function classifyAccessions(client, accessions, selection) {
   return { inTarget, hasMarker };
 }
 
+// src/lookup.ts
+var CROSSREF = "https://api.crossref.org/works";
+var EUROPE_PMC2 = "https://www.ebi.ac.uk/europepmc/webservices/rest";
+var LOOKUP_USER_AGENT = "OnePhylogeny (https://github.com/CaptiveOrigins/onephylogeny)";
+var words = (t) => new Set(t.toLowerCase().replace(/<[^>]+>/g, "").replace(/[^a-z0-9]+/g, " ").trim().split(" ").filter(Boolean));
+function titleMatch(a, b) {
+  const x = words(a);
+  const y = words(b);
+  const shared = [...x].filter((w) => y.has(w)).length;
+  const all = (/* @__PURE__ */ new Set([...x, ...y])).size;
+  return { score: all ? Math.round(shared / all * 1e3) / 1e3 : 0, exact: all > 0 && shared === all };
+}
+function paced(fetch, gapMs = 1e3, sleep = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now()) {
+  let last = -Infinity;
+  let queue = Promise.resolve();
+  return (url, init) => {
+    const run = queue.then(async () => {
+      const wait = last + gapMs - now();
+      if (wait > 0) await sleep(wait);
+      last = now();
+      return fetch(url, { ...init, headers: { "User-Agent": LOOKUP_USER_AGENT, ...init?.headers } });
+    });
+    queue = run.then(() => void 0, () => void 0);
+    return run;
+  };
+}
+async function getJson(fetch, url, service) {
+  const res = await fetch(url);
+  if (!res.ok) throw new AcquireError(`${service} answered HTTP ${res.status}`);
+  return JSON.parse(await res.text());
+}
+async function lookupCrossref(fetch, paper, at, rows = 5) {
+  const params = new URLSearchParams({ "query.bibliographic": paper.title, rows: String(rows), select: "DOI,title,issued,container-title" });
+  if (paper.authors) params.set("query.author", paper.authors.split(/[,;]| and /)[0].trim());
+  const url = `${CROSSREF}?${params}`;
+  const body = await getJson(fetch, url, "Crossref");
+  return (body.message?.items ?? []).filter((i) => i.title?.[0] && i.DOI).map((i) => {
+    const m = titleMatch(paper.title, i.title[0]);
+    const year = i.issued?.["date-parts"]?.[0]?.[0];
+    return {
+      service: "crossref",
+      title: i.title[0],
+      doi: String(i.DOI).toLowerCase(),
+      titleMatch: m.score,
+      exactTitle: m.exact,
+      ...year ? { year: Number(year) } : {},
+      ...i["container-title"]?.[0] ? { journal: i["container-title"][0] } : {},
+      source: { kind: "publication", citation: `Crossref ${url}`, retrievedAt: at }
+    };
+  });
+}
+async function lookupEuropePmc(fetch, paper, at, pageSize = 5) {
+  const title = paper.title.replace(/["\\]/g, " ");
+  const url = `${EUROPE_PMC2}/search?${new URLSearchParams({ query: `TITLE:"${title}"`, format: "json", resultType: "lite", pageSize: String(pageSize) })}`;
+  const body = await getJson(fetch, url, "Europe PMC");
+  return (body.resultList?.result ?? []).filter((r) => r.title).map((r) => {
+    const m = titleMatch(paper.title, r.title);
+    const open = r.isOpenAccess === "Y" && r.pmcid;
+    return {
+      service: "europepmc",
+      title: String(r.title).replace(/\.$/, ""),
+      titleMatch: m.score,
+      exactTitle: m.exact,
+      ...r.pubYear ? { year: Number(r.pubYear) } : {},
+      ...r.journalTitle ? { journal: r.journalTitle } : {},
+      ...r.doi ? { doi: String(r.doi).toLowerCase() } : {},
+      ...r.pmid ? { pmid: String(r.pmid) } : {},
+      ...r.pmcid ? { pmcid: String(r.pmcid) } : {},
+      ...open ? { supplementsUrl: `${EUROPE_PMC2}/${r.pmcid}/supplementaryFiles` } : {},
+      source: { kind: "publication", citation: `Europe PMC ${url}`, retrievedAt: at }
+    };
+  });
+}
+async function lookupPaper(fetch, paper, options = {}) {
+  const at = options.at ?? (/* @__PURE__ */ new Date()).toISOString();
+  const failed = [];
+  const attempt = async (name, work) => {
+    try {
+      return await work();
+    } catch (e) {
+      failed.push(`${name}: ${e.message}`);
+      return [];
+    }
+  };
+  const all = [...await attempt("Crossref", () => lookupCrossref(fetch, paper, at)), ...await attempt("Europe PMC", () => lookupEuropePmc(fetch, paper, at))];
+  const proposals = all.filter((p) => p.titleMatch >= (options.minMatch ?? 0.5)).sort((a, b) => Number(b.exactTitle) - Number(a.exactTitle) || b.titleMatch - a.titleMatch || a.service.localeCompare(b.service));
+  return { proposals, failed };
+}
+function approveLink(paper, proposal, meta) {
+  if (paper.doi && proposal.doi && paper.doi !== proposal.doi) throw new AcquireError(`This study already has DOI ${paper.doi}; remove it first to replace it`);
+  if (paper.pmid && proposal.pmid && paper.pmid !== proposal.pmid) throw new AcquireError(`This study already has PMID ${paper.pmid}; remove it first to replace it`);
+  return {
+    ...paper,
+    ...proposal.doi ? { doi: proposal.doi } : {},
+    ...proposal.pmid ? { pmid: proposal.pmid } : {},
+    linkSource: { ...proposal.doi ? { doi: proposal.doi } : {}, ...proposal.pmid ? { pmid: proposal.pmid } : {}, source: { ...proposal.source, ...meta.by ? { approvedBy: meta.by } : {}, approvedAt: meta.at } }
+  };
+}
+
 // src/index.ts
 var AcquireError = class extends Error {
 };
@@ -240,7 +339,8 @@ var NcbiClient = class {
     if (!name) throw new AcquireError(`Taxon ${taxid} was not found`);
     const lineage = /<LineageEx>([\s\S]*?)<\/LineageEx>/.exec(xml)?.[1] ?? "";
     const ancestors = [...lineage.matchAll(/<Taxon>\s*<TaxId>(\d+)<\/TaxId>\s*<ScientificName>([^<]+)<\/ScientificName>\s*<Rank>([^<]*)<\/Rank>/g)].map((m) => ({ id: m[1], name: m[2], rank: m[3] })).reverse();
-    return { scientificName: name, ancestors };
+    const mgc = /<MitoGeneticCode>\s*<MGCId>(\d+)<\/MGCId>/.exec(xml.replace(/<LineageEx>[\s\S]*?<\/LineageEx>/, ""))?.[1];
+    return { scientificName: name, ancestors, ...mgc ? { mitoGeneticCode: Number(mgc) } : {} };
   }
 };
 var quote = (s) => `"${s.replace(/"/g, "")}"`;
@@ -298,18 +398,47 @@ async function suggestOutgroups(client, selection, options = {}) {
   }
   return { ingroup: scientificName, steps, suggested, caveat: "Taxonomic rank is not evidence of ancestry. Check that the ingroup is monophyletic and these outgroups fall outside it in the finished tree." };
 }
+var CODE_LABELS = { 1: "Standard (1)", 2: "Vertebrate mitochondrial (2)" };
+async function suggestReference(client, selection, options = {}) {
+  const notes = [];
+  const mito = selection.marker.kind === "mitogenome" || selection.marker.kind === "gene" && selection.marker.organelle === "mitochondrion";
+  const taxid = selection.taxa.find((t) => t.source === "ncbi-taxonomy")?.id;
+  if (!taxid) return { from: null, candidates: [], geneticCode: { supported: false, reason: "Choose an NCBI taxon first" }, notes };
+  const lin = await client.lineage(taxid);
+  const code = lin.mitoGeneticCode;
+  const geneticCode = !mito ? { supported: false, reason: "Only mitochondrial markers get a suggested genetic code" } : code && CODE_LABELS[code] ? { code, supported: true, label: CODE_LABELS[code] } : { ...code ? { code } : {}, supported: false, reason: code ? `NCBI gives mitochondrial code ${code}, which the frame check doesn't support yet. Leave it unchecked.` : "NCBI gives no mitochondrial code for this taxon" };
+  if (!mito) return { from: null, candidates: [], geneticCode, notes: ["Only mitochondrial markers get a suggested reference so far. Enter one by hand."] };
+  const levels = [{ id: taxid, name: lin.scientificName, rank: "taxon" }, ...lin.ancestors.slice(0, options.maxSteps ?? 3)];
+  for (const [i, level] of levels.entries()) {
+    const found = await client.summaries(`txid${level.id}[Organism:exp] AND mitochondrion[Filter] AND refseq[Filter] AND 10000:25000[SLEN]`, 20);
+    if (!found.length) continue;
+    const candidates = found.map(({ accession, organism, length, title }) => ({ accession, organism, length, title })).sort((a, b) => Number(b.organism === lin.scientificName) - Number(a.organism === lin.scientificName) || b.length - a.length || a.accession.localeCompare(b.accession));
+    if (i > 0) notes.push(`No RefSeq mitochondrial genome for ${lin.scientificName} itself; these are from the ${level.rank} ${level.name}. Records are anchored to the reference's coordinates, so a relative works, but check that it holds the marker.`);
+    notes.push("A RefSeq genome is a convention, not proof it is typical of the species. Check that its marker is complete and annotated.");
+    return { from: { ...level, sameTaxon: i === 0 }, candidates, geneticCode, notes };
+  }
+  return { from: null, candidates: [], geneticCode, notes: [`No RefSeq mitochondrial genome within ${levels.length - 1} levels above ${lin.scientificName}. Enter a reference by hand, such as the longest complete record of the marker.`] };
+}
 export {
   AcquireError,
+  LOOKUP_USER_AGENT,
   NcbiClient,
+  approveLink,
   classifyAccessions,
   genomesHoldingMarkerQuery,
   linkedAccessions,
+  lookupCrossref,
+  lookupEuropePmc,
+  lookupPaper,
   markerTerm,
   mergePapers,
+  paced,
   searchEuropePmc,
   searchPapers,
   searchPubMed,
   selectionQuery,
   studiesFromRecords,
-  suggestOutgroups
+  suggestOutgroups,
+  suggestReference,
+  titleMatch
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@captiveorigins/onephylogeny-acquire",
-  "version": "0.1.0-alpha.5",
+  "version": "0.1.0-alpha.6",
   "description": "OnePhylogeny acquire: fetches sequence records by taxon and marker with safe retries and no duplicates, keeps raw files with hashes, and suggests outgroups outward through the taxonomy.",
   "keywords": [
     "phylogenetics",
@@ -36,7 +36,7 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@captiveorigins/onephylogeny-core": "^0.1.0-alpha.5"
+    "@captiveorigins/onephylogeny-core": "^0.1.0-alpha.6"
   },
   "scripts": {
     "build": "tsup src/index.ts --format esm --dts --clean --target es2022",