@de-otio/bibcheck 0.1.0

package/dist/existence.js

@@ -0,0 +1,308 @@
+/**
+ * Existence subcommand — commodity convenience layer.
+ *
+ * For each CSL-JSON bibliography entry, performs DOI/ISBN/title-search
+ * lookups against CrossRef, OpenAlex, and OpenLibrary, then assigns an
+ * ExistenceStatus based on the best result across all tried sources. Each
+ * layer also carries the Q2 evidence vocabulary (`evidence` / `checkedFor` /
+ * `notCheckedFor`) and a sanitized `error` string when a source crashed.
+ *
+ * Title metadata-match rule (H3): a hand-rolled token-set ratio over the
+ * normalised title tokens, so subtitle presence ("Liberty" vs "Liberty: A
+ * Study") and word reordering do not cause a false metadata-mismatch.
+ * Normalised Levenshtein is kept only as a tiebreaker for near-identical
+ * titles where the token sets diverge. Ratio ≥ 0.85 → match.
+ *
+ * First-author match:
+ *   Case-insensitive substring of first-author surname in any metadata
+ *   author string.
+ *
+ * Identifier short-circuit (T21/T22): a malformed or bad-checksum DOI/ISBN is
+ * a strong, cheap fabrication signal and cannot be looked up; callers pass
+ * `identifierInvalid` so the network call is skipped and the entry is recorded
+ * as `unverifiable` with a note (it is already gating via
+ * `summary.malformedIdentifiers`).
+ *
+ * WorldCat (OCLC Classify) was removed in 0.2.0 — the endpoint was
+ * decommissioned in 2019 (see tmp/design-review/worldcat.md). ISBN existence
+ * is covered by OpenLibrary.
+ */
+import { distance } from 'fastest-levenshtein';
+import { HttpError } from './http.js';
+// ---------------------------------------------------------------------------
+// Author helpers
+// ---------------------------------------------------------------------------
+export function getFirstAuthorSurname(entry) {
+    const first = entry.author?.[0];
+    if (first === undefined)
+        return undefined;
+    return first.family ?? first.literal?.split(/\s+/).pop();
+}
+export function getAllAuthorNames(entry) {
+    return (entry.author ?? []).map((a) => a.family ?? a.literal ?? '').filter((s) => s.length > 0);
+}
+// ---------------------------------------------------------------------------
+// Title normalisation and fuzzy match (H3: token-set ratio)
+// ---------------------------------------------------------------------------
+/**
+ * Normalise a title for comparison: lowercase, strip non-alphanumeric runs
+ * (punctuation + whitespace), collapse to single spaces.
+ */
+function normaliseTitle(t) {
+    return t
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, ' ')
+        .trim();
+}
+/** Split a normalised title into a de-duplicated set of word tokens. */
+function tokenSet(t) {
+    const norm = normaliseTitle(t);
+    if (norm === '')
+        return new Set();
+    return new Set(norm.split(' '));
+}
+/**
+ * Token-set ratio: |intersection| / |smaller set|. This is deliberately
+ * asymmetric in favour of subset relationships — when one title is a subset of
+ * the other (e.g. "Liberty" ⊂ "Liberty A Study"), every token of the smaller
+ * set is present, so the ratio is 1.0 and the titles match. Word reordering
+ * has no effect because sets are unordered.
+ */
+function tokenSetRatio(a, b) {
+    const smaller = a.size <= b.size ? a : b;
+    const larger = a.size <= b.size ? b : a;
+    if (smaller.size === 0)
+        return larger.size === 0 ? 1 : 0;
+    let inter = 0;
+    for (const tok of smaller) {
+        if (larger.has(tok))
+            inter += 1;
+    }
+    return inter / smaller.size;
+}
+/**
+ * Returns true if the two title strings are close enough to be the same work.
+ *
+ * Primary metric is the token-set ratio (≥ 0.85), which is robust to subtitle
+ * presence and word reordering. As a tiebreaker (e.g. single-token titles with
+ * a small typo, where the token sets are disjoint but the strings are close),
+ * the normalised Levenshtein ratio (≥ 0.85) is consulted.
+ */
+export function titlesMatch(a, b) {
+    const na = normaliseTitle(a);
+    const nb = normaliseTitle(b);
+    if (na === nb)
+        return true;
+    const setA = tokenSet(a);
+    const setB = tokenSet(b);
+    if (tokenSetRatio(setA, setB) >= 0.85)
+        return true;
+    // Levenshtein tiebreaker: catches near-identical titles whose tokenisation
+    // diverges (typos, hyphenation differences across single tokens).
+    const maxLen = Math.max(na.length, nb.length);
+    if (maxLen === 0)
+        return true;
+    const d = distance(na, nb);
+    const ratio = 1 - d / maxLen;
+    return ratio >= 0.85;
+}
+/**
+ * Returns true if the first-author surname of the entry appears (case-
+ * insensitively) as a substring in at least one of the metadata author strings.
+ */
+function authorsMatch(surname, metaAuthors) {
+    const lower = surname.toLowerCase();
+    return metaAuthors.some((a) => a.toLowerCase().includes(lower));
+}
+function compareMetadata(entry, meta) {
+    if (meta === null)
+        return 'metadata-mismatch';
+    const entryTitle = entry.title;
+    const metaTitle = meta.title;
+    const titleOk = entryTitle === undefined || metaTitle === undefined || titlesMatch(entryTitle, metaTitle);
+    const surname = getFirstAuthorSurname(entry);
+    const metaAuthors = meta.authors ?? [];
+    const authorOk = surname === undefined || metaAuthors.length === 0 || authorsMatch(surname, metaAuthors);
+    return titleOk && authorOk ? 'found' : 'metadata-mismatch';
+}
+// ---------------------------------------------------------------------------
+// Per-source check builders
+// ---------------------------------------------------------------------------
+function lookupResultToCheck(source, entry, lookup) {
+    if (!lookup.found) {
+        return { source, result: 'not-found', evidence: null };
+    }
+    const matchResult = compareMetadata(entry, lookup.metadata);
+    return {
+        source,
+        result: matchResult,
+        evidence: lookup.metadata ?? null,
+    };
+}
+/**
+ * Strip `mailto=<value>` query params from any URL embedded in a free-text
+ * error message, so the polite-pool email never reaches the output. We cannot
+ * rely on `sanitizeMailto` here because that only rewrites strings that ARE a
+ * URL; an HttpError message like "GET https://…?mailto=x failed" is a sentence
+ * with a URL inside it.
+ */
+function sanitizeErrorMessage(message) {
+    return message.replace(/([?&])mailto=[^&\s]*/gi, (_m, sep) => sep === '?' ? '?' : '');
+}
+async function safeCall(source, fn, entry) {
+    try {
+        const result = await fn();
+        return lookupResultToCheck(source, entry, result);
+    }
+    catch (err) {
+        if (err instanceof HttpError) {
+            // Sanitize so a ?mailto= / API key in the message cannot leak into output.
+            const message = sanitizeErrorMessage(err.message);
+            return { source, result: 'error', evidence: { error: message } };
+        }
+        throw err;
+    }
+}
+// ---------------------------------------------------------------------------
+// Status derivation
+// ---------------------------------------------------------------------------
+function deriveStatus(checks) {
+    const results = new Set(checks.map((c) => c.result));
+    if (results.has('found'))
+        return 'verified';
+    if (results.has('metadata-mismatch'))
+        return 'metadata-mismatch';
+    if (results.has('not-found')) {
+        // Only 'not-found' (and possibly 'error') entries — but at least one confirmed absence.
+        if ([...results].every((r) => r === 'not-found' || r === 'error')) {
+            // If there is at least one 'not-found', report that even if some errored.
+            return 'not-found-in-databases';
+        }
+    }
+    // All 'error' or all 'no-doi'.
+    return 'unverifiable';
+}
+// ---------------------------------------------------------------------------
+// Evidence vocabulary (Q2)
+// ---------------------------------------------------------------------------
+/**
+ * Map an existence status to its evidence vocabulary: the discrete
+ * evidence level plus the explicit checkedFor / notCheckedFor dimensions.
+ * `claim-support` is ALWAYS in notCheckedFor (bibcheck never verifies whether
+ * the source supports the prose's claim — that is the manual worklist's job).
+ */
+const ALL_DIMENSIONS = [
+    'existence',
+    'metadata',
+    'canonical-url',
+    'claim-support',
+];
+function evidenceFor(status) {
+    switch (status) {
+        case 'verified':
+            return { evidence: 'exists-metadata-match', checkedFor: ['existence', 'metadata'] };
+        case 'metadata-mismatch':
+            return { evidence: 'exists-metadata-mismatch', checkedFor: ['existence', 'metadata'] };
+        case 'not-found-in-databases':
+            return { evidence: 'absent', checkedFor: ['existence'] };
+        case 'unverifiable':
+            return { evidence: 'unverifiable', checkedFor: [] };
+    }
+}
+/**
+ * Build a complete ExistenceLayer from the per-source checks: derive the
+ * status, attach the evidence vocabulary, and surface a sanitized top-level
+ * error when EVERY check crashed (vs. a clean unverifiable result).
+ */
+function buildLayer(checks) {
+    const status = deriveStatus(checks);
+    const { evidence, checkedFor } = evidenceFor(status);
+    const notCheckedFor = ALL_DIMENSIONS.filter((d) => !checkedFor.includes(d));
+    // Surface a top-level error only when the layer failed to run cleanly: all
+    // checks were transport errors. (A mix of error + not-found is a real
+    // not-found-in-databases result and is not flagged as an error.)
+    let error = null;
+    if (checks.length > 0 && checks.every((c) => c.result === 'error')) {
+        const first = checks.find((c) => c.result === 'error');
+        const ev = first?.evidence;
+        if (ev !== null && typeof ev === 'object' && 'error' in ev) {
+            error = String(ev.error);
+        }
+        else {
+            error = 'All existence sources failed.';
+        }
+    }
+    return { status, evidence, checkedFor, notCheckedFor, checks, error };
+}
+// ---------------------------------------------------------------------------
+// Per-entry processing
+// ---------------------------------------------------------------------------
+/**
+ * Build the unverifiable layer for an entry whose local identifier validation
+ * (T21) failed. The network existence call is skipped entirely.
+ */
+function identifierInvalidLayer() {
+    const checks = [
+        {
+            source: 'crossref',
+            result: 'error',
+            evidence: { error: 'Identifier validation failed; existence lookup skipped.' },
+        },
+    ];
+    const { evidence, checkedFor } = evidenceFor('unverifiable');
+    const notCheckedFor = ALL_DIMENSIONS.filter((d) => !checkedFor.includes(d));
+    return {
+        status: 'unverifiable',
+        evidence,
+        checkedFor,
+        notCheckedFor,
+        checks,
+        error: 'Identifier validation failed; existence lookup skipped.',
+    };
+}
+async function processEntry(entry, clients, signal) {
+    const checks = [];
+    if (entry.doi !== undefined && entry.doi !== '') {
+        // DOI route: CrossRef + OpenAlex in parallel.
+        const [crCheck, oaCheck] = await Promise.all([
+            safeCall('crossref', () => clients.crossref.lookupByDoi(entry.doi, signal), entry),
+            safeCall('openalex', () => clients.openalex.lookupByDoi(entry.doi, signal), entry),
+        ]);
+        checks.push(crCheck, oaCheck);
+    }
+    else if (entry.isbn !== undefined && entry.isbn !== '') {
+        // ISBN route: OpenLibrary (the keyless ISBN source; WorldCat removed).
+        const olCheck = await safeCall('openlibrary', () => clients.openlibrary.lookupByIsbn(entry.isbn, signal), entry);
+        checks.push(olCheck);
+    }
+    else if (entry.title !== undefined && entry.title !== '') {
+        // Title-search route: OpenAlex only.
+        const authors = getAllAuthorNames(entry);
+        const oaCheck = await safeCall('openalex', () => clients.openalex.searchByTitleAuthor(entry.title, authors, signal), entry);
+        checks.push(oaCheck);
+    }
+    else {
+        // No identifier — unverifiable.
+        checks.push({ source: 'crossref', result: 'no-doi', evidence: null });
+    }
+    return buildLayer(checks);
+}
+// ---------------------------------------------------------------------------
+// runExistence
+// ---------------------------------------------------------------------------
+export async function runExistence(deps) {
+    const { bibliography, clients, identifierInvalid, signal } = deps;
+    const entries = [];
+    for (const entry of bibliography) {
+        if (signal.aborted) {
+            throw signal.reason instanceof Error ? signal.reason : new Error('Aborted');
+        }
+        // Skip the network call when local identifier validation already failed.
+        const existence = identifierInvalid?.has(entry.citekey) === true
+            ? identifierInvalidLayer()
+            : await processEntry(entry, clients, signal);
+        entries.push({ citekey: entry.citekey, existence });
+    }
+    return { entries };
+}
+//# sourceMappingURL=existence.js.map

package/dist/existence.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"existence.js","sourceRoot":"","sources":["../src/existence.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC/C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAwCtC,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E,MAAM,UAAU,qBAAqB,CAAC,KAAe;IACnD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAe;IAC/C,OAAO,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAClG,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,cAAc,CAAC,CAAS;IAC/B,OAAO,CAAC;SACL,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,wEAAwE;AACxE,SAAS,QAAQ,CAAC,CAAS;IACzB,MAAM,IAAI,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC/B,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,IAAI,GAAG,EAAE,CAAC;IAClC,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,CAAc,EAAE,CAAc;IACnD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,KAAK,IAAI,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;AAC9B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CAAC,CAAS,EAAE,CAAS;IAC9C,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC;IAC7B,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IAE3B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,IAAI,CAAC;IAEnD,2EAA2E;IAC3E,kEAAkE;IAClE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAC9B,MAAM,CAAC,GAAG,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC3B,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;IAC7B,OAAO,KAAK,IAAI,IAAI,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,OAAe,EAAE,WAAqB;IAC1D,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;AAClE,CAAC;AAQD,SAAS,eAAe,CAAC,KAAe,EAAE,IAAsC;IAC9E,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAE9C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC;IAC7B,MAAM,OAAO,GACX,UAAU,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAE5F,MAAM,OAAO,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;IAC7C,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;IACvC,MAAM,QAAQ,GACZ,OAAO,KAAK,SAAS,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAE1F,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,CAAC;AAC7D,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,SAAS,mBAAmB,CAC1B,MAAgC,EAChC,KAAe,EACf,MAA4B;IAE5B,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IACzD,CAAC;IACD,MAAM,WAAW,GAAG,eAAe,CAAC,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5D,OAAO;QACL,MAAM;QACN,MAAM,EAAE,WAAW;QACnB,QAAQ,EAAE,MAAM,CAAC,QAAmB,IAAI,IAAI;KAC7C,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,OAAO,CAAC,OAAO,CAAC,wBAAwB,EAAE,CAAC,EAAE,EAAE,GAAW,EAAE,EAAE,CACnE,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CACvB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,MAAgC,EAChC,EAAuC,EACvC,KAAe;IAEf,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;QAC1B,OAAO,mBAAmB,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;YAC7B,2EAA2E;YAC3E,MAAM,OAAO,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAClD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;QACnE,CAAC;QACD,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,YAAY,CAAC,MAAwB;IAC5C,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAErD,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,UAAU,CAAC;IAC5C,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAAE,OAAO,mBAAmB,CAAC;IACjE,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,wFAAwF;QACxF,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,WAAW,IAAI,CAAC,KAAK,OAAO,CAAC,EAAE,CAAC;YAClE,0EAA0E;YAC1E,OAAO,wBAAwB,CAAC;QAClC,CAAC;IACH,CAAC;IACD,+BAA+B;IAC/B,OAAO,cAAc,CAAC;AACxB,CAAC;AAED,8EAA8E;AAC9E,2BAA2B;AAC3B,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,cAAc,GAA8B;IAChD,WAAW;IACX,UAAU;IACV,eAAe;IACf,eAAe;CAChB,CAAC;AAEF,SAAS,WAAW,CAAC,MAAuB;IAI1C,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,UAAU;YACb,OAAO,EAAE,QAAQ,EAAE,uBAAuB,EAAE,UAAU,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,CAAC;QACtF,KAAK,mBAAmB;YACtB,OAAO,EAAE,QAAQ,EAAE,0BAA0B,EAAE,UAAU,EAAE,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,CAAC;QACzF,KAAK,wBAAwB;YAC3B,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC;QAC3D,KAAK,cAAc;YACjB,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAwB;IAC1C,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IACpC,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACrD,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,2EAA2E;IAC3E,sEAAsE;IACtE,iEAAiE;IACjE,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,CAAC;QACnE,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,KAAK,EAAE,QAAQ,CAAC;QAC3B,IAAI,EAAE,KAAK,IAAI,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,OAAO,IAAI,EAAE,EAAE,CAAC;YAC3D,KAAK,GAAG,MAAM,CAAE,EAAyB,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,+BAA+B,CAAC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,sBAAsB;IAC7B,MAAM,MAAM,GAAqB;QAC/B;YACE,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,OAAO;YACf,QAAQ,EAAE,EAAE,KAAK,EAAE,yDAAyD,EAAE;SAC/E;KACF,CAAC;IACF,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,cAAc,CAAC,CAAC;IAC7D,MAAM,aAAa,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5E,OAAO;QACL,MAAM,EAAE,cAAc;QACtB,QAAQ;QACR,UAAU;QACV,aAAa;QACb,MAAM;QACN,KAAK,EAAE,yDAAyD;KACjE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAe,EACf,OAAoC,EACpC,MAAmB;IAEnB,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,KAAK,EAAE,EAAE,CAAC;QAChD,8CAA8C;QAC9C,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3C,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,GAAI,EAAE,MAAM,CAAC,EAAE,KAAK,CAAC;YACnF,QAAQ,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,GAAI,EAAE,MAAM,CAAC,EAAE,KAAK,CAAC;SACpF,CAAC,CAAC;QACH,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC;SAAM,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;QACzD,uEAAuE;QACvE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAC5B,aAAa,EACb,GAAG,EAAE,CAAC,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,KAAK,CAAC,IAAK,EAAE,MAAM,CAAC,EAC3D,KAAK,CACN,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;SAAM,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,KAAK,EAAE,EAAE,CAAC;QAC3D,qCAAqC;QACrC,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,QAAQ,CAC5B,UAAU,EACV,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAM,EAAE,OAAO,EAAE,MAAM,CAAC,EACzE,KAAK,CACN,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvB,CAAC;SAAM,CAAC;QACN,gCAAgC;QAChC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,8EAA8E;AAC9E,eAAe;AACf,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAsB;IACvD,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAClE,MAAM,OAAO,GAAkC,EAAE,CAAC;IAElD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9E,CAAC;QACD,yEAAyE;QACzE,MAAM,SAAS,GACb,iBAAiB,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI;YAC5C,CAAC,CAAC,sBAAsB,EAAE;YAC1B,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC"}

package/dist/http.d.ts

@@ -0,0 +1,97 @@
+/**
+ * HTTP utility for URL verification.
+ *
+ * Provides an HttpClient backed by the native Node 20 fetch API with per-origin
+ * concurrency queuing, jittered-backoff retry on 5xx/network errors, manual
+ * redirect following with SSRF mitigation, and a higher-level headCheck utility
+ * for canonical-edition URL verification.
+ *
+ * Note: The spec calls for undici.request directly, but we use the native Node
+ * global fetch (which is backed by undici internally) because undici is not
+ * separately installed in this project. We use `redirect: 'manual'` to achieve
+ * the same manual redirect-following behaviour.
+ */
+import type { Cache } from './cache/fs-cache.js';
+export interface HttpResponse {
+    status: number;
+    headers: Record<string, string>;
+    body: unknown;
+}
+export interface HttpRequestOptions {
+    headers?: Record<string, string>;
+    signal?: AbortSignal;
+    timeoutMs?: number;
+}
+export interface HttpHeadOptions extends HttpRequestOptions {
+    followRedirects?: boolean;
+    maxRedirects?: number;
+}
+export interface HttpHeadResponse {
+    status: number;
+    finalUrl: string;
+    redirectChain: string[];
+}
+export interface HttpClient {
+    get(url: string, opts?: HttpRequestOptions): Promise<HttpResponse>;
+    head(url: string, opts?: HttpHeadOptions): Promise<HttpHeadResponse>;
+}
+export interface CreateHttpClientOptions {
+    userAgent?: string;
+    defaultTimeoutMs?: number;
+    maxRetries?: number;
+    retryBaseMs?: number;
+    perOriginConcurrency?: number;
+    totalDeadlineMs?: number;
+    /**
+     * Test-only escape hatch. When true, the per-hop private-IP SSRF guard is
+     * skipped so the in-process loopback test server (127.0.0.1) is reachable.
+     * Production callers MUST leave this unset/false — the secure default rejects
+     * private addresses on every hop.
+     */
+    allowPrivateHosts?: boolean;
+}
+export declare class HttpError extends Error {
+    name: "HttpError";
+    readonly status: number | undefined;
+    readonly cause: unknown;
+    constructor(message: string, status?: number, cause?: unknown);
+}
+export declare function isPrivateIp(ip: string): boolean;
+/**
+ * Decide whether a configured API base URL targets a private / loopback host.
+ *
+ * The per-hop SSRF guard exists to stop *untrusted bibliography URLs* reaching
+ * internal addresses. The `[apis] *_base` endpoints, by contrast, are
+ * operator-controlled configuration; pointing them at `http://127.0.0.1:PORT`
+ * (a local stub or dev mirror) is a legitimate, explicit choice. Callers use
+ * this to opt a configured-endpoint client into `allowPrivateHosts` so the
+ * operator's deliberate localhost config is honored without weakening the guard
+ * on bibliography-derived URLs. Returns false for any unparseable / public base.
+ */
+export declare function isPrivateApiBase(baseUrl: string | null | undefined): boolean;
+export declare function isHostAllowed(host: string, whitelist: string[]): boolean;
+/**
+ * Parse a Retry-After header value into a delay in milliseconds, or null if it
+ * is absent / unparseable. Accepts delta-seconds ("120") and an HTTP-date.
+ * `now` is injectable for deterministic tests.
+ */
+export declare function parseRetryAfterMs(value: string | undefined, now?: number): number | null;
+export declare function createHttpClient(opts?: CreateHttpClientOptions): HttpClient;
+export interface HeadCheckOptions {
+    http: HttpClient;
+    cache?: Cache;
+    trustedHosts: string[];
+}
+export type HeadCheckResult = {
+    ok: true;
+    status: number;
+    finalUrl: string;
+    redirectChain: string[];
+    host: string;
+} | {
+    ok: false;
+    reason: 'dead-url' | 'wrong-host' | 'too-many-redirects' | 'timeout' | 'network-error';
+    details: string;
+};
+export declare function headCheck(url: string, opts: HeadCheckOptions, signal: AbortSignal): Promise<HeadCheckResult>;
+//# sourceMappingURL=http.d.ts.map

package/dist/http.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../src/http.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,qBAAqB,CAAC;AAMjD,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,OAAO,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAgB,SAAQ,kBAAkB;IACzD,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACnE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAMD,qBAAa,SAAU,SAAQ,KAAK;IACzB,IAAI,EAAG,WAAW,CAAU;IACrC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,SAAkB,KAAK,EAAE,OAAO,CAAC;gBAErB,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO;CAQ9D;AA2CD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAmC/C;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAW5E;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAQxE;AAyDD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAAE,GAAG,GAAE,MAAmB,GAAG,MAAM,GAAG,IAAI,CAUpG;AAkGD,wBAAgB,gBAAgB,CAAC,IAAI,CAAC,EAAE,uBAAuB,GAAG,UAAU,CA0N3E;AAMD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACrF;IACE,EAAE,EAAE,KAAK,CAAC;IACV,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,oBAAoB,GAAG,SAAS,GAAG,eAAe,CAAC;IACvF,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEN,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,gBAAgB,EACtB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,eAAe,CAAC,CAsE1B"}