@byline/core 6.12.0 → 6.13.0

package/dist/extraction/internal.js

@@ -0,0 +1,93 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+const ISO_DATE_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/;
+export function isPlainObject(value) {
+    if (typeof value !== 'object' || value === null)
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+/** A string that both has the form of an ISO 8601 date-time and parses as one. */
+export function isIsoDateTime(value) {
+    return typeof value === 'string' && ISO_DATE_TIME.test(value) && !Number.isNaN(Date.parse(value));
+}
+export const isCount = (value) => Number.isSafeInteger(value) && value >= 0;
+/**
+ * Whether every slot of an array passes a check. It reads slots by index because
+ * `every` skips holes, and it stops at the first failure, so a very long sparse array
+ * cannot stall it.
+ */
+export function everySlot(value, check) {
+    if (!Array.isArray(value))
+        return false;
+    for (let i = 0; i < value.length; i += 1) {
+        if (!check(value[i]))
+            return false;
+    }
+    return true;
+}
+/**
+ * Sorted keys and no insignificant whitespace, for hashing a configuration. `undefined`
+ * object members are dropped, so an unset option hashes like an absent one. Any other
+ * value JSON cannot represent exactly is rejected with a `TypeError` rather than coerced,
+ * because two different values must never canonicalise the same. That includes array
+ * holes and symbol keys.
+ */
+export function canonicalJson(value) {
+    return canonicalise(value, [], true);
+}
+function canonicalise(value, ancestors, dropUndefined) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean') {
+        return JSON.stringify(value);
+    }
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value))
+            throw new TypeError(`${value} is not a JSON value`);
+        return JSON.stringify(value);
+    }
+    if (typeof value !== 'object') {
+        throw new TypeError(`A ${typeof value} is not a JSON value`);
+    }
+    if (ancestors.includes(value))
+        throw new TypeError('A cycle is not a JSON value');
+    const path = [...ancestors, value];
+    if (Array.isArray(value)) {
+        // Indexed, because `map` skips holes. A hole reads as `undefined`, which is rejected.
+        const items = [];
+        for (let i = 0; i < value.length; i += 1) {
+            items.push(canonicalise(value[i], path, dropUndefined));
+        }
+        return `[${items.join(',')}]`;
+    }
+    const prototype = Object.getPrototypeOf(value);
+    if (prototype !== Object.prototype && prototype !== null) {
+        throw new TypeError('Only plain objects are JSON values');
+    }
+    if (Object.getOwnPropertySymbols(value).length > 0) {
+        throw new TypeError('A symbol key is not a JSON value');
+    }
+    const members = Object.entries(value)
+        .filter(([, member]) => !dropUndefined || member !== undefined)
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+        .map(([key, member]) => `${JSON.stringify(key)}:${canonicalise(member, path, dropUndefined)}`);
+    return `{${members.join(',')}}`;
+}
+/**
+ * Whether a value is a strict `JsonValue`. Unlike `canonicalJson`, it rejects an
+ * `undefined` object member, which would be lost on persistence.
+ */
+export function isJsonValue(value) {
+    try {
+        canonicalise(value, [], false);
+        return true;
+    }
+    catch {
+        // Includes a RangeError from a structure nested too deeply to walk.
+        return false;
+    }
+}

package/dist/extraction/outcomes.d.ts

@@ -0,0 +1,20 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { EligibilityUpdateOutcome, ExclusionState, ProjectionOutcome } from '../@types/extraction-types.js';
+/**
+ * Derives whether ineligible text has left search. Reporting surfaces must not
+ * compute this any other way. `projection` is the outcome of the first write
+ * attempted after the eligibility update.
+ *
+ * When the eligibility update failed, the join still includes the old text, so a
+ * successful search write leaves it searchable. That case is reported as pending,
+ * never as excluded.
+ *
+ * @provisional
+ */
+export declare function exclusionState(eligibility: EligibilityUpdateOutcome, projection: ProjectionOutcome | undefined): ExclusionState;

package/dist/extraction/outcomes.js

@@ -0,0 +1,30 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+/**
+ * Derives whether ineligible text has left search. Reporting surfaces must not
+ * compute this any other way. `projection` is the outcome of the first write
+ * attempted after the eligibility update.
+ *
+ * When the eligibility update failed, the join still includes the old text, so a
+ * successful search write leaves it searchable. That case is reported as pending,
+ * never as excluded.
+ *
+ * @provisional
+ */
+export function exclusionState(eligibility, projection) {
+    if (eligibility.kind === 'not-required')
+        return { state: 'not-required' };
+    if (eligibility.kind === 'failed') {
+        return { state: 'pending', reason: 'eligibility-update-failed' };
+    }
+    if (projection === undefined)
+        return { state: 'pending', reason: 'not-attempted' };
+    return projection.kind === 'written'
+        ? { state: 'excluded' }
+        : { state: 'pending', reason: projection.reason };
+}

package/dist/extraction/outcomes.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/extraction/outcomes.test.node.js

@@ -0,0 +1,55 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { describe, expect, it } from 'vitest';
+import { exclusionState } from './outcomes.js';
+const written = { kind: 'written' };
+const pending = (reason) => ({
+    kind: 'pending',
+    reason,
+});
+describe('exclusionState', () => {
+    it.each([
+        ['written', written],
+        ['pending', pending('write-failed')],
+        ['not attempted', undefined],
+    ])('is not-required when no update was required and projection is %s', (_label, projection) => {
+        expect(exclusionState({ kind: 'not-required' }, projection)).toEqual({ state: 'not-required' });
+    });
+    it('reports pending when the eligibility mark failed and the search write succeeded', () => {
+        // The regression this rule exists for: the join still includes the old text, so
+        // a successful write leaves it searchable.
+        expect(exclusionState({ kind: 'failed' }, written)).toEqual({
+            state: 'pending',
+            reason: 'eligibility-update-failed',
+        });
+    });
+    it.each([
+        ['pending', pending('write-failed')],
+        ['not attempted', undefined],
+    ])('reports pending when the mark failed and projection is %s', (_label, projection) => {
+        expect(exclusionState({ kind: 'failed' }, projection)).toEqual({
+            state: 'pending',
+            reason: 'eligibility-update-failed',
+        });
+    });
+    it('reports excluded when the mark was recorded and the write acknowledged', () => {
+        expect(exclusionState({ kind: 'recorded' }, written)).toEqual({ state: 'excluded' });
+    });
+    it.each(['write-failed', 'not-attempted', 'stale-write'])('carries the %s reason when the mark was recorded', (reason) => {
+        expect(exclusionState({ kind: 'recorded' }, pending(reason))).toEqual({
+            state: 'pending',
+            reason,
+        });
+    });
+    it('reports not-attempted when the mark was recorded and no write followed', () => {
+        expect(exclusionState({ kind: 'recorded' }, undefined)).toEqual({
+            state: 'pending',
+            reason: 'not-attempted',
+        });
+    });
+});

package/dist/extraction/page-map.d.ts

@@ -0,0 +1,40 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+/**
+ * A code for one broken page map rule.
+ *
+ * @provisional
+ */
+export type PageMapIssueCode = 'invalid-shape' | 'invalid-version' | 'invalid-separator' | 'invalid-trailing' | 'invalid-offset' | 'surrogate-split' | 'invalid-page-index' | 'page-count-mismatch' | 'page-overlap' | 'separator-mismatch' | 'text-outside-ranges' | 'unanchored-overlap' | 'unanchored-before-last-page' | 'unanchored-after-last-page' | 'block-unknown-page' | 'block-outside-page' | 'block-overlap';
+/**
+ * One broken page map rule, and where it broke.
+ *
+ * @provisional
+ */
+export interface PageMapIssue {
+    code: PageMapIssueCode;
+    /** For example `pages[1]`. */
+    path?: string;
+    message: string;
+}
+/**
+ * Checks the structure of a page map against the exact text it describes.
+ *
+ * Structural validity is not provenance. Coverage does not prove that page 3's text
+ * came from physical page 3; only sentinel verification against a fixture does.
+ *
+ * @provisional
+ */
+export declare function validatePageMap(plainText: string, map: unknown, options?: {
+    pageCount?: number;
+}): {
+    ok: true;
+} | {
+    ok: false;
+    issues: PageMapIssue[];
+};

package/dist/extraction/page-map.js

@@ -0,0 +1,190 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { everySlot } from './internal.js';
+const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value);
+const overlaps = (a, b) => a.charStart < b.charEnd && b.charStart < a.charEnd;
+function splitsSurrogatePair(text, boundary) {
+    if (boundary <= 0 || boundary >= text.length)
+        return false;
+    const before = text.charCodeAt(boundary - 1);
+    const after = text.charCodeAt(boundary);
+    return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff;
+}
+/**
+ * Checks the structure of a page map against the exact text it describes.
+ *
+ * Structural validity is not provenance. Coverage does not prove that page 3's text
+ * came from physical page 3; only sentinel verification against a fixture does.
+ *
+ * @provisional
+ */
+export function validatePageMap(plainText, map, options) {
+    const issues = [];
+    const report = (code, message, path) => {
+        issues.push({ code, message, ...(path != null && { path }) });
+    };
+    const finish = () => issues.length === 0 ? { ok: true } : { ok: false, issues };
+    if (!isRecord(map)) {
+        report('invalid-shape', 'the page map is not an object');
+        return finish();
+    }
+    // Rule 1: version and separator.
+    if (map.version !== 1)
+        report('invalid-version', 'version must be 1', 'version');
+    const assembly = isRecord(map.assembly) ? map.assembly : {};
+    const separator = assembly.pageSeparator;
+    if (typeof separator !== 'string') {
+        report('invalid-separator', 'pageSeparator must be a string', 'assembly.pageSeparator');
+    }
+    const trailing = assembly.trailing;
+    if (trailing !== undefined && trailing !== 'embedded' && trailing !== 'none') {
+        report('invalid-trailing', "trailing must be 'embedded' or 'none'", 'assembly.trailing');
+    }
+    const listOf = (key, required) => {
+        const value = map[key];
+        if (value === undefined && !required)
+            return [];
+        if (!everySlot(value, isRecord)) {
+            report('invalid-shape', `${key} must be an array of objects, with no holes`, key);
+            return [];
+        }
+        return value;
+    };
+    const rawPages = listOf('pages', true);
+    const rawUnanchored = listOf('unanchored', false);
+    const rawBlocks = listOf('blocks', false);
+    rawPages.forEach((page, i) => {
+        if (page.label !== undefined && typeof page.label !== 'string') {
+            report('invalid-shape', 'a page label must be a string', `pages[${i}]`);
+        }
+    });
+    rawUnanchored.forEach((range, i) => {
+        if (typeof range.kind !== 'string') {
+            report('invalid-shape', 'an unanchored range needs a string kind', `unanchored[${i}]`);
+        }
+    });
+    rawBlocks.forEach((block, i) => {
+        if (typeof block.kind !== 'string') {
+            report('invalid-shape', 'a block needs a string kind', `blocks[${i}]`);
+        }
+    });
+    // Rules 2 and 3: every offset, in every list.
+    const offsetsAreValid = (entry, path) => {
+        const { charStart, charEnd } = entry;
+        if (!Number.isInteger(charStart) ||
+            !Number.isInteger(charEnd) ||
+            charStart < 0 ||
+            charStart > charEnd ||
+            charEnd > plainText.length) {
+            report('invalid-offset', 'offsets must be integers within the text, in order', path);
+            return false;
+        }
+        if (splitsSurrogatePair(plainText, charStart) ||
+            splitsSurrogatePair(plainText, charEnd)) {
+            report('surrogate-split', 'a boundary splits a UTF-16 surrogate pair', path);
+        }
+        return true;
+    };
+    const allOffsetsValid = [
+        ...rawPages.map((entry, i) => offsetsAreValid(entry, `pages[${i}]`)),
+        ...rawUnanchored.map((entry, i) => offsetsAreValid(entry, `unanchored[${i}]`)),
+        ...rawBlocks.map((entry, i) => offsetsAreValid(entry, `blocks[${i}]`)),
+    ].every(Boolean);
+    // Rule 4: page indexes.
+    let previousIndex = 0;
+    rawPages.forEach((page, i) => {
+        const { index } = page;
+        if (!Number.isInteger(index) || index <= previousIndex) {
+            report('invalid-page-index', 'indexes must be positive and strictly increase', `pages[${i}]`);
+        }
+        if (Number.isInteger(index))
+            previousIndex = Math.max(previousIndex, index);
+    });
+    if (options?.pageCount !== undefined) {
+        const exact = rawPages.length === options.pageCount && rawPages.every((page, i) => page.index === i + 1);
+        if (!exact) {
+            report('page-count-mismatch', `indexes must be exactly 1 to ${options.pageCount}; an empty page is a zero-length range`, 'pages');
+        }
+    }
+    // The remaining rules compare ranges and text, so they need sound offsets.
+    if (!allOffsetsValid || typeof separator !== 'string')
+        return finish();
+    const pages = rawPages;
+    const unanchored = rawUnanchored;
+    const blocks = rawBlocks;
+    // Rule 5: pages are ordered and do not overlap.
+    for (let i = 1; i < pages.length; i += 1) {
+        const previous = pages[i - 1];
+        const page = pages[i];
+        if (page.charStart < previous.charEnd) {
+            report('page-overlap', 'pages must be in text order and must not overlap', `pages[${i}]`);
+        }
+    }
+    // Rule 8: unanchored ranges.
+    unanchored.forEach((range, i) => {
+        if (pages.some((page) => overlaps(range, page))) {
+            report('unanchored-overlap', 'an unanchored range overlaps a page', `unanchored[${i}]`);
+        }
+        if (unanchored.slice(0, i).some((other) => overlaps(range, other))) {
+            report('unanchored-overlap', 'unanchored ranges overlap', `unanchored[${i}]`);
+        }
+    });
+    const lastPage = pages.at(-1);
+    if (lastPage !== undefined) {
+        unanchored.forEach((range, i) => {
+            if (trailing === 'embedded' && range.charStart < lastPage.charEnd) {
+                report('unanchored-before-last-page', "with trailing 'embedded', unanchored ranges follow the last page", `unanchored[${i}]`);
+            }
+            if (trailing !== 'embedded' && range.charStart >= lastPage.charEnd) {
+                report('unanchored-after-last-page', "without trailing 'embedded', no unanchored range follows the last page", `unanchored[${i}]`);
+            }
+        });
+    }
+    // Rules 6 and 7: complete coverage, in text order across both kinds of range.
+    const ranges = [...pages, ...unanchored].sort((a, b) => a.charStart - b.charStart || a.charEnd - b.charEnd);
+    const first = ranges[0];
+    const last = ranges.at(-1);
+    if (first === undefined || last === undefined) {
+        if (plainText.length > 0)
+            report('text-outside-ranges', 'the text has no ranges');
+    }
+    else {
+        if (first.charStart !== 0)
+            report('text-outside-ranges', 'text precedes the first range');
+        if (last.charEnd !== plainText.length) {
+            report('text-outside-ranges', 'text follows the last range');
+        }
+    }
+    for (let i = 1; i < ranges.length; i += 1) {
+        const previous = ranges[i - 1];
+        const range = ranges[i];
+        // An overlap was reported above; it has no gap to compare.
+        if (range.charStart < previous.charEnd)
+            continue;
+        if (plainText.slice(previous.charEnd, range.charStart) !== separator) {
+            report('separator-mismatch', `the text between offsets ${previous.charEnd} and ${range.charStart} is not the separator`);
+        }
+    }
+    // Rule 9: blocks.
+    blocks.forEach((block, i) => {
+        const page = pages.find((candidate) => candidate.index === block.page);
+        const path = `blocks[${i}]`;
+        if (page === undefined) {
+            report('block-unknown-page', `block names page ${String(block.page)}, which is absent`, path);
+            return;
+        }
+        if (block.charStart < page.charStart || block.charEnd > page.charEnd) {
+            report('block-outside-page', 'a block lies outside the page it names', path);
+        }
+        const earlier = blocks.slice(0, i).filter((other) => other.page === block.page);
+        if (earlier.some((other) => overlaps(block, other))) {
+            report('block-overlap', 'blocks of one page overlap', path);
+        }
+    });
+    return finish();
+}

package/dist/extraction/page-map.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};