@byline/core 6.12.1 → 6.13.0

package/dist/@types/extraction-types.d.ts

@@ -0,0 +1,423 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+/**
+ * The extraction-provider seam: the contract that makes document extractors
+ * interchangeable drivers, in the same way `SearchProvider` makes search engines
+ * interchangeable.
+ *
+ * Every export in this module is **provisional**. Until the contract is declared
+ * stable, a minor release may change it, and the changelog will say so.
+ * `ExtractionArtifact.schemaVersion` and `PageMapV1.version` version the persisted
+ * shapes independently of the package version.
+ *
+ * A provider is a pure `bytes -> output` driver. It never reads Byline documents,
+ * storage, search or access state, never persists anything and never decides
+ * eligibility. Those duties belong to orchestration.
+ */
+import type { JsonValue } from './field-data-types.js';
+import type { StorageValidator } from './storage-types.js';
+/**
+ * What a driver supports. Consumers gate on it and never assume it. A provider
+ * that declares `markdown`, `structure` or `pageMap` as `false` must not emit that
+ * member of its result.
+ *
+ * @provisional
+ */
+export interface ExtractorCapabilities {
+    /** Recognises text on image-only pages. */
+    ocr: boolean;
+    /** Emits structure-aware markdown. */
+    markdown: boolean;
+    /** Reconstructs table structure. */
+    tables: boolean;
+    /** Recognises formulas. */
+    formulas: boolean;
+    /** Attempts figure and chart understanding. */
+    charts: boolean;
+    /** Reports block positions or reading order. */
+    coordinates: boolean;
+    /** Emits a page map version 1 from a page-aware assembly. */
+    pageMap: boolean;
+    /** Emits provider-native `structure`. */
+    structure: boolean;
+    /** Accepted MIME types; `*` + `/*` or `type/*` wildcards are allowed. */
+    formats: string[];
+}
+/**
+ * Hints a driver can act on.
+ *
+ * @provisional
+ */
+export interface ExtractionHints {
+    /** The extension is a parser-selection hint. */
+    filename?: string;
+    /** The declared type; a parser-selection hint. */
+    mimeType?: string;
+    /** BCP 47; selects an OCR model. */
+    language?: string;
+    /** An explicit OCR language list, when it differs from `language`. */
+    ocrLanguages?: string[];
+}
+/**
+ * The name of one extraction hint.
+ *
+ * @provisional
+ */
+export type ExtractionHintName = keyof ExtractionHints;
+/**
+ * One file to extract.
+ *
+ * @provisional
+ */
+export interface ExtractionInput extends ExtractionHints {
+    /**
+     * Verified source bytes: a web stream for large files, or a byte array. A
+     * provider consumes the stream to its end or cancels it. The caller owns any
+     * spool file behind it.
+     */
+    data: ReadableStream<Uint8Array> | Uint8Array;
+    /** Aborts the extraction. The provider releases its resources. */
+    signal?: AbortSignal;
+}
+/**
+ * Document-level metadata a driver recovered from the file.
+ *
+ * @provisional
+ */
+export interface ExtractionMetadata {
+    /** Detected MIME type. */
+    contentType?: string;
+    /** Pages as reported by the file's metadata. Prefer `pageCount` on the result. */
+    pages?: number;
+    title?: string;
+    authors?: string[];
+    /** Declared or detected language, as embedded. */
+    language?: string;
+    /** ISO 8601. A provider that cannot normalise an embedded date omits this member. */
+    createdAt?: string;
+    /** The producing application. */
+    producer?: string;
+}
+/**
+ * Cheap classification inputs for a router. They are not page boundaries:
+ * `charsPerPage` must never be used to derive page-map offsets, because it is
+ * counted over text that is normalised afterwards.
+ *
+ * @provisional
+ */
+export interface RoutingSignals {
+    totalChars: number;
+    charsPerPage?: number[];
+    ocrCandidatePages?: number;
+    /** Share of pages with a usable text layer, from 0 to 1. */
+    textLayerCoverage?: number;
+}
+/**
+ * A page map locates each physical page of a paginated source within `plainText`.
+ * Offsets are half-open UTF-16 code-unit indices. `validatePageMap` checks the
+ * structure; only sentinel verification against a fixture proves that page 3's
+ * text came from physical page 3.
+ *
+ * @provisional
+ */
+export interface PageMapV1 {
+    version: 1;
+    assembly: {
+        /** Inserted between consecutive ranges. */
+        pageSeparator: string;
+        /** What follows the last page. */
+        trailing?: 'embedded' | 'none';
+    };
+    pages: Array<{
+        index: number;
+        label?: string;
+        charStart: number;
+        charEnd: number;
+    }>;
+    unanchored?: Array<{
+        kind: string;
+        charStart: number;
+        charEnd: number;
+    }>;
+    blocks?: Array<{
+        page: number;
+        charStart: number;
+        charEnd: number;
+        kind: string;
+    }>;
+}
+/**
+ * The normalised output of one extraction.
+ *
+ * @provisional
+ */
+export interface ExtractionResult {
+    /** Equals `ExtractionProvider.name`. */
+    provider: string;
+    /**
+     * The engine version as reported at run time. Absent when the provider cannot
+     * report it. A provider that implements `resolveVersion` reports the same value.
+     */
+    providerVersion?: string;
+    /** Always present; `''` when nothing is extractable. */
+    plainText: string;
+    /** Only when `capabilities.markdown`. */
+    markdown?: string;
+    /** Provider-native and opaque. Only when `capabilities.structure`. */
+    structure?: JsonValue;
+    /** Only when `capabilities.pageMap`. */
+    pageMap?: PageMapV1;
+    /** Physical pages, when the format has pages. */
+    pageCount?: number;
+    metadata: ExtractionMetadata;
+    signals: RoutingSignals;
+}
+/**
+ * An extraction driver.
+ *
+ * @provisional
+ */
+export interface ExtractionProvider {
+    /** Stable driver identifier, recorded on every result. */
+    readonly name: string;
+    /** What the driver supports. */
+    readonly capabilities: ExtractorCapabilities;
+    /**
+     * Effective output-affecting configuration, as JSON. Included in the producer
+     * configuration hash. Omit secrets and endpoints.
+     */
+    readonly configuration?: Readonly<Record<string, JsonValue>>;
+    /** Hints from `ExtractionHints` that change this driver's output. */
+    readonly outputAffectingHints?: readonly ExtractionHintName[];
+    /**
+     * The engine version `extract` would report now, without extracting. Rejects
+     * with an `infrastructure` error when the service cannot answer.
+     */
+    resolveVersion?(options?: {
+        signal?: AbortSignal;
+    }): Promise<string>;
+    /** Extracts one file. Rejects with `ExtractionProviderError` on failure. */
+    extract(input: ExtractionInput): Promise<ExtractionResult>;
+    /** Boot-time health check for a remote service. */
+    ping?(): Promise<void>;
+}
+/**
+ * Whether a provider failure describes the input file or its infrastructure.
+ * An `infrastructure` failure must not record the file as bad and must preserve
+ * existing artifacts.
+ *
+ * @provisional
+ */
+export type ExtractionFailureScope = 'file' | 'infrastructure';
+/**
+ * The persisted output of one successful extraction. Document, version, path,
+ * access, selection and verification evidence are deliberately absent: they belong
+ * to references, so one artifact can serve every authorised reference to the same
+ * bytes and producer. An artifact is immutable.
+ *
+ * A reader that meets an unknown `schemaVersion` treats the artifact as unusable
+ * for reuse, never as malformed source data.
+ *
+ * @provisional
+ */
+export interface ExtractionArtifact {
+    schemaVersion: 1;
+    /** SHA-256 of the source bytes; `null` only for legacy imports. */
+    contentHash: string | null;
+    /** `true` exactly when `contentHash` came from a verified read. */
+    sourceVerified: boolean;
+    provider: string;
+    providerVersion: string | null;
+    /**
+     * `null` only for a historical import whose configuration was never recorded. A
+     * new extraction always records the hash from its producer identity. A current
+     * hash is never written onto historical output.
+     */
+    producerConfigHash: string | null;
+    plainText: string;
+    markdown?: string;
+    structure?: JsonValue;
+    pageMap?: PageMapV1;
+    pageCount?: number;
+    metadata: ExtractionMetadata;
+    signals: RoutingSignals;
+    /** ISO 8601. */
+    producedAt: string;
+}
+/**
+ * The stored object cannot be confirmed to be the file the document records.
+ *
+ * @provisional
+ */
+export type SourceIdentityProblem = 'missing-hash' | 'missing-object' | 'hash-mismatch';
+/**
+ * Why a source could not be read. None of these says anything about the file.
+ *
+ * @provisional
+ */
+export type SourceUnavailableReason = 'download-unsupported' | 'provider-mismatch' | 'read-failed' | 'aborted';
+/**
+ * Why a read that expected a validator fell back to one unconditional read. A
+ * changed token does not prove changed content; the hash decides.
+ *
+ * @provisional
+ */
+export type PreconditionFallbackReason = 'changed' | 'unsupported-kind' | 'unestablishable';
+/**
+ * The record that binds an extraction to one verified read at one time. It is not
+ * continuous storage monitoring, and a hash copied from the saved file value is not
+ * evidence. It is persisted with the reference, never on the immutable artifact.
+ *
+ * @provisional
+ */
+export interface SourceVerificationEvidence {
+    /** The reference's `fileHash` at read time. */
+    expectedHash: string;
+    /** SHA-256 measured from the bytes read; equals `expectedHash`. */
+    contentHash: string;
+    /** The validator returned with those bytes. */
+    validator: StorageValidator;
+    /** ISO 8601. */
+    verifiedAt: string;
+    size?: number;
+    readBasis?: 'expected' | 'unconditional' | 'fallback';
+    preconditionFallback?: PreconditionFallbackReason;
+}
+/**
+ * Why extraction work is required.
+ *
+ * @provisional
+ */
+export type ExtractionWorkReason = 'force'
+/** Nothing usable exists for this reference and producer. */
+ | 'no-artifact'
+/** The last attempt failed or hit an identity problem. */
+ | 'previous-attempt-unsuccessful'
+/** The reference's `fileHash` differs from the artifact's `contentHash`. */
+ | 'source-changed'
+/** There is no valid evidence for the current hash. */
+ | 'unverified'
+/** The provider, version or configuration hash differs. */
+ | 'producer-changed'
+/** The artifact's `schemaVersion` is not the current one. */
+ | 'schema-changed';
+/**
+ * The result of `decideExtractionWork`.
+ *
+ * `versionUnchecked` is reduced assurance: the current engine version and the
+ * deployment identifier are both unknown, so an engine upgrade is not detected.
+ * `producerUnchecked` is reduced assurance of the same kind, and is `true` only for
+ * historical output with no recorded configuration: that configuration was not
+ * compared, and a `null` recorded engine version was not compared either.
+ *
+ * @provisional
+ */
+export type ExtractionWorkDecision = {
+    kind: 'identity-problem';
+    problem: 'missing-hash';
+} | {
+    kind: 'work';
+    reasons: ExtractionWorkReason[];
+    versionUnchecked: boolean;
+} | {
+    kind: 'reuse';
+    evidence: SourceVerificationEvidence;
+    versionUnchecked: boolean;
+    producerUnchecked: boolean;
+};
+/**
+ * Why a promotion was refused.
+ *
+ * @provisional
+ */
+export type PromotionRefusalReason = 'hash-changed' | 'version-changed' | 'access-changed' | 'artifact-mismatch' | 'claim-lost';
+/**
+ * Was the source verified, and was extraction reused, completed, unavailable,
+ * blocked or refused?
+ *
+ * @provisional
+ */
+export type DerivationOutcome = {
+    kind: 'reused';
+} | {
+    kind: 'extracted';
+} | {
+    kind: 'identity-problem';
+    problem: SourceIdentityProblem;
+} | {
+    kind: 'source-unavailable';
+    reason: SourceUnavailableReason;
+} | {
+    kind: 'provider-failed';
+    failureScope: ExtractionFailureScope;
+    retryable: boolean;
+} | {
+    kind: 'promotion-refused';
+    reason: PromotionRefusalReason;
+};
+/**
+ * Whether making previously eligible text ineligible was durably recorded.
+ *
+ * @provisional
+ */
+export type EligibilityUpdateOutcome = 
+/** Nothing previously eligible became ineligible. */
+{
+    kind: 'not-required';
+}
+/** Ineligibility was durably recorded. */
+ | {
+    kind: 'recorded';
+}
+/** Recording failed; the join still includes the old text. */
+ | {
+    kind: 'failed';
+};
+/**
+ * Whether search reflects the current eligible selection. `written` means the
+ * provider acknowledged a write. It is not a durable exclusion guarantee.
+ *
+ * @provisional
+ */
+export type ProjectionOutcome = {
+    kind: 'written';
+} | {
+    kind: 'pending';
+    reason: 'write-failed' | 'not-attempted' | 'stale-write';
+};
+/**
+ * Whether ineligible text has left search. Derive it only with `exclusionState`.
+ * `excluded` means the eventual level succeeded for this write; it is not the
+ * durable read-time guarantee.
+ *
+ * @provisional
+ */
+export type ExclusionState = {
+    state: 'not-required';
+} | {
+    state: 'excluded';
+} | {
+    state: 'pending';
+    reason: 'eligibility-update-failed' | 'write-failed' | 'not-attempted' | 'stale-write';
+};
+/**
+ * The independent facts about one attachment after processing. A save, an extraction
+ * and a search update cannot be reported with one `success` flag.
+ *
+ * @provisional
+ */
+export interface AttachmentProcessingReport {
+    documentId: string;
+    documentVersionId?: string;
+    storagePath: string;
+    runId?: string;
+    derivation: DerivationOutcome;
+    eligibility?: EligibilityUpdateOutcome;
+    projection?: ProjectionOutcome;
+    exclusion?: ExclusionState;
+}

package/dist/@types/extraction-types.js

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/@types/index.d.ts

@@ -9,6 +9,7 @@ export * from './admin-types.js';
 export * from './collection-types.js';
 export * from './db-types.js';
 export * from './document-revision.js';
+export * from './extraction-types.js';
 export * from './field-data-types.js';
 export * from './field-types.js';
 export * from './populate-types.js';

package/dist/@types/index.js

@@ -9,6 +9,7 @@ export * from './admin-types.js';
 export * from './collection-types.js';
 export * from './db-types.js';
 export * from './document-revision.js';
+export * from './extraction-types.js';
 export * from './field-data-types.js';
 export * from './field-types.js';
 export * from './populate-types.js';

package/dist/extraction/artifact.d.ts

@@ -0,0 +1,46 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { type PageMapIssue } from './page-map.js';
+/** The persisted artifact shape this release reads and writes. */
+export declare const EXTRACTION_ARTIFACT_SCHEMA_VERSION = 1;
+/**
+ * A code for one broken artifact rule.
+ *
+ * @provisional
+ */
+export type ExtractionArtifactIssueCode = 'invalid-shape' | 'unsupported-schema-version' | 'invalid-content-hash' | 'source-verified-mismatch' | 'invalid-provider' | 'invalid-provider-version' | 'invalid-producer-config-hash' | 'unrecorded-producer' | 'invalid-plain-text' | 'invalid-markdown' | 'invalid-structure' | 'invalid-page-count' | 'invalid-page-map' | 'invalid-metadata' | 'invalid-signals' | 'invalid-produced-at';
+/**
+ * One broken artifact rule.
+ *
+ * @provisional
+ */
+export interface ExtractionArtifactIssue {
+    code: ExtractionArtifactIssueCode;
+    message: string;
+    /** The page map's own issues, for `invalid-page-map`. */
+    pageMapIssues?: PageMapIssue[];
+}
+/**
+ * Checks a persisted artifact. A `null` `producerConfigHash` fails unless
+ * `allowUnrecordedProducer` is `true`, which only code reading or importing
+ * historical records passes. A new extraction always records the real hash from its
+ * `ExtractionProducerIdentity`.
+ *
+ * An unknown `schemaVersion` is reported alone: the artifact is unusable for reuse,
+ * and its other members cannot be judged against this version's rules.
+ *
+ * @provisional
+ */
+export declare function validateExtractionArtifact(artifact: unknown, options?: {
+    allowUnrecordedProducer?: boolean;
+}): {
+    ok: true;
+} | {
+    ok: false;
+    issues: ExtractionArtifactIssue[];
+};

package/dist/extraction/artifact.js

@@ -0,0 +1,121 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { CONTENT_HASH_PATTERN } from '../storage/content-hash.js';
+import { everySlot, isCount, isIsoDateTime, isJsonValue, isPlainObject } from './internal.js';
+import { validatePageMap } from './page-map.js';
+/** The persisted artifact shape this release reads and writes. */
+export const EXTRACTION_ARTIFACT_SCHEMA_VERSION = 1;
+const isOptional = (value, check) => value === undefined || check(value);
+const isString = (value) => typeof value === 'string';
+/** Recognised members only; unknown members are ignored, as in stored evidence. */
+function isValidMetadata(metadata) {
+    if (!isPlainObject(metadata))
+        return false;
+    const { contentType, pages, title, authors, language, createdAt, producer } = metadata;
+    return ([contentType, title, language, producer].every((member) => isOptional(member, isString)) &&
+        isOptional(createdAt, isIsoDateTime) &&
+        isOptional(pages, isCount) &&
+        isOptional(authors, (value) => everySlot(value, isString)));
+}
+/** Recognised members only; unknown members are ignored, as in stored evidence. */
+function isValidSignals(signals) {
+    if (!isPlainObject(signals) || !isCount(signals.totalChars))
+        return false;
+    const { charsPerPage, ocrCandidatePages, textLayerCoverage } = signals;
+    return (isOptional(charsPerPage, (value) => everySlot(value, isCount)) &&
+        isOptional(ocrCandidatePages, isCount) &&
+        isOptional(textLayerCoverage, (value) => typeof value === 'number' && value >= 0 && value <= 1));
+}
+/**
+ * Checks a persisted artifact. A `null` `producerConfigHash` fails unless
+ * `allowUnrecordedProducer` is `true`, which only code reading or importing
+ * historical records passes. A new extraction always records the real hash from its
+ * `ExtractionProducerIdentity`.
+ *
+ * An unknown `schemaVersion` is reported alone: the artifact is unusable for reuse,
+ * and its other members cannot be judged against this version's rules.
+ *
+ * @provisional
+ */
+export function validateExtractionArtifact(artifact, options) {
+    const issues = [];
+    const report = (code, message) => {
+        issues.push({ code, message });
+    };
+    if (!isPlainObject(artifact)) {
+        return {
+            ok: false,
+            issues: [{ code: 'invalid-shape', message: 'the artifact is not a plain object' }],
+        };
+    }
+    if (artifact.schemaVersion !== EXTRACTION_ARTIFACT_SCHEMA_VERSION) {
+        return {
+            ok: false,
+            issues: [
+                {
+                    code: 'unsupported-schema-version',
+                    message: `schemaVersion ${String(artifact.schemaVersion)} is not supported`,
+                },
+            ],
+        };
+    }
+    const { contentHash, producerConfigHash, providerVersion, plainText, pageCount } = artifact;
+    if (contentHash !== null &&
+        (typeof contentHash !== 'string' || !CONTENT_HASH_PATTERN.test(contentHash))) {
+        report('invalid-content-hash', 'contentHash must be a lower-case SHA-256 or null');
+    }
+    if (artifact.sourceVerified !== (contentHash !== null)) {
+        report('source-verified-mismatch', 'sourceVerified must be true exactly when contentHash is set');
+    }
+    if (typeof artifact.provider !== 'string' || artifact.provider === '') {
+        report('invalid-provider', 'provider must be a non-empty string');
+    }
+    if (providerVersion !== null && typeof providerVersion !== 'string') {
+        report('invalid-provider-version', 'providerVersion must be a string or null');
+    }
+    if (producerConfigHash === null) {
+        if (options?.allowUnrecordedProducer !== true) {
+            report('unrecorded-producer', 'a new artifact must record its producerConfigHash');
+        }
+    }
+    else if (typeof producerConfigHash !== 'string' ||
+        !CONTENT_HASH_PATTERN.test(producerConfigHash)) {
+        report('invalid-producer-config-hash', 'producerConfigHash must be a lower-case SHA-256');
+    }
+    if (typeof plainText !== 'string')
+        report('invalid-plain-text', 'plainText must be a string');
+    if (artifact.markdown !== undefined && typeof artifact.markdown !== 'string') {
+        report('invalid-markdown', 'markdown must be a string');
+    }
+    if (artifact.structure !== undefined && !isJsonValue(artifact.structure)) {
+        report('invalid-structure', 'structure must be a JSON value');
+    }
+    if (pageCount !== undefined && !isCount(pageCount)) {
+        report('invalid-page-count', 'pageCount must be a non-negative integer');
+    }
+    if (!isValidMetadata(artifact.metadata)) {
+        report('invalid-metadata', 'metadata must be an object whose known members have valid types');
+    }
+    if (!isValidSignals(artifact.signals)) {
+        report('invalid-signals', 'signals must be an object whose known members have valid types');
+    }
+    if (!isIsoDateTime(artifact.producedAt)) {
+        report('invalid-produced-at', 'producedAt must be an ISO 8601 date-time');
+    }
+    if (artifact.pageMap !== undefined && typeof plainText === 'string') {
+        const pageMap = validatePageMap(plainText, artifact.pageMap, isCount(pageCount) ? { pageCount: pageCount } : undefined);
+        if (!pageMap.ok) {
+            issues.push({
+                code: 'invalid-page-map',
+                message: 'pageMap breaks the page map rules',
+                pageMapIssues: pageMap.issues,
+            });
+        }
+    }
+    return issues.length === 0 ? { ok: true } : { ok: false, issues };
+}

package/dist/extraction/artifact.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};