@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/parsers/large-document/capability-discovery.js

@@ -0,0 +1,76 @@
+// Capability discovery for optional extraction capabilities (Epic #1160, Issue #1286).
+//
+// OCR and multimodal extraction are optional. They are discovered by probing an injected adapter
+// and classified as available | unavailable | degraded | failing. Missing or failing capabilities
+// degrade retrieval coverage but never compromise pipeline stability: the indexing job still
+// completes deterministically with partial-coverage quality warnings. No third-party OCR/vision
+// service is bundled or required by default; the null adapters report "unavailable".
+export const nullMultimodalAdapter = Object.freeze({
+    kind: "multimodal",
+    describeImage: () => Promise.resolve({ ok: false, reason: "not-configured" }),
+});
+const DEFAULT_PROBE_TIMEOUT_MS = 5_000;
+async function withTimeout(work, timeoutMs, onTimeout) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => {
+            resolve(onTimeout());
+        }, timeoutMs);
+    });
+    try {
+        return await Promise.race([work, timeout]);
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+    }
+}
+export async function probeOcrCapability(adapter, timeoutMs) {
+    if (adapter === undefined)
+        return "unavailable";
+    try {
+        const result = await withTimeout(adapter.ocrPage({ bytes: new Uint8Array(0), pageNumber: 1 }), timeoutMs, () => ({ ok: false, reason: "timeout" }));
+        if (result.ok)
+            return "available";
+        switch (result.reason) {
+            case "ocr-not-configured":
+                return "unavailable";
+            case "timeout":
+                return "degraded";
+            case "unsupported-input":
+                // The engine is installed but rejected the empty probe input — still configured.
+                return "available";
+        }
+    }
+    catch {
+        return "failing";
+    }
+}
+export async function probeMultimodalCapability(adapter, timeoutMs) {
+    if (adapter === undefined)
+        return "unavailable";
+    try {
+        const result = await withTimeout(adapter.describeImage({ bytes: new Uint8Array(0), pageNumber: 1 }), timeoutMs, () => ({ ok: false, reason: "timeout" }));
+        if (result.ok)
+            return "available";
+        switch (result.reason) {
+            case "not-configured":
+                return "unavailable";
+            case "timeout":
+                return "degraded";
+            case "unsupported-input":
+                return "available";
+        }
+    }
+    catch {
+        return "failing";
+    }
+}
+export async function discoverExtractionCapabilities(deps = {}) {
+    const timeoutMs = deps.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
+    const [ocr, multimodal] = await Promise.all([
+        probeOcrCapability(deps.ocr, timeoutMs),
+        probeMultimodalCapability(deps.multimodal, timeoutMs),
+    ]);
+    return { ocr, multimodal };
+}

package/dist/parsers/large-document/diagnostics.d.ts

@@ -0,0 +1,3 @@
+import type { DocumentId, LargeDocumentDiagnosticCode, ParserDiagnostic, ParserDiagnosticSeverity } from "@oscharko-dev/keiko-contracts";
+export declare function largeDocumentDiagnostic(code: LargeDocumentDiagnosticCode, message: string, documentId: DocumentId, severity?: ParserDiagnosticSeverity, pageNumber?: number): ParserDiagnostic;
+//# sourceMappingURL=diagnostics.d.ts.map

package/dist/parsers/large-document/diagnostics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/diagnostics.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,UAAU,EACV,2BAA2B,EAC3B,gBAAgB,EAChB,wBAAwB,EACzB,MAAM,+BAA+B,CAAC;AAEvC,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,2BAA2B,EACjC,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,UAAU,EACtB,QAAQ,GAAE,wBAAoC,EAC9C,UAAU,CAAC,EAAE,MAAM,GAClB,gBAAgB,CAIlB"}

package/dist/parsers/large-document/diagnostics.js

@@ -0,0 +1,11 @@
+// Content-free diagnostic builders for the bounded large-document ingestion path
+// (Epic #1160, Issue #1286). The contract `ParserDiagnostic.code` is a free `string`, so the
+// large-document codes (PARTIAL_COVERAGE, OCR_CAPABILITY_UNAVAILABLE, ...) coexist with the
+// closed `ParserErrorCode` union the existing adapters use. These diagnostics are never
+// error-severity for capability/coverage limits, so downstream layers do not treat a missing
+// optional capability as a hard document failure.
+export function largeDocumentDiagnostic(code, message, documentId, severity = "warning", pageNumber) {
+    return pageNumber === undefined
+        ? { severity, code, message, documentId }
+        : { severity, code, message, documentId, pageNumber };
+}

package/dist/parsers/large-document/index.d.ts

@@ -0,0 +1,15 @@
+export { largeDocumentDiagnostic } from "./diagnostics.js";
+export type { ProgressiveExtractionSource, ProgressiveExtractionWindow, ProgressiveExtractionOptions, ProgressiveExtractor, ProgressiveExtractionSink, ProgressiveExtractionSummary, ProgressiveStopReason, } from "./progressive-extraction.js";
+export { runProgressiveExtraction } from "./progressive-extraction.js";
+export { WindowTextBuilder } from "./window-builder.js";
+export type { AddedPage } from "./window-builder.js";
+export { createProgressivePdfExtractor, PROGRESSIVE_PDF_PARSER_ID, PROGRESSIVE_PDF_PARSER_VERSION, PROGRESSIVE_PDF_DEPENDENCY_VERSIONS, } from "./progressive-pdf.js";
+export type { OcrPageFn, ProgressivePdfExtractorDeps } from "./progressive-pdf.js";
+export { syntheticStreamingSource, syntheticProgressiveExtractor } from "./synthetic-source.js";
+export type { SyntheticStreamingConfig } from "./synthetic-source.js";
+export { discoverExtractionCapabilities, probeOcrCapability, probeMultimodalCapability, nullMultimodalAdapter, } from "./capability-discovery.js";
+export type { CapabilityProbeDeps, MultimodalAdapter, MultimodalResult, } from "./capability-discovery.js";
+export { isLegacyBinaryOfficeFormat, legacyFormatGuidance, legacyFormatDiagnostic, } from "./legacy-format.js";
+export { classifyLargeDocument, usesProgressivePath } from "./preflight.js";
+export type { PreflightInput } from "./preflight.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/parsers/large-document/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,YAAY,EACV,2BAA2B,EAC3B,2BAA2B,EAC3B,4BAA4B,EAC5B,oBAAoB,EACpB,yBAAyB,EACzB,4BAA4B,EAC5B,qBAAqB,GACtB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,wBAAwB,EAAE,MAAM,6BAA6B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EACL,6BAA6B,EAC7B,yBAAyB,EACzB,8BAA8B,EAC9B,mCAAmC,GACpC,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,SAAS,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,MAAM,uBAAuB,CAAC;AAChG,YAAY,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EACL,8BAA8B,EAC9B,kBAAkB,EAClB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,2BAA2B,CAAC;AACnC,YAAY,EACV,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC5E,YAAY,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/parsers/large-document/index.js

@@ -0,0 +1,10 @@
+// Public surface for the bounded large-document ingestion parser layer (Epic #1160,
+// Issue #1286).
+export { largeDocumentDiagnostic } from "./diagnostics.js";
+export { runProgressiveExtraction } from "./progressive-extraction.js";
+export { WindowTextBuilder } from "./window-builder.js";
+export { createProgressivePdfExtractor, PROGRESSIVE_PDF_PARSER_ID, PROGRESSIVE_PDF_PARSER_VERSION, PROGRESSIVE_PDF_DEPENDENCY_VERSIONS, } from "./progressive-pdf.js";
+export { syntheticStreamingSource, syntheticProgressiveExtractor } from "./synthetic-source.js";
+export { discoverExtractionCapabilities, probeOcrCapability, probeMultimodalCapability, nullMultimodalAdapter, } from "./capability-discovery.js";
+export { isLegacyBinaryOfficeFormat, legacyFormatGuidance, legacyFormatDiagnostic, } from "./legacy-format.js";
+export { classifyLargeDocument, usesProgressivePath } from "./preflight.js";

package/dist/parsers/large-document/legacy-format.d.ts

@@ -0,0 +1,5 @@
+import type { DocumentId, ParserDiagnostic } from "@oscharko-dev/keiko-contracts";
+export declare function isLegacyBinaryOfficeFormat(extension: string): boolean;
+export declare function legacyFormatGuidance(extension: string): string | undefined;
+export declare function legacyFormatDiagnostic(extension: string, documentId: DocumentId): ParserDiagnostic | undefined;
+//# sourceMappingURL=legacy-format.d.ts.map

package/dist/parsers/large-document/legacy-format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"legacy-format.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/legacy-format.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAWlF,wBAAgB,0BAA0B,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAErE;AAED,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAE1E;AAED,wBAAgB,sBAAsB,CACpC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,UAAU,GACrB,gBAAgB,GAAG,SAAS,CAS9B"}

package/dist/parsers/large-document/legacy-format.js

@@ -0,0 +1,25 @@
+// Controlled legacy-format handling (Epic #1160, Issue #1286).
+//
+// Legacy binary office formats (.doc, .ppt, .xls) predate the OOXML containers the docx/xlsx
+// parsers handle. Keiko does not bundle a converter for them. Rather than crashing or emitting a
+// generic "unknown format" signal, the large-document path reports a stable CONVERTER_UNAVAILABLE
+// diagnostic with actionable, content-free guidance, leaving the indexing job stable.
+import { LARGE_DOCUMENT_DIAGNOSTIC_CODES } from "@oscharko-dev/keiko-contracts";
+import { largeDocumentDiagnostic } from "./diagnostics.js";
+const LEGACY_FORMAT_GUIDANCE = Object.freeze({
+    doc: "Legacy Microsoft Word (.doc) is not supported; re-save the file as .docx or PDF and reindex.",
+    ppt: "Legacy Microsoft PowerPoint (.ppt) is not supported; re-save the file as .pptx or PDF and reindex.",
+    xls: "Legacy Microsoft Excel (.xls) is not supported; re-save the file as .xlsx or CSV and reindex.",
+});
+export function isLegacyBinaryOfficeFormat(extension) {
+    return extension.toLowerCase() in LEGACY_FORMAT_GUIDANCE;
+}
+export function legacyFormatGuidance(extension) {
+    return LEGACY_FORMAT_GUIDANCE[extension.toLowerCase()];
+}
+export function legacyFormatDiagnostic(extension, documentId) {
+    const guidance = legacyFormatGuidance(extension);
+    if (guidance === undefined)
+        return undefined;
+    return largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.CONVERTER_UNAVAILABLE, guidance, documentId, "warning");
+}

package/dist/parsers/large-document/preflight.d.ts

@@ -0,0 +1,9 @@
+import type { LargeDocumentPreflight, LargeDocumentResourcePolicy } from "@oscharko-dev/keiko-contracts";
+export interface PreflightInput {
+    readonly extension: string;
+    readonly mediaType: string;
+    readonly sizeBytes: number;
+}
+export declare function classifyLargeDocument(input: PreflightInput, policy: LargeDocumentResourcePolicy): LargeDocumentPreflight;
+export declare function usesProgressivePath(preflight: LargeDocumentPreflight): boolean;
+//# sourceMappingURL=preflight.d.ts.map

package/dist/parsers/large-document/preflight.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"preflight.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/preflight.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,sBAAsB,EACtB,2BAA2B,EAC5B,MAAM,+BAA+B,CAAC;AAIvC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAQD,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,cAAc,EACrB,MAAM,EAAE,2BAA2B,GAClC,sBAAsB,CA8BxB;AAID,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,sBAAsB,GAAG,OAAO,CAE9E"}

package/dist/parsers/large-document/preflight.js

@@ -0,0 +1,43 @@
+// Preflight large-file classification (Epic #1160, Issue #1286).
+//
+// Runs before parser execution. Decides the extraction strategy from the file's size, extension,
+// and media type against the resource policy, so a file above the policy ceiling is rejected
+// before any unsafe work begins, and a large supported PDF is routed to the progressive path.
+// Pure function — no IO, no clock.
+import { isLegacyBinaryOfficeFormat } from "./legacy-format.js";
+function isPdf(input) {
+    return (input.extension.toLowerCase() === "pdf" || input.mediaType.toLowerCase() === "application/pdf");
+}
+export function classifyLargeDocument(input, policy) {
+    if (input.sizeBytes > policy.maxRawFileBytes) {
+        return {
+            strategy: "oversized",
+            decision: "reject-oversized",
+            sizeBytes: input.sizeBytes,
+            reason: `file size exceeds the configured ${String(policy.maxRawFileBytes)}-byte raw ceiling`,
+        };
+    }
+    if (isLegacyBinaryOfficeFormat(input.extension)) {
+        return {
+            strategy: "unsupported",
+            decision: "accept-standard",
+            sizeBytes: input.sizeBytes,
+            reason: "legacy binary office format has no bundled converter",
+        };
+    }
+    if (isPdf(input) && input.sizeBytes >= policy.largeFileThresholdBytes) {
+        const estimatedWindows = Math.max(1, Math.ceil(input.sizeBytes / (policy.largeFileThresholdBytes / 4)));
+        return {
+            strategy: "progressive-pdf",
+            decision: "accept-progressive",
+            sizeBytes: input.sizeBytes,
+            estimatedWindows,
+        };
+    }
+    return { strategy: "standard-buffer", decision: "accept-standard", sizeBytes: input.sizeBytes };
+}
+// True when the preflight decision means the discovery layer should run the progressive,
+// page-windowed extraction path instead of the existing full-buffer path.
+export function usesProgressivePath(preflight) {
+    return preflight.decision === "accept-progressive";
+}

package/dist/parsers/large-document/progressive-extraction.d.ts

@@ -0,0 +1,55 @@
+import type { CoverageQuality, DocumentId, LargeDocumentExtractionStrategy, LargeDocumentResourcePolicy, PageRecord, ParsedUnit, ParserDependencyVersion, ParserDiagnostic } from "@oscharko-dev/keiko-contracts";
+export interface ProgressiveExtractionSource {
+    readonly totalBytes: number;
+    readonly loadFullBuffer?: () => Promise<Uint8Array>;
+    readonly readWindow?: (startByte: number, length: number) => Promise<Uint8Array>;
+}
+export interface ProgressiveExtractionWindow {
+    readonly windowIndex: number;
+    readonly pages: readonly PageRecord[];
+    readonly units: readonly ParsedUnit[];
+    readonly text: string;
+    readonly characterStart: number;
+    readonly objectCursor: number;
+    readonly lastPageNumber: number;
+    readonly diagnostics: readonly ParserDiagnostic[];
+}
+export interface ProgressiveExtractionOptions {
+    readonly documentId: DocumentId;
+    readonly extension: string;
+    readonly mediaType: string;
+    readonly policy: LargeDocumentResourcePolicy;
+    readonly signal?: AbortSignal;
+    readonly now: () => number;
+    readonly resumeFromPage?: number;
+    readonly resumeCharacterStart?: number;
+    readonly resumeWindowIndex?: number;
+    readonly resumeObjectCursor?: number;
+    readonly resumeExtractedTextBytes?: number;
+}
+export interface ProgressiveExtractor {
+    readonly strategyId: LargeDocumentExtractionStrategy;
+    readonly parserVersion: string;
+    readonly dependencyVersions?: readonly ParserDependencyVersion[];
+    readonly matches: (input: {
+        readonly extension: string;
+        readonly mediaType: string;
+    }) => boolean;
+    readonly extractWindows: (source: ProgressiveExtractionSource, options: ProgressiveExtractionOptions) => AsyncIterable<ProgressiveExtractionWindow>;
+}
+export interface ProgressiveExtractionSink {
+    readonly onWindow: (window: ProgressiveExtractionWindow) => Promise<void> | void;
+}
+export type ProgressiveStopReason = "completed" | "cancelled" | "extracted-text-limit" | "object-limit" | "unit-limit" | "timeout";
+export interface ProgressiveExtractionSummary {
+    readonly pageCount: number;
+    readonly windowCount: number;
+    readonly extractedTextBytes: number;
+    readonly objectCursor: number;
+    readonly lastPageNumber: number;
+    readonly diagnostics: readonly ParserDiagnostic[];
+    readonly stopReason: ProgressiveStopReason;
+    readonly coverage: CoverageQuality;
+}
+export declare function runProgressiveExtraction(extractor: ProgressiveExtractor, source: ProgressiveExtractionSource, options: ProgressiveExtractionOptions, sink: ProgressiveExtractionSink): Promise<ProgressiveExtractionSummary>;
+//# sourceMappingURL=progressive-extraction.d.ts.map

package/dist/parsers/large-document/progressive-extraction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressive-extraction.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/progressive-extraction.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,+BAA+B,EAC/B,2BAA2B,EAC3B,UAAU,EACV,UAAU,EACV,uBAAuB,EACvB,gBAAgB,EACjB,MAAM,+BAA+B,CAAC;AAMvC,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAE5B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC;IAGpD,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;CAClF;AAGD,MAAM,WAAW,2BAA2B;IAC1C,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAE7B,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IAItC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAEhC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAE9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACnD;AAGD,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,2BAA2B,CAAC;IAC7C,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;IAG3B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,MAAM,CAAC;IACvC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,MAAM,CAAC;CAC5C;AAGD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,UAAU,EAAE,+BAA+B,CAAC;IACrD,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,SAAS,uBAAuB,EAAE,CAAC;IACjE,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC;IACjG,QAAQ,CAAC,cAAc,EAAE,CACvB,MAAM,EAAE,2BAA2B,EACnC,OAAO,EAAE,4BAA4B,KAClC,aAAa,CAAC,2BAA2B,CAAC,CAAC;CACjD;AAGD,MAAM,WAAW,yBAAyB;IAGxC,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,2BAA2B,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CAClF;AAED,MAAM,MAAM,qBAAqB,GAC7B,WAAW,GACX,WAAW,GACX,sBAAsB,GACtB,cAAc,GACd,YAAY,GACZ,SAAS,CAAC;AAEd,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAClD,QAAQ,CAAC,UAAU,EAAE,qBAAqB,CAAC;IAC3C,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;CACpC;AAuGD,wBAAsB,wBAAwB,CAC5C,SAAS,EAAE,oBAAoB,EAC/B,MAAM,EAAE,2BAA2B,EACnC,OAAO,EAAE,4BAA4B,EACrC,IAAI,EAAE,yBAAyB,GAC9B,OAAO,CAAC,4BAA4B,CAAC,CA+CvC"}

package/dist/parsers/large-document/progressive-extraction.js

@@ -0,0 +1,123 @@
+// Progressive (page-windowed) extraction contract + driver for bounded large-document
+// ingestion (Epic #1160, Issue #1286).
+//
+// This contract coexists with the existing full-buffer `ParserAdapter`/`AsyncParserAdapter`
+// contracts. A `ProgressiveExtractor` yields the document as an ordered sequence of bounded
+// `ProgressiveExtractionWindow` values. The driver `runProgressiveExtraction` pulls one window
+// at a time, hands it to a sink that persists it durably, and never retains more than one
+// window's text — so the extraction working set does not scale with document size.
+//
+// The `ProgressiveExtractionSource` abstracts byte access: a `loadFullBuffer` source backs the
+// real PDF strategy (pdfjs-dist requires the whole buffer to open a document — see the
+// architecture doc), while a `readWindow` source backs truly-streaming inputs that never
+// materialize the whole document (used by the bounded-RSS regression).
+import { LARGE_DOCUMENT_DIAGNOSTIC_CODES } from "@oscharko-dev/keiko-contracts";
+import { largeDocumentDiagnostic } from "./diagnostics.js";
+function utf8ByteLength(text) {
+    return Buffer.byteLength(text, "utf8");
+}
+function isAborted(signal) {
+    return signal?.aborted === true;
+}
+// Evaluates the policy against the state that *would* result from accepting `window`. Returns a
+// stop reason when the window would breach a bound, so the over-limit window is never persisted.
+function limitStop(state, window, options, startedAt) {
+    const { policy } = options;
+    if (isAborted(options.signal))
+        return "cancelled";
+    if (options.now() - startedAt > policy.maxWallClockMs)
+        return "timeout";
+    if (state.extractedTextBytes + utf8ByteLength(window.text) > policy.maxExtractedTextBytes) {
+        return "extracted-text-limit";
+    }
+    if (state.extractedTextBytes + utf8ByteLength(window.text) >
+        policy.maxPersistedStorageGrowthBytes) {
+        return "extracted-text-limit";
+    }
+    if (window.objectCursor > policy.maxParserObjects)
+        return "object-limit";
+    if (state.pageCount + window.pages.length > policy.maxParserUnits)
+        return "unit-limit";
+    return undefined;
+}
+function stopDiagnostic(reason, documentId) {
+    switch (reason) {
+        case "extracted-text-limit":
+        case "object-limit":
+        case "unit-limit":
+            return largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.RESOURCE_POLICY_EXCEEDED, `extraction stopped at the ${reason} resource bound; document indexed with partial coverage`, documentId, "warning");
+        case "timeout":
+            return largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.RESOURCE_POLICY_EXCEEDED, "extraction stopped at the wall-clock deadline; document indexed with partial coverage", documentId, "warning");
+        case "cancelled":
+        case "completed":
+            return undefined;
+    }
+}
+function hasPartialCoverageWarning(diagnostics) {
+    return diagnostics.some((diagnostic) => diagnostic.severity !== "info" &&
+        (diagnostic.code === LARGE_DOCUMENT_DIAGNOSTIC_CODES.PARTIAL_COVERAGE ||
+            diagnostic.code === LARGE_DOCUMENT_DIAGNOSTIC_CODES.OCR_CAPABILITY_UNAVAILABLE ||
+            diagnostic.code === LARGE_DOCUMENT_DIAGNOSTIC_CODES.MULTIMODAL_CAPABILITY_UNAVAILABLE));
+}
+function coverageFor(stopReason, pageCount, diagnostics) {
+    if (pageCount === 0)
+        return "none";
+    if (hasPartialCoverageWarning(diagnostics))
+        return "partial";
+    if (stopReason === "completed")
+        return "complete";
+    if (stopReason === "cancelled")
+        return "partial";
+    // A resource/timeout stop means useful text was indexed but the document was truncated.
+    return "partial";
+}
+function appendDiagnostics(target, diagnostics) {
+    target.push(...diagnostics);
+}
+// Pulls windows one at a time, enforces the resource policy between windows, and flushes each
+// window through the sink. Returns a content-free summary. The driver never holds more than one
+// window's text in memory.
+export async function runProgressiveExtraction(extractor, source, options, sink) {
+    const startedAt = options.now();
+    const diagnostics = [];
+    const state = {
+        pageCount: options.resumeFromPage ?? 0,
+        windowCount: options.resumeWindowIndex ?? 0,
+        extractedTextBytes: options.resumeExtractedTextBytes ?? utf8ByteLength(""),
+        objectCursor: options.resumeObjectCursor ?? 0,
+        lastPageNumber: options.resumeFromPage ?? 0,
+    };
+    let stopReason = "completed";
+    for await (const window of extractor.extractWindows(source, options)) {
+        const stop = limitStop(state, window, options, startedAt);
+        if (stop !== undefined) {
+            stopReason = stop;
+            break;
+        }
+        appendDiagnostics(diagnostics, window.diagnostics);
+        state.pageCount += window.pages.length;
+        state.windowCount += 1;
+        state.extractedTextBytes += utf8ByteLength(window.text);
+        state.objectCursor = window.objectCursor;
+        state.lastPageNumber = window.lastPageNumber;
+        await sink.onWindow(window);
+        // Re-check cancellation between sink flushes so cancellation lands within one window.
+        if (isAborted(options.signal)) {
+            stopReason = "cancelled";
+            break;
+        }
+    }
+    const stopDiag = stopDiagnostic(stopReason, options.documentId);
+    if (stopDiag !== undefined)
+        diagnostics.push(stopDiag);
+    return {
+        pageCount: state.pageCount,
+        windowCount: state.windowCount,
+        extractedTextBytes: state.extractedTextBytes,
+        objectCursor: state.objectCursor,
+        lastPageNumber: state.lastPageNumber,
+        diagnostics,
+        stopReason,
+        coverage: coverageFor(stopReason, state.pageCount, diagnostics),
+    };
+}

package/dist/parsers/large-document/progressive-pdf.d.ts

@@ -0,0 +1,20 @@
+import type { ExtractionCapabilityStatus, ParserDependencyVersion } from "@oscharko-dev/keiko-contracts";
+import { type PdfDocumentLike } from "../pdf-parser.js";
+import type { OcrPageResult } from "../ocr/types.js";
+import type { ProgressiveExtractionSource, ProgressiveExtractor } from "./progressive-extraction.js";
+export type OcrPageFn = (input: {
+    readonly pageNumber: number;
+    readonly bytes: Uint8Array;
+}) => Promise<OcrPageResult>;
+export interface ProgressivePdfExtractorDeps {
+    readonly loadDocument?: (source: ProgressiveExtractionSource) => Promise<PdfDocumentLike>;
+    readonly ocr?: {
+        readonly status: ExtractionCapabilityStatus;
+        readonly ocrPage: OcrPageFn;
+    };
+}
+export declare function createProgressivePdfExtractor(deps?: ProgressivePdfExtractorDeps): ProgressiveExtractor;
+export declare const PROGRESSIVE_PDF_PARSER_ID = "progressive-pdf";
+export declare const PROGRESSIVE_PDF_PARSER_VERSION = "1";
+export declare const PROGRESSIVE_PDF_DEPENDENCY_VERSIONS: readonly ParserDependencyVersion[];
+//# sourceMappingURL=progressive-pdf.d.ts.map

package/dist/parsers/large-document/progressive-pdf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressive-pdf.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/progressive-pdf.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAEV,0BAA0B,EAC1B,uBAAuB,EAExB,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EAA2C,KAAK,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIrD,OAAO,KAAK,EAEV,2BAA2B,EAE3B,oBAAoB,EACrB,MAAM,6BAA6B,CAAC;AASrC,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;AAE7B,MAAM,WAAW,2BAA2B;IAE1C,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,2BAA2B,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;IAG1F,QAAQ,CAAC,GAAG,CAAC,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,0BAA0B,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAA;KAAE,CAAC;CAC7F;AA4LD,wBAAgB,6BAA6B,CAC3C,IAAI,GAAE,2BAAgC,GACrC,oBAAoB,CAYtB;AAED,eAAO,MAAM,yBAAyB,oBAAY,CAAC;AACnD,eAAO,MAAM,8BAA8B,MAAiB,CAAC;AAC7D,eAAO,MAAM,mCAAmC,oCAAsB,CAAC"}

package/dist/parsers/large-document/progressive-pdf.js

@@ -0,0 +1,145 @@
+// Progressive PDF text-layer extractor (Epic #1160, Issue #1286).
+//
+// Reuses the existing pdfjs loader and per-page streaming text reader from `pdf-parser.ts`; the
+// only behavior change is that pages are emitted in bounded windows instead of accumulated into
+// one result. pdfjs-dist requires the whole byte buffer to open a document (documented upstream
+// limitation), so the source's `loadFullBuffer` is used — but the extracted text, page records,
+// and (downstream) chunks are flushed per window, so the extraction working set does not grow
+// with document size and useful work begins after the first window.
+//
+// No-text-layer pages degrade gracefully: when an injected OCR capability is available, the
+// page's OCR text is indexed with page provenance; otherwise a content-free partial-coverage
+// quality warning is recorded and the page is skipped. Missing OCR never fails the job.
+import { LARGE_DOCUMENT_DIAGNOSTIC_CODES } from "@oscharko-dev/keiko-contracts";
+import { buildParserOptions } from "../registry.js";
+import { loadPdfDocumentFromSource, readPageText } from "../pdf-parser.js";
+import { largeDocumentDiagnostic } from "./diagnostics.js";
+import { WindowTextBuilder } from "./window-builder.js";
+const PARSER_ID = "progressive-pdf";
+const PARSER_VERSION = "1";
+const DEPENDENCY_VERSIONS = Object.freeze([
+    Object.freeze({ packageName: "pdfjs-dist", version: "6.0.227" }),
+]);
+function selectionInput(documentId) {
+    return {
+        documentId,
+        bytes: new Uint8Array(0),
+        extension: "pdf",
+        mediaType: "application/pdf",
+    };
+}
+function parserOptionsFromPolicy(options) {
+    const base = {
+        maxBytes: options.policy.maxRawFileBytes,
+        maxUnitsPerDocument: options.policy.maxParserUnits,
+        maxObjectsPerDocument: options.policy.maxParserObjects,
+        timeoutMs: options.policy.maxWallClockMs,
+        now: options.now,
+    };
+    return buildParserOptions(options.signal === undefined ? base : { ...base, signal: options.signal });
+}
+async function ocrTextForPage(deps, documentId, pageNumber) {
+    if (deps.ocr?.status !== "available") {
+        return {
+            diagnostic: largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.OCR_CAPABILITY_UNAVAILABLE, "page has no extractable text layer and no OCR capability is configured; indexed with partial coverage", documentId, "warning", pageNumber),
+        };
+    }
+    // A no-text-layer page needs a rasterized image for OCR. Keiko does not bundle a rasterizer, so
+    // production OCR remains an injected capability; the injected adapter receives the page number
+    // and returns text in tests. The empty byte view is the documented placeholder for the page
+    // image the rasterizer would supply.
+    const result = await deps.ocr.ocrPage({ pageNumber, bytes: new Uint8Array(0) });
+    if (result.ok && result.text.trim().length > 0) {
+        return { text: result.text, diagnostic: ocrIndexedDiagnostic(documentId, pageNumber) };
+    }
+    return {
+        diagnostic: largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.PARTIAL_COVERAGE, "OCR returned no text for a no-text-layer page; indexed with partial coverage", documentId, "warning", pageNumber),
+    };
+}
+function ocrIndexedDiagnostic(documentId, pageNumber) {
+    return largeDocumentDiagnostic(LARGE_DOCUMENT_DIAGNOSTIC_CODES.PARTIAL_COVERAGE, "page text recovered through the injected OCR capability with page provenance", documentId, "info", pageNumber);
+}
+// Reads a single page's text layer, falling back to the injected OCR capability for no-text-layer
+// pages. Never throws; capability gaps become content-free quality warnings.
+async function readOrOcrPage(doc, pageNumber, ctx, parserOptions, startedAt, emittedUnits, scannedObjects) {
+    const page = await doc.getPage(pageNumber);
+    const read = await readPageText(page, {
+        input: selectionInput(ctx.documentId),
+        options: parserOptions,
+        startedAt,
+        emittedUnits,
+        scannedObjects,
+    });
+    const diagnostics = [];
+    if (read.diagnostic !== undefined)
+        diagnostics.push(read.diagnostic);
+    if (read.text.length > 0) {
+        return { text: read.text, scannedObjects: read.scannedObjects, diagnostics };
+    }
+    const ocr = await ocrTextForPage(ctx, ctx.documentId, pageNumber);
+    diagnostics.push(ocr.diagnostic);
+    return ocr.text === undefined
+        ? { scannedObjects: read.scannedObjects, diagnostics }
+        : { text: ocr.text, scannedObjects: read.scannedObjects, diagnostics };
+}
+async function extractPdfWindow(doc, firstPage, lastPage, state, ctx, parserOptions, startedAt) {
+    const builder = new WindowTextBuilder(ctx.documentId, state.cursor, state.anyPageEmitted);
+    const diagnostics = [];
+    for (let pageNumber = firstPage; pageNumber <= lastPage; pageNumber += 1) {
+        const outcome = await readOrOcrPage(doc, pageNumber, ctx, parserOptions, startedAt, state.emittedUnits, state.scannedObjects);
+        state.scannedObjects = outcome.scannedObjects;
+        diagnostics.push(...outcome.diagnostics);
+        if (outcome.text !== undefined && outcome.text.length > 0) {
+            builder.addPage(pageNumber, outcome.text);
+            state.emittedUnits += 1;
+        }
+    }
+    state.cursor = builder.nextCursor;
+    state.anyPageEmitted = builder.hasEmittedAnyPage;
+    const window = {
+        windowIndex: state.windowIndex,
+        pages: builder.snapshotPages(),
+        units: builder.snapshotUnits(),
+        text: builder.text(),
+        characterStart: builder.characterStart,
+        objectCursor: state.scannedObjects,
+        lastPageNumber: lastPage,
+        diagnostics,
+    };
+    state.windowIndex += 1;
+    return window;
+}
+async function* pdfExtractWindows(loadDocument, ctx, source, options) {
+    const parserOptions = parserOptionsFromPolicy(options);
+    const doc = await loadDocument(source);
+    const resumeFromPage = options.resumeFromPage ?? 0;
+    const windowPages = Math.max(1, options.policy.extractionWindowPages);
+    const startedAt = options.now();
+    const state = {
+        cursor: options.resumeCharacterStart ?? 0,
+        anyPageEmitted: resumeFromPage > 0,
+        scannedObjects: options.resumeObjectCursor ?? 0,
+        emittedUnits: 0,
+        windowIndex: options.resumeWindowIndex ?? 0,
+    };
+    for (let firstPage = resumeFromPage + 1; firstPage <= doc.numPages; firstPage += windowPages) {
+        if (options.signal?.aborted === true)
+            return;
+        const lastPage = Math.min(firstPage + windowPages - 1, doc.numPages);
+        yield extractPdfWindow(doc, firstPage, lastPage, state, ctx, parserOptions, startedAt);
+    }
+}
+export function createProgressivePdfExtractor(deps = {}) {
+    const loadDocument = deps.loadDocument ?? loadPdfDocumentFromSource;
+    return {
+        strategyId: "progressive-pdf",
+        parserVersion: PARSER_VERSION,
+        dependencyVersions: DEPENDENCY_VERSIONS,
+        matches: (input) => input.extension.toLowerCase() === "pdf" ||
+            input.mediaType.toLowerCase() === "application/pdf",
+        extractWindows: (source, options) => pdfExtractWindows(loadDocument, { ...deps, documentId: options.documentId }, source, options),
+    };
+}
+export const PROGRESSIVE_PDF_PARSER_ID = PARSER_ID;
+export const PROGRESSIVE_PDF_PARSER_VERSION = PARSER_VERSION;
+export const PROGRESSIVE_PDF_DEPENDENCY_VERSIONS = DEPENDENCY_VERSIONS;

package/dist/parsers/large-document/synthetic-source.d.ts

@@ -0,0 +1,9 @@
+import type { ProgressiveExtractionSource, ProgressiveExtractor } from "./progressive-extraction.js";
+export interface SyntheticStreamingConfig {
+    readonly totalPages: number;
+    readonly pageChars: number;
+    readonly pagesPerWindow: number;
+}
+export declare function syntheticStreamingSource(config: SyntheticStreamingConfig): ProgressiveExtractionSource;
+export declare function syntheticProgressiveExtractor(config: SyntheticStreamingConfig, parserVersion?: string): ProgressiveExtractor;
+//# sourceMappingURL=synthetic-source.d.ts.map

package/dist/parsers/large-document/synthetic-source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"synthetic-source.d.ts","sourceRoot":"","sources":["../../../src/parsers/large-document/synthetic-source.ts"],"names":[],"mappings":"AAeA,OAAO,KAAK,EAEV,2BAA2B,EAE3B,oBAAoB,EACrB,MAAM,6BAA6B,CAAC;AAGrC,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAyBD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,wBAAwB,GAC/B,2BAA2B,CAc7B;AAgFD,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,wBAAwB,EAChC,aAAa,SAAgB,GAC5B,oBAAoB,CAOtB"}