@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/discovery/extract.js

@@ -0,0 +1,906 @@
+// Per-file extraction (Epic #189, Issue #194). Given a discovered file, a parser registry,
+// and an open KnowledgeStore, this module:
+//
+//   1. Resolves the file's realPath through the WorkspaceFs port and re-asserts the
+//      realpath-containment gate (defence in depth — the walker already filtered, but a
+//      consumer calling extractDocument() directly must not bypass the boundary).
+//   2. Reads bytes via WorkspaceFs.readFileBytes (the boundary-checked byte-read path).
+//   3. Computes the content hash (SHA-256 hex) over the raw bytes.
+//   4. Detects the incremental fast-path: if a documents row with the same id already has
+//      this content_hash AND status="extracted"/"unsupported", we skip the parse entirely
+//      and leave last_extracted_at untouched.
+//   5. Resolves a parser through the registry; rejects an oversized file BEFORE we hand it
+//      to the parser (the OVERSIZED_FILE diagnostic is the same code parsers emit).
+//   6. Inside a single transaction: REPLACEs the documents row, deletes prior dependent
+//      rows, then inserts the new pages/sections/parsed_units/diagnostics.
+import { createHash } from "node:crypto";
+import { DEFAULT_EXTRACTION_CAPABILITY_AVAILABILITY, DEFAULT_LARGE_DOCUMENT_RESOURCE_POLICY, isSafeScopePath, } from "@oscharko-dev/keiko-contracts";
+import { isDenied } from "@oscharko-dev/keiko-workspace";
+import { buildParserOptions, createProgressivePdfExtractor, classifyLargeDocument, isLegacyBinaryOfficeFormat, legacyFormatDiagnostic, unsupportedParser, usesProgressivePath, } from "../parsers/index.js";
+import { DEFAULT_CHUNKING_STRATEGY_KEY } from "../chunking/types.js";
+import { extractDocumentProgressive, selectProgressiveExtractor, } from "./extract-progressive.js";
+import { redactDiagnosticMessage } from "../privacy/diagnostic-redactor.js";
+import { basenameOf, extensionOf, mediaTypeFor } from "./media-type.js";
+import { compileGlobList, matchesAny } from "./glob.js";
+import { deleteDependentRows, insertDiagnosticRow, insertDocumentRow, insertDocumentTextRow, insertPageRow, insertParsedUnitRow, insertSectionRow, readExistingDocumentRow, } from "./persist.js";
+import { documentIdFor, } from "./types.js";
+// ─── Path helpers (re-derived to keep extract.ts self-contained for the realpath gate) ──
+// On Windows, WorkspaceFs.realPath() may return backslash-separated paths
+// (e.g. C:\Users\workspace\file). Normalise both sides to forward slashes so
+// containment checks work cross-platform.
+function normaliseSep(p) {
+    return p.replace(/\\/g, "/");
+}
+function isContained(absoluteRoot, absolutePath) {
+    const normRoot = normaliseSep(absoluteRoot);
+    const normPath = normaliseSep(absolutePath);
+    if (normPath === normRoot)
+        return true;
+    const prefix = normRoot.endsWith("/") ? normRoot : `${normRoot}/`;
+    return normPath.startsWith(prefix);
+}
+function joinAbs(root, rel) {
+    if (root.endsWith("/"))
+        return `${root}${rel}`;
+    return `${root}/${rel}`;
+}
+function toPosixRelative(absoluteRoot, absolutePath) {
+    const normRoot = normaliseSep(absoluteRoot);
+    const normPath = normaliseSep(absolutePath);
+    if (normPath === normRoot)
+        return "";
+    const prefix = normRoot.endsWith("/") ? normRoot : `${normRoot}/`;
+    return normPath.startsWith(prefix) ? normPath.slice(prefix.length) : normPath;
+}
+function scopeRoot(source) {
+    const { scope } = source;
+    if (scope.kind === "folder")
+        return scope.rootPath;
+    if (scope.kind === "repository")
+        return scope.repositoryRoot;
+    return scope.rootPath;
+}
+function hashBytes(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+function safeDisplay(relativePath) {
+    const base = basenameOf(relativePath);
+    return base.length === 0 ? relativePath : base;
+}
+function safeRelativePath(relativePath) {
+    const normalised = normaliseSep(relativePath);
+    if (normalised.startsWith("/")) {
+        return {
+            code: "INVALID_SCOPE",
+            message: "file path failed the selected-scope policy",
+            relativePath,
+        };
+    }
+    if (!isSafeScopePath(normalised)) {
+        return {
+            code: "INVALID_SCOPE",
+            message: "file path failed the selected-scope policy",
+            relativePath,
+        };
+    }
+    return normalised;
+}
+function redactionPrefixFor(source) {
+    return scopeRoot(source);
+}
+function redactMessage(message, source) {
+    return redactDiagnosticMessage(message, redactionPrefixFor(source));
+}
+function redactDiagnostic(diagnostic, source) {
+    return {
+        ...diagnostic,
+        message: redactMessage(diagnostic.message, source),
+    };
+}
+function redactDiagnostics(diagnostics, source) {
+    return diagnostics.map((diagnostic) => redactDiagnostic(diagnostic, source));
+}
+function redactParserResult(parserResult, source) {
+    return {
+        ...parserResult,
+        diagnostics: redactDiagnostics(parserResult.diagnostics, source),
+    };
+}
+// ─── Failure / unsupported helpers ───────────────────────────────────────────
+function persistFailureRow(deps, params, documentId, document, diagnostic, now) {
+    const db = deps.store._internal.db;
+    db.exec("BEGIN");
+    try {
+        insertDocumentRow(db, {
+            id: documentId,
+            capsuleId: params.capsuleId,
+            sourceId: String(params.source.id),
+            documentPath: document.documentPath,
+            sizeBytes: document.sizeBytes,
+            mediaType: document.mediaType,
+            contentHash: document.contentHash,
+            parserId: document.parser.parserId,
+            parserVersion: document.parser.parserVersion,
+            lastExtractedAt: document.lastExtractedAt,
+            status: document.status,
+            safeDisplayName: document.safeDisplayName,
+        });
+        deleteDependentRows(db, params.capsuleId, documentId);
+        insertDiagnosticRow(db, {
+            id: `${String(documentId)}#d0`,
+            capsuleId: params.capsuleId,
+            diagnostic,
+            createdAt: now(),
+        });
+        db.exec("COMMIT");
+    }
+    catch (cause) {
+        db.exec("ROLLBACK");
+        throw cause;
+    }
+}
+// GRD-010: decide whether to persist a cascade-deleting failure row. A TRANSIENT IO failure
+// (READ_FAILED / STAT_FAILED) on an incremental refresh must NOT destroy a previously-good index
+// — persisting overwrites the document row and CASCADE-deletes its chunks+vectors. When a prior
+// NON-failed row exists, skip persistence so a momentary lock / NFS hiccup / permission flap
+// preserves retrievable content (the orchestrator then reports a non-destructive skip). Permanent
+// failures (MALFORMED_INPUT, PARSER_FAILED, OVERSIZED_FILE, …) and first-time failures still persist.
+function shouldPersistFailureRow(deps, params, documentId, error, optionPersist) {
+    if (!optionPersist)
+        return false;
+    const isTransient = error.code === "READ_FAILED" || error.code === "STAT_FAILED";
+    if (!isTransient)
+        return true;
+    const existing = readExistingDocumentRow(deps.store._internal.db, params.capsuleId, documentId);
+    return existing === undefined || existing.status === "failed";
+}
+function buildFailureResult(deps, params, documentId, error, options = { persist: true }) {
+    const now = deps.store._internal.now;
+    const redactedMessage = redactMessage(error.message, params.source);
+    const diagnostic = {
+        severity: "error",
+        code: error.code,
+        message: redactedMessage,
+        documentId,
+    };
+    const document = {
+        id: documentId,
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        documentPath: params.file.relativePath,
+        sizeBytes: params.file.sizeBytes,
+        mediaType: mediaTypeFor(extensionOf(params.file.relativePath)),
+        contentHash: "",
+        parser: { parserId: "none", parserVersion: "0" },
+        lastExtractedAt: now(),
+        status: "failed",
+        safeDisplayName: safeDisplay(params.file.relativePath),
+    };
+    if (shouldPersistFailureRow(deps, params, documentId, error, options.persist)) {
+        persistFailureRow(deps, params, documentId, document, diagnostic, now);
+    }
+    const outcome = {
+        kind: "failed",
+        document,
+        error: { ...error, message: redactedMessage, relativePath: params.file.relativePath },
+    };
+    return {
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+        outcome,
+        diagnostics: [diagnostic],
+    };
+}
+// ─── Persist helpers (run inside the per-file transaction) ───────────────────
+function persistDependentRows(deps, capsuleId, documentId, parserResult, now) {
+    const db = deps.store._internal.db;
+    deleteDependentRows(db, capsuleId, documentId);
+    if (parserResult.normalizedText !== undefined) {
+        insertDocumentTextRow(db, deps.store._internal.contentCipher, capsuleId, documentId, parserResult.normalizedText);
+    }
+    for (const page of parserResult.pages)
+        insertPageRow(db, capsuleId, page);
+    for (const section of parserResult.sections) {
+        insertSectionRow(db, deps.store._internal.contentCipher, capsuleId, section);
+    }
+    parserResult.units.forEach((unit, index) => {
+        insertParsedUnitRow(db, deps.store._internal.contentCipher, capsuleId, `${String(documentId)}#u${String(index)}`, unit);
+    });
+    parserResult.diagnostics.forEach((diagnostic, index) => {
+        insertDiagnosticRow(db, {
+            id: `${String(documentId)}#d${String(index)}`,
+            capsuleId,
+            diagnostic,
+            createdAt: now(),
+        });
+    });
+}
+function buildDocumentRecord(input) {
+    return {
+        id: input.documentId,
+        capsuleId: input.params.capsuleId,
+        sourceId: input.params.source.id,
+        documentPath: input.params.file.relativePath,
+        sizeBytes: input.params.file.sizeBytes,
+        mediaType: input.mediaType,
+        contentHash: input.contentHash,
+        parser: input.parserResult.parser,
+        lastExtractedAt: input.parserResult.extractedAt,
+        status: input.status,
+        safeDisplayName: safeDisplay(input.params.file.relativePath),
+    };
+}
+function persistDocumentAndDependents(deps, params, documentId, document, parserResult, now) {
+    const db = deps.store._internal.db;
+    db.exec("BEGIN");
+    try {
+        insertDocumentRow(db, {
+            id: documentId,
+            capsuleId: params.capsuleId,
+            sourceId: String(params.source.id),
+            documentPath: document.documentPath,
+            sizeBytes: document.sizeBytes,
+            mediaType: document.mediaType,
+            contentHash: document.contentHash,
+            parserId: document.parser.parserId,
+            parserVersion: document.parser.parserVersion,
+            lastExtractedAt: document.lastExtractedAt,
+            status: document.status,
+            safeDisplayName: document.safeDisplayName,
+        });
+        persistDependentRows(deps, params.capsuleId, documentId, parserResult, now);
+        db.exec("COMMIT");
+    }
+    catch (cause) {
+        db.exec("ROLLBACK");
+        throw cause;
+    }
+}
+const HIDDEN_OR_GENERATED_DIRS = new Set([
+    ".git",
+    ".hg",
+    ".svn",
+    ".next",
+    ".turbo",
+    "node_modules",
+    "dist",
+    "build",
+    "coverage",
+    "out",
+]);
+function deriveScopePolicy(scope) {
+    if (scope.kind === "folder") {
+        if (!isSafeScopePath(scope.rootPath)) {
+            return { code: "INVALID_SCOPE", message: "scope.rootPath failed the safe-path gate" };
+        }
+        return {
+            rootPath: scope.rootPath,
+            recursive: scope.recursive,
+            includeGlobs: compileGlobList(scope.includeGlobs),
+            excludeGlobs: compileGlobList(scope.excludeGlobs),
+        };
+    }
+    if (scope.kind === "repository") {
+        if (!isSafeScopePath(scope.repositoryRoot)) {
+            return { code: "INVALID_SCOPE", message: "scope.repositoryRoot failed the safe-path gate" };
+        }
+        return {
+            rootPath: scope.repositoryRoot,
+            recursive: true,
+            includeGlobs: compileGlobList(scope.includeGlobs),
+            excludeGlobs: compileGlobList(scope.excludeGlobs),
+        };
+    }
+    if (!isSafeScopePath(scope.rootPath)) {
+        return { code: "INVALID_SCOPE", message: "scope.rootPath failed the safe-path gate" };
+    }
+    const explicitFiles = new Set();
+    for (const entry of scope.files) {
+        const safeEntry = safeRelativePath(entry);
+        if (typeof safeEntry !== "string") {
+            return {
+                code: "INVALID_SCOPE",
+                message: `scope.files entry failed the safe-path gate: ${entry}`,
+            };
+        }
+        explicitFiles.add(safeEntry);
+    }
+    return {
+        rootPath: scope.rootPath,
+        recursive: false,
+        includeGlobs: [],
+        excludeGlobs: [],
+        explicitFiles,
+    };
+}
+function matchesSourceGlobs(policy, relativePath) {
+    if (matchesAny(policy.excludeGlobs, relativePath, false))
+        return false;
+    return matchesAny(policy.includeGlobs, relativePath, true);
+}
+function hasHiddenOrGeneratedParent(relativePath) {
+    const segments = relativePath.split("/").filter((segment) => segment.length > 0);
+    for (const segment of segments.slice(0, -1)) {
+        if (segment.startsWith(".") || HIDDEN_OR_GENERATED_DIRS.has(segment))
+            return true;
+    }
+    return false;
+}
+function isSelectedByScope(policy, relativePath) {
+    if (isDenied(relativePath))
+        return false;
+    if (policy.explicitFiles !== undefined)
+        return policy.explicitFiles.has(relativePath);
+    if (!policy.recursive && relativePath.includes("/"))
+        return false;
+    if (hasHiddenOrGeneratedParent(relativePath))
+        return false;
+    return matchesSourceGlobs(policy, relativePath);
+}
+function targetError(error, persistFailure) {
+    return { error, persistFailure };
+}
+function selectedRelativePath(policy, rawRelativePath) {
+    const relativePath = safeRelativePath(rawRelativePath);
+    if (typeof relativePath !== "string") {
+        return targetError(relativePath, false);
+    }
+    if (!isSelectedByScope(policy, relativePath)) {
+        return targetError({
+            code: "INVALID_SCOPE",
+            message: "file is outside the selected source scope",
+            relativePath,
+        }, false);
+    }
+    return relativePath;
+}
+function resolveRealPathTarget(deps, path, relativePath, message) {
+    try {
+        return deps.fs.realPath(path);
+    }
+    catch {
+        return targetError({ code: "READ_FAILED", message, relativePath }, true);
+    }
+}
+function containedRealFileTarget(realRoot, real, relativePath) {
+    if (!isContained(realRoot, real)) {
+        return targetError({
+            code: "PATH_ESCAPE",
+            message: `realpath escapes scope root: ${relativePath}`,
+            relativePath,
+        }, true);
+    }
+    const realRelativePath = toPosixRelative(realRoot, real);
+    if (isDenied(realRelativePath)) {
+        return targetError({
+            code: "READ_FAILED",
+            message: "resolved file is denied by workspace policy",
+            relativePath,
+        }, true);
+    }
+    return undefined;
+}
+function resolveTargetPath(deps, params) {
+    const policy = deriveScopePolicy(params.source.scope);
+    if ("code" in policy) {
+        return targetError(policy, false);
+    }
+    const relativePath = selectedRelativePath(policy, params.file.relativePath);
+    if (typeof relativePath !== "string") {
+        return relativePath;
+    }
+    const root = policy.rootPath;
+    const absolute = joinAbs(root, relativePath);
+    const realRoot = resolveRealPathTarget(deps, root, relativePath, "realPath failed for selected source root");
+    if (typeof realRoot !== "string")
+        return realRoot;
+    const real = resolveRealPathTarget(deps, absolute, relativePath, "realPath failed for selected file");
+    if (typeof real !== "string")
+        return real;
+    const containmentError = containedRealFileTarget(realRoot, real, relativePath);
+    if (containmentError !== undefined)
+        return containmentError;
+    // Normalise to forward slashes so subsequent IO calls (readFileBytes, stat) receive
+    // a consistent path even when realPath returned a Windows backslash path.
+    return { absolutePath: normaliseSep(real), requestedAbsolutePath: absolute, relativePath };
+}
+function validateRequestedTarget(deps, params, target) {
+    try {
+        deps.fs.stat(target.requestedAbsolutePath);
+        return undefined;
+    }
+    catch {
+        return {
+            code: "STAT_FAILED",
+            message: "stat failed for selected file",
+            relativePath: params.file.relativePath,
+        };
+    }
+}
+function validateResolvedTarget(deps, params, target) {
+    try {
+        const realStat = deps.fs.stat(target.absolutePath);
+        if (!realStat.isFile) {
+            return {
+                code: "READ_FAILED",
+                message: "selected path is not a file",
+                relativePath: params.file.relativePath,
+            };
+        }
+        if (realStat.hardLinkCount === undefined || realStat.hardLinkCount <= 1)
+            return undefined;
+        return {
+            code: "READ_FAILED",
+            message: "selected file is not eligible for extraction",
+            relativePath: params.file.relativePath,
+        };
+    }
+    catch {
+        return {
+            code: "STAT_FAILED",
+            message: "stat failed for selected file",
+            relativePath: params.file.relativePath,
+        };
+    }
+}
+async function readBytes(deps, params, target, maxBytes) {
+    const reader = deps.fs.readFileBytes;
+    if (reader === undefined) {
+        return {
+            code: "READ_FAILED",
+            message: "WorkspaceFs.readFileBytes is unavailable",
+            relativePath: params.file.relativePath,
+        };
+    }
+    const requestedError = validateRequestedTarget(deps, params, target);
+    if (requestedError !== undefined)
+        return requestedError;
+    const resolvedError = validateResolvedTarget(deps, params, target);
+    if (resolvedError !== undefined)
+        return resolvedError;
+    try {
+        return await reader(target.absolutePath, maxBytes);
+    }
+    catch {
+        return {
+            code: "READ_FAILED",
+            message: "readFileBytes failed for selected file",
+            relativePath: params.file.relativePath,
+        };
+    }
+}
+// ─── Incremental fast-path ───────────────────────────────────────────────────
+function readUnchangedFastPath(deps, params, documentId, contentHash) {
+    const existing = readExistingDocumentRow(deps.store._internal.db, params.capsuleId, documentId);
+    if (existing === undefined)
+        return undefined;
+    if (existing.content_hash !== contentHash)
+        return undefined;
+    // Skip only terminal-good states. A `pending` row is an interrupted progressive extraction
+    // (Issue #1286) and `failed` should be retried; both must re-extract rather than be skipped.
+    if (existing.status !== "extracted" && existing.status !== "unsupported")
+        return undefined;
+    const document = {
+        id: documentId,
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        documentPath: existing.document_path,
+        sizeBytes: existing.size_bytes,
+        mediaType: existing.media_type,
+        contentHash: existing.content_hash,
+        parser: {
+            parserId: existing.parser_id,
+            parserVersion: existing.parser_version,
+        },
+        lastExtractedAt: existing.last_extracted_at,
+        status: existing.status,
+        safeDisplayName: existing.safe_display_name,
+    };
+    return {
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+        outcome: { kind: "skipped", document, reason: "unchanged" },
+        diagnostics: [],
+    };
+}
+// ─── Large-document routing (Epic #1160, Issue #1286) ────────────────────────
+let defaultProgressiveExtractorsCache;
+function defaultProgressiveExtractors() {
+    defaultProgressiveExtractorsCache ??= [createProgressivePdfExtractor()];
+    return defaultProgressiveExtractorsCache;
+}
+function largeDocumentContextFor(deps, resolved, options) {
+    return {
+        policy: deps.largeDocumentPolicy ?? DEFAULT_LARGE_DOCUMENT_RESOURCE_POLICY,
+        capabilities: deps.extractionCapabilities ?? DEFAULT_EXTRACTION_CAPABILITY_AVAILABILITY,
+        extractors: deps.progressiveExtractors ?? defaultProgressiveExtractors(),
+        jobId: deps.largeDocumentJobId ?? "extract",
+        chunkingStrategyVersion: deps.chunkingStrategyVersion ?? DEFAULT_CHUNKING_STRATEGY_KEY,
+        absolutePath: resolved.absolutePath,
+        relativePath: resolved.relativePath,
+        ...(options.signal === undefined ? {} : { signal: options.signal }),
+    };
+}
+const PROGRESSIVE_HASH_CHUNK_BYTES = 4 * 1024 * 1024;
+async function readRange(deps, params, target, startByte, length) {
+    const reader = deps.fs.readFileRange;
+    if (reader === undefined) {
+        return {
+            code: "READ_FAILED",
+            message: "WorkspaceFs.readFileRange is unavailable",
+            relativePath: params.file.relativePath,
+        };
+    }
+    const requestedError = validateRequestedTarget(deps, params, target);
+    if (requestedError !== undefined)
+        return requestedError;
+    const resolvedError = validateResolvedTarget(deps, params, target);
+    if (resolvedError !== undefined)
+        return resolvedError;
+    try {
+        return await reader(target.absolutePath, startByte, length);
+    }
+    catch {
+        return {
+            code: "READ_FAILED",
+            message: "readFileRange failed for selected file",
+            relativePath: params.file.relativePath,
+        };
+    }
+}
+function progressiveRangeSource(deps, params, target) {
+    if (deps.fs.readFileRange === undefined) {
+        return {
+            code: "READ_FAILED",
+            message: "WorkspaceFs.readFileRange is unavailable",
+            relativePath: params.file.relativePath,
+        };
+    }
+    return {
+        totalBytes: params.file.sizeBytes,
+        readWindow: async (startByte, length) => {
+            const bytes = await readRange(deps, params, target, startByte, length);
+            if (bytes instanceof Uint8Array)
+                return bytes;
+            throw new Error(bytes.message);
+        },
+    };
+}
+async function hashProgressiveSource(source) {
+    if (source.readWindow === undefined) {
+        throw new Error("progressive source does not support bounded range reads");
+    }
+    const hash = createHash("sha256");
+    for (let offset = 0; offset < source.totalBytes; offset += PROGRESSIVE_HASH_CHUNK_BYTES) {
+        const bytes = await source.readWindow(offset, Math.min(PROGRESSIVE_HASH_CHUNK_BYTES, source.totalBytes - offset));
+        if (bytes.byteLength === 0)
+            break;
+        hash.update(bytes);
+    }
+    return hash.digest("hex");
+}
+// Legacy binary office formats (.doc/.ppt/.xls) get the existing unsupported path plus a stable
+// CONVERTER_UNAVAILABLE diagnostic with actionable guidance, leaving the job stable.
+function appendLegacyDiagnostic(deps, params, documentId, extension, result) {
+    const diagnostic = legacyFormatDiagnostic(extension, documentId);
+    if (diagnostic === undefined)
+        return result;
+    insertDiagnosticRow(deps.store._internal.db, {
+        id: `${String(documentId)}#legacy`,
+        capsuleId: params.capsuleId,
+        diagnostic,
+        createdAt: deps.store._internal.now(),
+    });
+    return { ...result, diagnostics: [...result.diagnostics, diagnostic] };
+}
+// ─── Top-level entry point ───────────────────────────────────────────────────
+function selectionInput(documentId, relativePath, bytes) {
+    const extension = extensionOf(relativePath);
+    return {
+        documentId,
+        bytes,
+        extension,
+        mediaType: mediaTypeFor(extension),
+    };
+}
+function isUnsupportedResult(result) {
+    return (result.parser.parserId === "unsupported" ||
+        (result.units.length > 0 && result.units.every((unit) => unit.kind === "unsupported-media")));
+}
+const FAILED_PARSER_DIAGNOSTIC_CODES = new Set([
+    "OVERSIZED_FILE",
+    "PARSER_TIMEOUT",
+    "PARSER_CANCELLED",
+    "MALFORMED_INPUT",
+    "OBJECT_LIMIT_REACHED",
+]);
+function firstParserFailureDiagnostic(result) {
+    return result.diagnostics.find((diagnostic) => diagnostic.severity === "error" || FAILED_PARSER_DIAGNOSTIC_CODES.has(diagnostic.code));
+}
+function statusForResult(result) {
+    if (isUnsupportedResult(result))
+        return "unsupported";
+    if (firstParserFailureDiagnostic(result) !== undefined)
+        return "failed";
+    return "extracted";
+}
+function discoveryErrorCodeForParserDiagnostic(diagnostic) {
+    if (diagnostic.code === "OVERSIZED_FILE")
+        return "OVERSIZED_FILE";
+    if (diagnostic.code === "PARSER_CANCELLED")
+        return "CANCELLED";
+    if (diagnostic.code === "MALFORMED_INPUT")
+        return "MALFORMED_INPUT";
+    if (diagnostic.code === "PARSER_TIMEOUT")
+        return "PARSER_TIMEOUT";
+    return "PARSER_FAILED";
+}
+function parserFailureOutcome(document, diagnostic, relativePath) {
+    return {
+        kind: "failed",
+        document,
+        error: {
+            code: discoveryErrorCodeForParserDiagnostic(diagnostic),
+            message: diagnostic.message,
+            relativePath,
+        },
+    };
+}
+const SOURCE_TEXT_PARSER_IDS = new Set(["text", "json", "csv", "html"]);
+function decodeUtf8ForStorage(bytes) {
+    const raw = new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+    return raw.length > 0 && raw.charCodeAt(0) === 0xfeff ? raw.slice(1) : raw;
+}
+function normalizedTextForPersistence(parserResult, bytes) {
+    if (parserResult.normalizedText !== undefined) {
+        return parserResult.normalizedText;
+    }
+    if (!SOURCE_TEXT_PARSER_IDS.has(parserResult.parser.parserId)) {
+        return undefined;
+    }
+    if (parserResult.units.length === 0) {
+        return undefined;
+    }
+    return decodeUtf8ForStorage(bytes);
+}
+function withPersistedNormalizedText(parserResult, bytes) {
+    const normalizedText = normalizedTextForPersistence(parserResult, bytes);
+    return normalizedText === undefined ? parserResult : { ...parserResult, normalizedText };
+}
+function hasAsyncParse(adapter) {
+    return typeof adapter.parseAsync === "function";
+}
+async function runParser(deps, documentId, params, bytes, options) {
+    const input = selectionInput(documentId, params.file.relativePath, bytes);
+    const resolution = deps.parserRegistry.resolve(input);
+    const adapter = resolution.kind === "matched" ? resolution.adapter : unsupportedParser;
+    if (hasAsyncParse(adapter)) {
+        return adapter.parseAsync(input, options);
+    }
+    return adapter.parse(input, options);
+}
+async function runParserForPersistence(deps, documentId, params, bytes, options) {
+    const result = await runParser(deps, documentId, params, bytes, options);
+    return withPersistedNormalizedText(result, bytes);
+}
+function persistExtractedDocument(deps, params, documentId, document, parserResult) {
+    persistDocumentAndDependents(deps, params, documentId, document, parserResult, deps.store._internal.now);
+}
+async function readBoundedDocumentBytes(deps, params, documentId, target, options) {
+    const bytes = await readBytes(deps, params, target, options.maxBytes + 1);
+    if (!(bytes instanceof Uint8Array)) {
+        return buildFailureResult(deps, params, documentId, bytes);
+    }
+    if (bytes.byteLength > options.maxBytes) {
+        return buildOversizedFailure(deps, params, documentId, options, bytes.byteLength);
+    }
+    return bytes;
+}
+function parserExtractionResult(params, document, parserResult, status) {
+    const failureDiagnostic = firstParserFailureDiagnostic(parserResult);
+    return {
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+        outcome: status === "failed" && failureDiagnostic !== undefined
+            ? parserFailureOutcome(document, failureDiagnostic, params.file.relativePath)
+            : { kind: "persisted", document },
+        diagnostics: parserResult.diagnostics,
+    };
+}
+function paramsWithRelativePath(params, relativePath) {
+    if (params.file.relativePath === relativePath)
+        return params;
+    return { ...params, file: { ...params.file, relativePath } };
+}
+function extractionDocumentId(params) {
+    return documentIdFor({
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+    });
+}
+function targetResolutionFailure(deps, params, resolved) {
+    const failureParams = resolved.error.relativePath === undefined
+        ? params
+        : paramsWithRelativePath(params, resolved.error.relativePath);
+    return buildFailureResult(deps, failureParams, extractionDocumentId(failureParams), resolved.error, {
+        persist: resolved.persistFailure,
+    });
+}
+async function parseAndPersistDocument(deps, params, documentId, bytes, contentHash, options) {
+    let parserResult;
+    try {
+        parserResult = await runParserForPersistence(deps, documentId, params, bytes, options);
+    }
+    catch {
+        return buildFailureResult(deps, params, documentId, {
+            code: "PARSER_FAILED",
+            message: "parser adapter failed while extracting document",
+            relativePath: params.file.relativePath,
+        });
+    }
+    const redactedParserResult = redactParserResult(parserResult, params.source);
+    const status = statusForResult(redactedParserResult);
+    const document = buildDocumentRecord({
+        documentId,
+        params,
+        mediaType: mediaTypeFor(extensionOf(params.file.relativePath)),
+        contentHash,
+        parserResult: redactedParserResult,
+        status,
+    });
+    persistExtractedDocument(deps, params, documentId, document, redactedParserResult);
+    return parserExtractionResult(params, document, redactedParserResult, status);
+}
+async function progressiveExtractionResult(deps, params, resolved, documentId, options, extension, mediaType) {
+    const context = largeDocumentContextFor(deps, resolved, options);
+    const preflight = classifyLargeDocument({ extension, mediaType, sizeBytes: params.file.sizeBytes }, context.policy);
+    if (preflight.decision === "reject-oversized") {
+        return buildOversizedFailure(deps, params, documentId, {
+            ...options,
+            maxBytes: context.policy.maxRawFileBytes,
+        });
+    }
+    const extractor = params.file.sizeBytes >= context.policy.largeFileThresholdBytes
+        ? selectProgressiveExtractor(context, extension, mediaType)
+        : undefined;
+    if (!usesProgressivePath(preflight) && extractor === undefined)
+        return undefined;
+    if (extractor === undefined)
+        return undefined;
+    const source = progressiveRangeSource(deps, params, resolved);
+    if (!("totalBytes" in source))
+        return buildFailureResult(deps, params, documentId, source);
+    let contentHash;
+    try {
+        contentHash = await hashProgressiveSource(source);
+    }
+    catch {
+        return buildFailureResult(deps, params, documentId, {
+            code: "READ_FAILED",
+            message: "readFileRange failed for selected file",
+            relativePath: params.file.relativePath,
+        });
+    }
+    const fast = readUnchangedFastPath(deps, params, documentId, contentHash);
+    return (fast ??
+        (await extractDocumentProgressive(deps, params, context, source, contentHash, extractor)));
+}
+async function standardExtractionResult(deps, params, resolved, documentId, options, extension) {
+    if (params.file.sizeBytes > options.maxBytes) {
+        return buildOversizedFailure(deps, params, documentId, options);
+    }
+    const bytes = await readBoundedDocumentBytes(deps, params, documentId, resolved, options);
+    if (!(bytes instanceof Uint8Array))
+        return bytes;
+    const contentHash = hashBytes(bytes);
+    const fast = readUnchangedFastPath(deps, params, documentId, contentHash);
+    if (fast !== undefined)
+        return fast;
+    const result = await parseAndPersistDocument(deps, params, documentId, bytes, contentHash, options);
+    return isLegacyBinaryOfficeFormat(extension)
+        ? appendLegacyDiagnostic(deps, params, documentId, extension, result)
+        : result;
+}
+export async function extractDocument(deps, params) {
+    const resolved = resolveTargetPath(deps, params);
+    if ("error" in resolved) {
+        return targetResolutionFailure(deps, params, resolved);
+    }
+    const canonicalParams = paramsWithRelativePath(params, resolved.relativePath);
+    const documentId = extractionDocumentId(canonicalParams);
+    const options = canonicalParams.parserOptions ?? buildParserOptions();
+    const extension = extensionOf(canonicalParams.file.relativePath);
+    const mediaType = mediaTypeFor(extension);
+    const progressive = await progressiveExtractionResult(deps, canonicalParams, resolved, documentId, options, extension, mediaType);
+    if (progressive !== undefined)
+        return progressive;
+    return standardExtractionResult(deps, canonicalParams, resolved, documentId, options, extension);
+}
+export function recordExtractionFailure(deps, params) {
+    const documentId = documentIdFor({
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+    });
+    return buildFailureResult(deps, params, documentId, params.error);
+}
+function oversizedDocumentRecord(params, documentId, lastExtractedAt, observedSizeBytes) {
+    const sizeBytes = observedSizeBytes === undefined
+        ? params.file.sizeBytes
+        : Math.max(params.file.sizeBytes, observedSizeBytes);
+    return {
+        id: documentId,
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        documentPath: params.file.relativePath,
+        sizeBytes,
+        mediaType: mediaTypeFor(extensionOf(params.file.relativePath)),
+        contentHash: "",
+        parser: { parserId: "none", parserVersion: "0" },
+        lastExtractedAt,
+        status: "failed",
+        safeDisplayName: safeDisplay(params.file.relativePath),
+    };
+}
+function persistOversizedRow(deps, params, documentId, document, diagnostic, now) {
+    const db = deps.store._internal.db;
+    db.exec("BEGIN");
+    try {
+        insertDocumentRow(db, {
+            id: documentId,
+            capsuleId: params.capsuleId,
+            sourceId: String(params.source.id),
+            documentPath: document.documentPath,
+            sizeBytes: document.sizeBytes,
+            mediaType: document.mediaType,
+            contentHash: document.contentHash,
+            parserId: document.parser.parserId,
+            parserVersion: document.parser.parserVersion,
+            lastExtractedAt: document.lastExtractedAt,
+            status: document.status,
+            safeDisplayName: document.safeDisplayName,
+        });
+        deleteDependentRows(db, params.capsuleId, documentId);
+        insertDiagnosticRow(db, {
+            id: `${String(documentId)}#d0`,
+            capsuleId: params.capsuleId,
+            diagnostic,
+            createdAt: now(),
+        });
+        db.exec("COMMIT");
+    }
+    catch (cause) {
+        db.exec("ROLLBACK");
+        throw cause;
+    }
+}
+function buildOversizedFailure(deps, params, documentId, options, observedSizeBytes) {
+    const now = deps.store._internal.now;
+    const sizeBytes = observedSizeBytes === undefined
+        ? params.file.sizeBytes
+        : Math.max(params.file.sizeBytes, observedSizeBytes);
+    const message = redactMessage(`file size ${String(sizeBytes)} exceeds maxBytes=${String(options.maxBytes)}`, params.source);
+    const diagnostic = {
+        severity: "error",
+        code: "OVERSIZED_FILE",
+        message,
+        documentId,
+    };
+    const document = oversizedDocumentRecord(params, documentId, now(), observedSizeBytes);
+    persistOversizedRow(deps, params, documentId, document, diagnostic, now);
+    return {
+        capsuleId: params.capsuleId,
+        sourceId: params.source.id,
+        relativePath: params.file.relativePath,
+        outcome: {
+            kind: "failed",
+            document,
+            error: { code: "OVERSIZED_FILE", message },
+        },
+        diagnostics: [diagnostic],
+    };
+}