@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/indexing/orchestrator.js

@@ -0,0 +1,1151 @@
+// Indexing orchestrator (Epic #189, Issue #196). Composes #194 discovery, #195 chunking,
+// and #192 embedding into a single streaming pipeline that produces `vectors` rows for a
+// capsule. Every state change emits one `IndexingEvent`; consumers drive the AsyncIterable
+// to back-pressure the pipeline.
+//
+// Pipeline shape per source:
+//
+//   discoverAndExtract() ── (per file) ──┐
+//                                        ├─ document-discovered
+//                                        ├─ extraction skipped (unchanged): document-skipped
+//                                        ├─ extraction persisted: document-extracted →
+//                                        │      chunkDocument → document-chunked →
+//                                        │      embedChunkBatch* → document-embedded
+//                                        └─ extraction failed: document-failed
+//
+// Cancellation: a single `AbortSignal` flows into discovery, chunking, AND the embedding
+// batcher. Aborting mid-document terminates the run with a `job-cancelled` event; rows
+// already persisted for completed documents are kept (the source-of-truth for resume is
+// the chunks/vectors tables, not the in-flight buffer).
+//
+// Force mode: passes `force=true` into the chunker per document so existing chunks are
+// replaced from the current source text, then re-embeds. Discovery's incremental
+// fast-path is bypassed (the skipped outcome is re-shaped to persisted in
+// handleFileExtracted) so chunk-and-embed re-runs even for unchanged file hashes.
+// Recovery mode (partial vector coverage, non-force): re-embeds using existing chunks
+// only — the chunker runs with force=false so it reuses the already-correct chunk rows.
+import { randomUUID } from "node:crypto";
+import { checkpointCompatibility, DEFAULT_LARGE_DOCUMENT_RESOURCE_POLICY, LARGE_DOCUMENT_DIAGNOSTIC_CODES, largeDocumentPolicyFingerprint, } from "@oscharko-dev/keiko-contracts";
+import { assertCompatibleEmbeddingIdentity, verifyEmbeddingCapability, } from "@oscharko-dev/keiko-model-gateway";
+import { chunkDocument } from "../chunking/chunker-runner.js";
+import { countChunksForDocument, deleteChunksForDocument, hasStaleChunksForDocument, } from "../chunking/chunker-persist.js";
+import { chunkingStrategyKey } from "../chunking/index.js";
+import { getCapsule, updateCapsuleState } from "../capsule-lifecycle.js";
+import { discoverAndExtract } from "../discovery/discovery-runner.js";
+import { DEFAULT_DISCOVERY_OPTIONS } from "../discovery/index.js";
+import { deleteDocumentRow, insertDiagnosticRow, listPersistedDocumentsForSource, readDocumentTextRow, updateDocumentStatusRow, } from "../discovery/persist.js";
+import { listCapsuleSources } from "../source-lifecycle.js";
+import { BoundedIndexingCancelledError, BoundedIndexingPolicyError, chunkDocumentBounded, embedDocumentChunksBounded, } from "./bounded-indexing.js";
+import { selectExtractionCheckpoint, upsertExtractionCheckpoint } from "./checkpoint-persist.js";
+import { finalizeJobRow, isJobCancellationRequested, insertJobRow, updateJobCounters, } from "./job-persist.js";
+import { embedChunkBatch } from "./embedding-batcher.js";
+import { countVectorsForDocument, deleteVectorsForDocument, selectChunksForDocument, } from "./vector-persist.js";
+import { DEFAULT_INDEXING_BATCH_SIZE, DEFAULT_INDEXING_CONCURRENCY, IndexingError, } from "./types.js";
+// ─── Abort helper ─────────────────────────────────────────────────────────────
+// Reads `signal?.aborted` through a function call so TypeScript's control-flow analysis
+// does NOT narrow the optional chain after the first false branch. Mirrors the pattern in
+// `discovery/discovery-runner.ts` and `discovery/walk.ts`.
+function aborted(signal) {
+    return signal?.aborted === true;
+}
+function cancellationRequested(state) {
+    return (aborted(state.options.signal) ||
+        isJobCancellationRequested(state.options.store._internal.db, state.jobId));
+}
+// ─── Bounded options ──────────────────────────────────────────────────────────
+function clampBatchSize(raw) {
+    const v = raw ?? DEFAULT_INDEXING_BATCH_SIZE;
+    return Math.max(1, Math.min(DEFAULT_INDEXING_BATCH_SIZE, Math.floor(v)));
+}
+function clampConcurrency(raw) {
+    const v = raw ?? DEFAULT_INDEXING_CONCURRENCY;
+    return Math.max(1, Math.min(DEFAULT_INDEXING_CONCURRENCY, Math.floor(v)));
+}
+function clampDiscoveryInteger(raw, fallback) {
+    if (raw === undefined || !Number.isFinite(raw))
+        return fallback;
+    return Math.max(1, Math.min(fallback, Math.floor(raw)));
+}
+function resolvedDiscoveryOptions(state) {
+    const raw = state.options.discoveryOptions;
+    const base = {
+        maxDepth: clampDiscoveryInteger(raw?.maxDepth, DEFAULT_DISCOVERY_OPTIONS.maxDepth),
+        maxFiles: clampDiscoveryInteger(raw?.maxFiles, DEFAULT_DISCOVERY_OPTIONS.maxFiles),
+    };
+    const signal = raw?.signal ?? state.options.signal;
+    return signal === undefined ? base : { ...base, signal };
+}
+// ─── Source resolution ────────────────────────────────────────────────────────
+function resolveSources(options, capsule) {
+    const all = listCapsuleSources(options.store, capsule.id);
+    if (all.length === 0) {
+        throw new IndexingError("INVALID_OPTIONS", `Capsule ${String(capsule.id)} has no attached sources to index.`);
+    }
+    if (options.sourceIds === undefined)
+        return all;
+    const allow = new Set(options.sourceIds.map((s) => String(s)));
+    if (allow.size === 0) {
+        throw new IndexingError("INVALID_OPTIONS", "sourceIds must contain at least one source id.");
+    }
+    const selected = all.filter((s) => allow.has(String(s.id)));
+    if (selected.length !== allow.size) {
+        throw new IndexingError("INVALID_OPTIONS", "sourceIds must reference sources attached to the target capsule.");
+    }
+    return selected;
+}
+function buildCounters(state) {
+    return {
+        total: state.totalDocuments,
+        processed: state.processedDocuments,
+        failed: state.failedDocuments,
+        skipped: state.skippedDocuments,
+        resumeToken: state.lastResumeToken === null ? null : String(state.lastResumeToken),
+    };
+}
+function persistJobProgress(state) {
+    updateJobCounters(state.options.store._internal.db, state.jobId, buildCounters(state));
+}
+function emitProgress(options, event) {
+    if (options.progress === undefined)
+        return;
+    // Caller-provided callback; isolate so a throwing consumer cannot crash the orchestrator
+    // mid-document. Errors are surfaced as a document-failed event would be — but we never
+    // mutate state on a progress-callback throw because that would couple the caller's bug
+    // to our run accounting.
+    try {
+        options.progress(event);
+    }
+    catch {
+        // intentionally swallowed — progress sinks must not affect run correctness
+    }
+}
+function clearDocumentArtifacts(state, documentId, options) {
+    deleteVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
+    if (options.deleteChunks) {
+        deleteChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId);
+    }
+}
+function markDocumentFailed(state, documentId) {
+    updateDocumentStatusRow(state.options.store._internal.db, state.capsule.id, documentId, "failed");
+}
+const SELECT_CHUNKS_WITH_OFFSETS_SQL = [
+    "SELECT c.id, c.capsule_id, c.source_id, c.document_id, c.parsed_unit_id, c.order_index,",
+    "  COALESCE(c.character_start, pu.character_start) AS char_start,",
+    "  COALESCE(c.character_end, pu.character_end) AS char_end",
+    "FROM chunks AS c",
+    "JOIN parsed_units AS pu ON pu.capsule_id = c.capsule_id AND pu.id = c.parsed_unit_id",
+    "WHERE c.capsule_id = :c AND c.document_id = :d",
+    "ORDER BY c.order_index ASC",
+].join(" ");
+function selectChunkProjections(state, documentId) {
+    const rows = state.options.store._internal.db
+        .prepare(SELECT_CHUNKS_WITH_OFFSETS_SQL)
+        .all({ c: state.capsule.id, d: documentId });
+    return rows;
+}
+function projectChunksToEmbed(state, documentId, sourceText) {
+    const projections = selectChunkProjections(state, documentId);
+    const out = [];
+    for (const row of projections) {
+        const start = row.char_start ?? 0;
+        const end = row.char_end ?? sourceText.length;
+        const text = sourceText.slice(start, end);
+        out.push({
+            id: row.id,
+            capsuleId: row.capsule_id,
+            sourceId: row.source_id,
+            documentId,
+            text,
+        });
+    }
+    return out;
+}
+function scopeRootOf(source) {
+    const scope = source.scope;
+    if (scope.kind === "folder")
+        return { absoluteRoot: scope.rootPath };
+    if (scope.kind === "repository")
+        return { absoluteRoot: scope.repositoryRoot };
+    return { absoluteRoot: scope.rootPath };
+}
+function joinAbs(root, rel) {
+    if (root.endsWith("/"))
+        return `${root}${rel}`;
+    return `${root}/${rel}`;
+}
+function normaliseSep(p) {
+    return p.replace(/\\/g, "/");
+}
+function isContained(absoluteRoot, absolutePath) {
+    const normRoot = normaliseSep(absoluteRoot);
+    const normPath = normaliseSep(absolutePath);
+    if (normPath === normRoot)
+        return true;
+    const prefix = normRoot.endsWith("/") ? normRoot : `${normRoot}/`;
+    return normPath.startsWith(prefix);
+}
+function readSourceText(state, source, relativePath) {
+    const { absoluteRoot } = scopeRootOf(source);
+    const abs = joinAbs(absoluteRoot, relativePath);
+    let real;
+    try {
+        real = state.options.workspaceFs.realPath(abs);
+    }
+    catch (cause) {
+        throw new IndexingError("PERSISTENCE_FAILED", "source text could not be read before embedding", { cause });
+    }
+    if (!isContained(absoluteRoot, real)) {
+        throw new IndexingError("PERSISTENCE_FAILED", `source realpath escapes scope root before embedding: ${relativePath}`);
+    }
+    try {
+        return state.options.workspaceFs.readFileUtf8(normaliseSep(real));
+    }
+    catch (cause) {
+        throw new IndexingError("PERSISTENCE_FAILED", "source text could not be read before embedding", { cause });
+    }
+}
+function resolveChunkSourceText(state, documentId, source, relativePath) {
+    const persistedText = readDocumentTextRow(state.options.store._internal.db, state.options.store._internal.contentCipher, state.capsule.id, documentId);
+    if (persistedText !== undefined) {
+        return persistedText;
+    }
+    return readSourceText(state, source, relativePath);
+}
+// ─── Batch boundaries ─────────────────────────────────────────────────────────
+function sliceIntoBatches(items, batchSize) {
+    if (items.length === 0)
+        return [];
+    const out = [];
+    for (let i = 0; i < items.length; i += batchSize) {
+        out.push(items.slice(i, i + batchSize));
+    }
+    return out;
+}
+function cancellationError() {
+    return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
+}
+function recordCancellationIfRequested(state, errors) {
+    if (cancellationRequested(state) && !errors.some((error) => error.code === "CANCELLED")) {
+        errors.push(cancellationError());
+    }
+}
+async function embedDocumentChunks(state, documentId, source, relativePath) {
+    // Text-like documents are re-read from disk; binary parsers persist a normalized text
+    // projection so chunk slicing stays aligned with extracted content.
+    const sourceText = resolveChunkSourceText(state, documentId, source, relativePath);
+    const chunks = projectChunksToEmbed(state, documentId, sourceText);
+    if (chunks.length === 0) {
+        return { vectorCount: 0, errors: [], lastChunkId: null };
+    }
+    const batches = sliceIntoBatches(chunks, state.batchSize);
+    const errors = [];
+    let vectorCount = 0;
+    let lastChunkId = null;
+    for (const batch of batches) {
+        if (cancellationRequested(state))
+            break;
+        const result = await embedChunkBatch(batch, {
+            adapter: state.options.embeddingAdapter,
+            store: state.options.store,
+            pinnedIdentity: state.capsule.embeddingModelIdentity,
+            concurrency: state.concurrency,
+            ...(state.options.signal !== undefined ? { signal: state.options.signal } : {}),
+            now: state.now,
+            idSource: state.idSource,
+        });
+        vectorCount += result.vectors.length;
+        errors.push(...result.errors);
+        if (result.vectors.length > 0) {
+            const last = result.vectors[result.vectors.length - 1];
+            if (last !== undefined)
+                lastChunkId = last.chunkId;
+        }
+        // Identity-incompatibility is detected by the batcher — stop emitting further batches
+        // for this document so the orchestrator can mark the whole job failed.
+        if (result.errors.some((e) => e.code === "INCOMPATIBLE_EMBEDDING_IDENTITY")) {
+            break;
+        }
+        if (cancellationRequested(state))
+            break;
+    }
+    recordCancellationIfRequested(state, errors);
+    return { vectorCount, errors, lastChunkId };
+}
+// ─── Document handlers ────────────────────────────────────────────────────────
+function handleExtractionSkipped(state, result) {
+    state.skippedDocuments += 1;
+    return {
+        kind: "document-skipped",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
+        documentId: result.outcome.kind === "skipped" ? result.outcome.document.id : "",
+        reason: "unchanged",
+    };
+}
+// GRD-010: transient IO failure codes that must NOT destroy a previously-good index on an
+// incremental refresh. Mirrors the gate in discovery/extract.ts buildFailureResult.
+const TRANSIENT_DISCOVERY_CODES = new Set(["READ_FAILED", "STAT_FAILED"]);
+function handleExtractionFailed(state, result) {
+    const errMessage = result.outcome.kind === "failed" ? result.outcome.error.message : "extraction failed";
+    const errCode = result.outcome.kind === "failed" ? result.outcome.error.code : "READ_FAILED";
+    if (result.outcome.kind === "failed") {
+        const documentId = result.outcome.document.id;
+        // GRD-010: a transient re-read failure on a document that still has a prior good index
+        // (extract.ts preserved its chunks/vectors) is reported as a non-destructive skip, NOT a
+        // failure — the retrievable content survives until a successful re-extraction.
+        if (TRANSIENT_DISCOVERY_CODES.has(errCode) &&
+            countChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId) > 0) {
+            state.skippedDocuments += 1;
+            return {
+                kind: "document-skipped",
+                jobId: state.jobId,
+                capsuleId: state.capsule.id,
+                sourceId: result.sourceId,
+                documentId,
+                reason: "unchanged",
+            };
+        }
+        state.failedDocuments += 1;
+        clearDocumentArtifacts(state, documentId, { deleteChunks: true });
+        markDocumentFailed(state, documentId);
+        const error = { code: `DISCOVERY_FAILED:${errCode}`, message: errMessage };
+        state.lastError = error;
+        return {
+            kind: "document-failed",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: result.sourceId,
+            documentId,
+            relativePath: result.relativePath,
+            error,
+        };
+    }
+    state.failedDocuments += 1;
+    const error = { code: `DISCOVERY_FAILED:${errCode}`, message: errMessage };
+    state.lastError = error;
+    return {
+        kind: "document-failed",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
+        relativePath: result.relativePath,
+        error,
+    };
+}
+function resolveChunkCount(state, documentId, skippedExisting, freshChunkIds) {
+    if (!skippedExisting)
+        return freshChunkIds.length;
+    // When skippedExisting, the chunks table already holds the rows from a prior run; count
+    // them so the chunked event still reports an accurate number.
+    return selectChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId)
+        .length;
+}
+function embeddingCoverage(state, documentId) {
+    return {
+        chunkCount: countChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId),
+        vectorCount: countVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId),
+    };
+}
+function hasCompleteVectorCoverage(state, documentId) {
+    const coverage = embeddingCoverage(state, documentId);
+    return coverage.chunkCount > 0 && coverage.vectorCount === coverage.chunkCount;
+}
+function persistedDocumentId(result) {
+    if (result.outcome.kind !== "persisted") {
+        throw new IndexingError("INVALID_OPTIONS", "chunkPersistedDocument called with non-persisted result");
+    }
+    return result.outcome.document.id;
+}
+function chunkPersistedDocument(state, result) {
+    const documentId = persistedDocumentId(result);
+    const sourceText = resolveChunkSourceText(state, documentId, sourceForResult(state, result), result.relativePath);
+    const chunkResult = chunkDocument(state.options.store, {
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
+        documentId,
+        sourceText,
+        force: state.options.force === true,
+        ...(state.options.signal !== undefined ? { signal: state.options.signal } : {}),
+    }, state.options.chunkingOptions);
+    const chunkCount = resolveChunkCount(state, documentId, chunkResult.skippedExisting, chunkResult.chunkIds);
+    return {
+        events: chunkedDocumentEvents(state, result.sourceId, documentId, result.relativePath, chunkCount),
+        documentId,
+        chunkCount,
+    };
+}
+function chunkedDocumentEvents(state, sourceId, documentId, relativePath, chunkCount) {
+    return [
+        {
+            kind: "document-extracted",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId,
+            documentId,
+            relativePath,
+        },
+        {
+            kind: "document-chunked",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId,
+            documentId,
+            chunkCount,
+        },
+    ];
+}
+function sourceForResult(state, result) {
+    // Sources are resolved once at job start (see buildInitialState) and cached on RunState.
+    // The capsule lifecycleState gates concurrent mutation, so the map stays consistent for
+    // the duration of the run — no per-document SELECT against capsule_sources.
+    const match = state.sourcesById.get(String(result.sourceId));
+    if (match === undefined) {
+        throw new IndexingError("INVALID_OPTIONS", `result references unknown source ${String(result.sourceId)}`);
+    }
+    return match;
+}
+// Incremental fast-path: skips embedding when vectors already exist (non-force run), or
+// deletes prior vectors to prepare for a forced re-embed.
+// Returns a PersistedHandling to short-circuit when already-embedded, undefined to continue.
+function applyIncrementalFastPath(state, sourceId, documentId) {
+    const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId, chunkingStrategyKey(state.options.chunkingOptions));
+    if (state.options.force !== true) {
+        const coverage = embeddingCoverage(state, documentId);
+        if (coverage.chunkCount > 0 && coverage.vectorCount === coverage.chunkCount && !staleChunks) {
+            state.skippedDocuments += 1;
+            return {
+                events: [
+                    {
+                        kind: "document-skipped",
+                        jobId: state.jobId,
+                        capsuleId: state.capsule.id,
+                        sourceId,
+                        documentId,
+                        reason: "already-embedded",
+                    },
+                ],
+            };
+        }
+        if (coverage.vectorCount > 0) {
+            deleteVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
+        }
+        return undefined;
+    }
+    return undefined;
+}
+// Runs the chunker and returns its result, or a PersistedHandling failure event on throw.
+function tryChunkDocument(state, result, documentId) {
+    try {
+        return { chunked: chunkPersistedDocument(state, result) };
+    }
+    catch {
+        if (cancellationRequested(state)) {
+            clearDocumentArtifacts(state, documentId, { deleteChunks: true });
+            return { events: [] };
+        }
+        state.failedDocuments += 1;
+        clearDocumentArtifacts(state, documentId, { deleteChunks: true });
+        markDocumentFailed(state, documentId);
+        const error = {
+            code: "CHUNKING_FAILED",
+            message: "document chunking failed",
+        };
+        state.lastError = error;
+        return {
+            events: [
+                {
+                    kind: "document-failed",
+                    jobId: state.jobId,
+                    capsuleId: state.capsule.id,
+                    sourceId: result.sourceId,
+                    documentId,
+                    relativePath: result.relativePath,
+                    error,
+                },
+            ],
+        };
+    }
+}
+function appendDocumentFailure(state, events, sourceId, documentId, relativePath, error, options) {
+    state.failedDocuments += 1;
+    clearDocumentArtifacts(state, documentId, options);
+    markDocumentFailed(state, documentId);
+    state.lastError = error;
+    events.push({
+        kind: "document-failed",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId,
+        documentId,
+        relativePath,
+        error,
+    });
+    return { events };
+}
+function completeEmbeddedDocument(state, events, sourceId, documentId, embedResult) {
+    state.processedDocuments += 1;
+    state.vectorsPersisted += embedResult.vectorCount;
+    if (embedResult.lastChunkId !== null)
+        state.lastResumeToken = embedResult.lastChunkId;
+    events.push({
+        kind: "document-embedded",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId,
+        documentId,
+        vectorCount: embedResult.vectorCount,
+        resumeToken: embedResult.lastChunkId ?? `${String(documentId)}#empty`,
+    });
+    return { events };
+}
+function isCancellationOnlyEmbedResult(state, embedResult) {
+    return (cancellationRequested(state) &&
+        embedResult.errors.length > 0 &&
+        embedResult.errors.every((error) => error.code === "CANCELLED"));
+}
+// Maps an EmbedDocumentResult into PersistedHandling events, mutating run-state counters.
+function applyEmbedResult(state, sourceId, documentId, relativePath, priorEvents, embedResult) {
+    const events = [...priorEvents];
+    if (isCancellationOnlyEmbedResult(state, embedResult)) {
+        return { events };
+    }
+    const identityErr = embedResult.errors.find((e) => e.code === "INCOMPATIBLE_EMBEDDING_IDENTITY");
+    if (identityErr !== undefined) {
+        return {
+            ...appendDocumentFailure(state, events, sourceId, documentId, relativePath, identityErr, {
+                deleteChunks: false,
+            }),
+            identityFailure: identityErr,
+        };
+    }
+    if (embedResult.errors.length > 0) {
+        const firstErr = embedResult.errors[0] ?? {
+            code: "EMBEDDING_ADAPTER_FAILED",
+            message: "embedding adapter failed",
+        };
+        return appendDocumentFailure(state, events, sourceId, documentId, relativePath, firstErr, {
+            deleteChunks: false,
+        });
+    }
+    return completeEmbeddedDocument(state, events, sourceId, documentId, embedResult);
+}
+function* persistedEvents(handling) {
+    for (const event of handling.events) {
+        yield event;
+    }
+}
+// ─── Bounded large-document chunk + embed + resume (Epic #1160, Issue #1286) ─────
+function boundedCurrentFingerprint(state, checkpoint) {
+    const policy = boundedPolicy(state);
+    return {
+        ...checkpoint.fingerprint,
+        policyFingerprint: largeDocumentPolicyFingerprint(policy),
+        chunkingStrategyVersion: chunkingStrategyKey(state.options.chunkingOptions),
+        embeddingIdentity: state.capsule.embeddingModelIdentity,
+    };
+}
+function boundedPolicy(state) {
+    return state.options.largeDocumentPolicy ?? DEFAULT_LARGE_DOCUMENT_RESOURCE_POLICY;
+}
+function writeBoundedCheckpoint(state, checkpoint, fingerprint, phase, chunkCursor, embeddedChunkCursor, lastEmbeddedChunkId, terminalDiagnostics = checkpoint.terminalDiagnostics) {
+    upsertExtractionCheckpoint(state.options.store._internal.db, {
+        capsuleId: checkpoint.capsuleId,
+        documentId: checkpoint.documentId,
+        jobId: state.jobId,
+        strategy: checkpoint.strategy,
+        phase,
+        pageCursor: checkpoint.pageCursor,
+        sectionCursor: checkpoint.sectionCursor,
+        objectCursor: checkpoint.objectCursor,
+        extractedTextBytes: checkpoint.extractedTextBytes,
+        chunkCursor,
+        embeddedChunkCursor,
+        ...(lastEmbeddedChunkId !== null ? { lastEmbeddedChunkId } : {}),
+        retryCount: checkpoint.retryCount,
+        coverage: checkpoint.coverage,
+        fingerprint,
+        terminalDiagnostics,
+        createdAt: checkpoint.createdAt,
+        updatedAt: state.now(),
+    });
+}
+function persistCheckpointIncompatibleDiagnostic(state, documentId, reasons) {
+    insertDiagnosticRow(state.options.store._internal.db, {
+        id: `${String(documentId)}#checkpoint-incompatible`,
+        capsuleId: state.capsule.id,
+        diagnostic: {
+            severity: "warning",
+            code: LARGE_DOCUMENT_DIAGNOSTIC_CODES.CHECKPOINT_INCOMPATIBLE,
+            message: `resume refused and restarted; changed: ${reasons.join(", ")}`,
+            documentId,
+        },
+        createdAt: state.now(),
+    });
+}
+function boundedNeedsRechunk(state, documentId, fingerprint, incompatible) {
+    const db = state.options.store._internal.db;
+    return (incompatible ||
+        state.options.force === true ||
+        countChunksForDocument(db, state.capsule.id, documentId) === 0 ||
+        hasStaleChunksForDocument(db, state.capsule.id, documentId, fingerprint.chunkingStrategyVersion));
+}
+function boundedEmbedDeps(state, documentId, fingerprint, checkpoint, chunkCount) {
+    return {
+        store: state.options.store,
+        capsuleId: state.capsule.id,
+        documentId,
+        adapter: state.options.embeddingAdapter,
+        identity: state.capsule.embeddingModelIdentity,
+        batchSize: state.batchSize,
+        concurrency: state.concurrency,
+        now: state.now,
+        idSource: state.idSource,
+        policy: boundedPolicy(state),
+        ...(state.options.signal !== undefined ? { signal: state.options.signal } : {}),
+        onBatch: (cursor, lastId) => {
+            writeBoundedCheckpoint(state, checkpoint, fingerprint, "embedding", chunkCount, cursor, lastId);
+        },
+    };
+}
+// Bounded path for a progressively-extracted document: resumes a compatible checkpoint, restarts an
+// incompatible one with a CHECKPOINT_INCOMPATIBLE diagnostic, chunks + embeds through SUBSTR-backed
+// readers, and advances the durable checkpoint between batches.
+// Reconciles existing chunks/vectors against the current fingerprint: refuses an incompatible
+// checkpoint (diagnostic + delete vectors) and re-chunks when stale/forced/missing. Returns the
+// chunk count.
+function prepareBoundedChunks(state, result, documentId, checkpoint, fingerprint) {
+    const db = state.options.store._internal.db;
+    const compat = checkpointCompatibility(checkpoint.fingerprint, fingerprint);
+    if (!compat.compatible) {
+        persistCheckpointIncompatibleDiagnostic(state, documentId, compat.reasons);
+        deleteVectorsForDocument(db, state.capsule.id, documentId);
+    }
+    if (boundedNeedsRechunk(state, documentId, fingerprint, !compat.compatible)) {
+        chunkDocumentBounded(state.options.store, { capsuleId: state.capsule.id, sourceId: result.sourceId, documentId }, state.options.chunkingOptions, state.options.signal, boundedPolicy(state));
+        deleteVectorsForDocument(db, state.capsule.id, documentId);
+    }
+    return countChunksForDocument(db, state.capsule.id, documentId);
+}
+function boundedChunkPreparationFailure(state, result, documentId, checkpoint, fingerprint, cause) {
+    const db = state.options.store._internal.db;
+    if (cause instanceof BoundedIndexingCancelledError || cancellationRequested(state)) {
+        updateDocumentStatusRow(db, state.capsule.id, documentId, "pending");
+        writeBoundedCheckpoint(state, checkpoint, fingerprint, "cancelled", countChunksForDocument(db, state.capsule.id, documentId), countVectorsForDocument(db, state.capsule.id, documentId), null);
+        return { events: [] };
+    }
+    const error = cause instanceof BoundedIndexingPolicyError
+        ? cause.toIndexingError()
+        : { code: "CHUNKING_FAILED", message: "document chunking failed" };
+    writeBoundedCheckpoint(state, checkpoint, fingerprint, "failed", countChunksForDocument(db, state.capsule.id, documentId), countVectorsForDocument(db, state.capsule.id, documentId), null, [{ severity: "error", code: error.code, message: error.message, documentId }]);
+    return appendDocumentFailure(state, [], result.sourceId, documentId, result.relativePath, error, {
+        deleteChunks: true,
+    });
+}
+function prepareBoundedChunksSafely(state, result, documentId, checkpoint, fingerprint) {
+    try {
+        return { chunkCount: prepareBoundedChunks(state, result, documentId, checkpoint, fingerprint) };
+    }
+    catch (cause) {
+        return boundedChunkPreparationFailure(state, result, documentId, checkpoint, fingerprint, cause);
+    }
+}
+function persistBoundedEmbedCheckpoint(state, checkpoint, fingerprint, documentId, chunkCount, embedResult) {
+    writeBoundedCheckpoint(state, checkpoint, fingerprint, embedResult.errors.length === 0 ? "complete" : "failed", chunkCount, embedResult.embeddedCursor, embedResult.lastChunkId, embedResult.errors.map((error) => ({
+        severity: "error",
+        code: error.code,
+        message: error.message,
+        documentId,
+    })));
+}
+function persistBoundedEmbedCancellation(state, checkpoint, fingerprint, documentId, chunkCount, embedResult) {
+    updateDocumentStatusRow(state.options.store._internal.db, state.capsule.id, documentId, "pending");
+    writeBoundedCheckpoint(state, checkpoint, fingerprint, "cancelled", chunkCount, embedResult.embeddedCursor, embedResult.lastChunkId, embedResult.errors.map((error) => ({
+        severity: "info",
+        code: error.code,
+        message: error.message,
+        documentId,
+    })));
+}
+async function persistBoundedEmbeddingResult(state, result, documentId, checkpoint, fingerprint, chunkCount) {
+    // The bounded embed self-resumes from chunks that have no vector yet.
+    const embedResult = await embedDocumentChunksBounded(boundedEmbedDeps(state, documentId, fingerprint, checkpoint, chunkCount));
+    if (embedResult.cancelled) {
+        persistBoundedEmbedCancellation(state, checkpoint, fingerprint, documentId, chunkCount, embedResult);
+        return undefined;
+    }
+    persistBoundedEmbedCheckpoint(state, checkpoint, fingerprint, documentId, chunkCount, embedResult);
+    return applyEmbedResult(state, result.sourceId, documentId, result.relativePath, [], {
+        vectorCount: embedResult.vectorCount,
+        errors: embedResult.errors,
+        lastChunkId: embedResult.lastChunkId,
+    });
+}
+async function* handleBoundedDocument(state, result, documentId, checkpoint) {
+    const db = state.options.store._internal.db;
+    const sourceId = result.sourceId;
+    const fingerprint = boundedCurrentFingerprint(state, checkpoint);
+    const prepared = prepareBoundedChunksSafely(state, result, documentId, checkpoint, fingerprint);
+    if (!("chunkCount" in prepared)) {
+        yield* persistedEvents(prepared);
+        return;
+    }
+    const { chunkCount } = prepared;
+    yield* chunkedDocumentEvents(state, sourceId, documentId, result.relativePath, chunkCount);
+    persistJobProgress(state);
+    const alreadyEmbedded = countVectorsForDocument(db, state.capsule.id, documentId);
+    writeBoundedCheckpoint(state, checkpoint, fingerprint, "embedding", chunkCount, alreadyEmbedded, null);
+    const embedded = await persistBoundedEmbeddingResult(state, result, documentId, checkpoint, fingerprint, chunkCount);
+    if (embedded !== undefined)
+        yield* persistedEvents(embedded);
+}
+// Wraps the chunk-then-embed pipeline for a single persisted document. Extraction/chunking
+// events are yielded before awaiting embeddings, so progress consumers see pre-model work
+// immediately instead of only after all embedding batches finish.
+function* handleUnsupportedDocument(state, result, documentId) {
+    clearDocumentArtifacts(state, documentId, { deleteChunks: true });
+    state.skippedDocuments += 1;
+    yield {
+        kind: "document-extracted",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
+        documentId,
+        relativePath: result.relativePath,
+    };
+    yield {
+        kind: "document-skipped",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
+        documentId,
+        reason: "unsupported",
+    };
+}
+async function* handlePersistedDocument(state, result) {
+    const documentId = result.outcome.kind === "persisted" ? result.outcome.document.id : null;
+    if (documentId === null)
+        return;
+    if (result.outcome.document.status === "unsupported") {
+        yield* handleUnsupportedDocument(state, result, documentId);
+        return;
+    }
+    // A document with a durable extraction checkpoint took the progressive page-windowed path; route
+    // it to the bounded chunk/embed pass (which owns its own resume + fast-path logic).
+    const checkpoint = selectExtractionCheckpoint(state.options.store._internal.db, state.capsule.id, documentId);
+    if (checkpoint !== undefined) {
+        yield* handleBoundedDocument(state, result, documentId, checkpoint);
+        return;
+    }
+    const fastPath = applyIncrementalFastPath(state, result.sourceId, documentId);
+    if (fastPath !== undefined) {
+        yield* persistedEvents(fastPath);
+        return;
+    }
+    const chunkStep = tryChunkDocument(state, result, documentId);
+    if (!("chunked" in chunkStep)) {
+        yield* persistedEvents(chunkStep);
+        return;
+    }
+    yield* chunkStep.chunked.events;
+    persistJobProgress(state);
+    const embedResult = await embedDocumentChunks(state, documentId, sourceForResult(state, result), result.relativePath);
+    yield* persistedEvents(applyEmbedResult(state, result.sourceId, documentId, result.relativePath, [], embedResult));
+}
+function* handleExtractionSkippedEvents(state, result) {
+    yield handleExtractionSkipped(state, result);
+}
+function* handleExtractionFailedEvents(state, result) {
+    yield handleExtractionFailed(state, result);
+}
+// Routes a file-extracted event: force-skipped docs are re-shaped to persisted so the
+// standard chunk-and-embed pipeline runs on them.
+async function* handleFileExtracted(state, result) {
+    if (result.outcome.kind === "skipped") {
+        // In force mode, an "unchanged" document still needs chunk-and-embed because the
+        // caller explicitly requested a fresh embedding pass. Re-shape the skipped outcome as
+        // a persisted outcome (the document row exists and is valid) so the standard pipeline
+        // runs. Outside force/recovery mode, surface the skip as-is.
+        const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, result.outcome.document.id, chunkingStrategyKey(state.options.chunkingOptions));
+        const missingVectors = result.outcome.document.status === "extracted" &&
+            !hasCompleteVectorCoverage(state, result.outcome.document.id);
+        if (state.options.force === true || staleChunks || missingVectors) {
+            const synthetic = {
+                capsuleId: result.capsuleId,
+                sourceId: result.sourceId,
+                relativePath: result.relativePath,
+                outcome: { kind: "persisted", document: result.outcome.document },
+                diagnostics: result.diagnostics,
+            };
+            yield* handlePersistedDocument(state, synthetic);
+            return;
+        }
+        yield* handleExtractionSkippedEvents(state, result);
+        return;
+    }
+    if (result.outcome.kind === "failed") {
+        yield* handleExtractionFailedEvents(state, result);
+        return;
+    }
+    yield* handlePersistedDocument(state, result);
+}
+async function* handleDiscoveryEvent(state, source, evt) {
+    if (evt.kind === "file-discovered") {
+        state.totalDocuments += 1;
+        yield {
+            kind: "document-discovered",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: source.id,
+            relativePath: evt.relativePath,
+            sizeBytes: evt.sizeBytes,
+        };
+        return;
+    }
+    if (evt.kind === "scope-error") {
+        state.failedDocuments += 1;
+        const err = {
+            code: `DISCOVERY_FAILED:${evt.error.code}`,
+            message: evt.error.message,
+        };
+        state.lastError = err;
+        yield {
+            kind: "document-failed",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: source.id,
+            ...(evt.error.relativePath !== undefined ? { relativePath: evt.error.relativePath } : {}),
+            error: err,
+        };
+        return;
+    }
+    if (evt.kind === "cancelled" || evt.kind === "completed") {
+        // No-op at this level: the outer loop drives terminal events.
+        return;
+    }
+    // evt.kind === "file-extracted"
+    yield* handleFileExtracted(state, evt.result);
+}
+function shouldStopAfterEvent(event) {
+    return event.kind === "document-failed" && event.error.code === "INCOMPATIBLE_EMBEDDING_IDENTITY";
+}
+async function* streamDiscoveryEvent(state, source, evt) {
+    for await (const event of handleDiscoveryEvent(state, source, evt)) {
+        persistJobProgress(state);
+        yield event;
+        if (shouldStopAfterEvent(event)) {
+            return true;
+        }
+    }
+    return false;
+}
+// ─── Per-source pipeline ──────────────────────────────────────────────────────
+async function* runOneSource(state, source) {
+    const stream = discoverAndExtract({
+        fs: state.options.workspaceFs,
+        store: state.options.store,
+        parserRegistry: state.options.parserRegistry,
+        ...(state.options.largeDocumentPolicy !== undefined
+            ? { largeDocumentPolicy: state.options.largeDocumentPolicy }
+            : {}),
+        ...(state.options.progressiveExtractors !== undefined
+            ? { progressiveExtractors: state.options.progressiveExtractors }
+            : {}),
+        ...(state.options.extractionCapabilities !== undefined
+            ? { extractionCapabilities: state.options.extractionCapabilities }
+            : {}),
+        largeDocumentJobId: state.jobId,
+        chunkingStrategyVersion: chunkingStrategyKey(state.options.chunkingOptions),
+    }, sourceDiscoveryParams(state, source));
+    const progress = {
+        cancelled: false,
+        sawScopeError: false,
+        completed: false,
+        discoveredPaths: new Set(),
+    };
+    for await (const evt of stream) {
+        observeSourceEvent(progress, evt);
+        if (cancellationRequested(state)) {
+            progress.cancelled = true;
+            break;
+        }
+        const shouldStop = yield* streamDiscoveryEvent(state, source, evt);
+        if (shouldStop) {
+            return;
+        }
+        // After yielding a batch we re-check the signal — the consumer's awaiting iterator
+        // may have aborted between events.
+        if (cancellationRequested(state)) {
+            progress.cancelled = true;
+            break;
+        }
+    }
+    finalizeSourceRun(state, source, progress);
+}
+function sourceDiscoveryParams(state, source) {
+    return {
+        capsuleId: state.capsule.id,
+        source,
+        discovery: resolvedDiscoveryOptions(state),
+    };
+}
+function observeSourceEvent(progress, evt) {
+    if (evt.kind === "file-discovered") {
+        progress.discoveredPaths.add(evt.relativePath);
+        return;
+    }
+    if (evt.kind === "scope-error") {
+        progress.sawScopeError = true;
+        return;
+    }
+    if (evt.kind === "cancelled") {
+        progress.cancelled = true;
+        return;
+    }
+    if (evt.kind === "completed") {
+        progress.completed = true;
+    }
+}
+function pruneDeletedSourceDocuments(state, source, discoveredPaths) {
+    const persisted = listPersistedDocumentsForSource(state.options.store._internal.db, state.capsule.id, source.id);
+    for (const document of persisted) {
+        if (discoveredPaths.has(document.document_path))
+            continue;
+        deleteDocumentRow(state.options.store._internal.db, state.capsule.id, document.id);
+    }
+}
+function finalizeSourceRun(state, source, progress) {
+    if (progress.cancelled)
+        return;
+    if (!progress.completed || progress.sawScopeError)
+        return;
+    if (progress.discoveredPaths.size >= resolvedDiscoveryOptions(state).maxFiles)
+        return;
+    pruneDeletedSourceDocuments(state, source, progress.discoveredPaths);
+}
+// ─── Capsule resolution + job lifecycle ───────────────────────────────────────
+function resolveCapsule(options) {
+    const capsule = getCapsule(options.store, options.capsuleId);
+    if (capsule === undefined) {
+        throw new IndexingError("CAPSULE_NOT_FOUND", `capsule not found: ${String(options.capsuleId)}`);
+    }
+    return capsule;
+}
+function buildInitialState(options, capsule, sources, jobId, startedAt) {
+    const sourcesById = new Map();
+    for (const source of sources)
+        sourcesById.set(String(source.id), source);
+    return {
+        jobId,
+        capsule,
+        options,
+        batchSize: clampBatchSize(options.batchSize),
+        concurrency: clampConcurrency(options.concurrency),
+        now: options.now ?? options.store._internal.now,
+        idSource: options.idSource ?? (() => randomUUID()),
+        startedAt,
+        sourcesById,
+        totalDocuments: 0,
+        processedDocuments: 0,
+        failedDocuments: 0,
+        skippedDocuments: 0,
+        vectorsPersisted: 0,
+        lastResumeToken: null,
+    };
+}
+function buildResult(state, status, finishedAt) {
+    return {
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        status,
+        totalDocuments: state.totalDocuments,
+        processedDocuments: state.processedDocuments,
+        failedDocuments: state.failedDocuments,
+        skippedDocuments: state.skippedDocuments,
+        vectorsPersisted: state.vectorsPersisted,
+        startedAt: state.startedAt,
+        finishedAt,
+        ...(state.lastError !== undefined ? { lastError: state.lastError } : {}),
+        embeddingIdentity: state.capsule.embeddingModelIdentity,
+    };
+}
+async function verifyEmbeddingPreflight(state) {
+    try {
+        const result = await verifyEmbeddingCapability(state.options.embeddingAdapter, {
+            modelId: state.capsule.embeddingModelIdentity.modelId,
+            provider: state.capsule.embeddingModelIdentity.provider,
+            vectorMetric: state.capsule.embeddingModelIdentity.vectorMetric,
+            expectedDimensions: state.capsule.embeddingModelIdentity.vectorDimensions,
+            ...(state.options.signal !== undefined ? { signal: state.options.signal } : {}),
+        });
+        if (result.ok) {
+            const compatibility = assertCompatibleEmbeddingIdentity(state.capsule.embeddingModelIdentity, result.identity);
+            if (compatibility.ok)
+                return undefined;
+            return {
+                code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
+                message: compatibility.safeMessage,
+            };
+        }
+        return {
+            code: result.reason === "dimension-mismatch"
+                ? "INCOMPATIBLE_EMBEDDING_IDENTITY"
+                : "EMBEDDING_ADAPTER_FAILED",
+            message: result.safeMessage,
+        };
+    }
+    catch (cause) {
+        if (cancellationRequested(state) ||
+            (cause instanceof DOMException && cause.name === "AbortError")) {
+            return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
+        }
+        return {
+            code: "EMBEDDING_ADAPTER_FAILED",
+            message: "embedding capability preflight failed before indexing started",
+        };
+    }
+}
+function persistStartedJob(state, sources) {
+    insertJobRow(state.options.store._internal.db, {
+        id: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceIds: sources.map((source) => source.id),
+        startedAt: state.startedAt,
+    });
+    try {
+        updateCapsuleState(state.options.store, state.capsule.id, "indexing");
+    }
+    catch {
+        // The capsule state column is informational — failing to flip it must not abort the
+        // run. The events stream remains the source of truth.
+    }
+}
+function sourceIdsForState(state) {
+    return [...state.sourcesById.values()].map((source) => source.id);
+}
+function emitJobStarted(state, sources) {
+    const event = {
+        kind: "job-started",
+        jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceIds: sources.map((source) => source.id),
+        startedAt: state.startedAt,
+    };
+    state.options.auditSink?.emit({
+        kind: "indexing-job-started",
+        capsuleId: state.capsule.id,
+        sourceIds: sources.map((source) => source.id),
+        jobId: state.jobId,
+        occurredAt: state.startedAt,
+    });
+    return emit(state, event);
+}
+async function* runSourcesWithProgress(state, sources) {
+    let identityFailure;
+    for (const source of sources) {
+        if (cancellationRequested(state) || identityFailure !== undefined) {
+            break;
+        }
+        identityFailure = yield* iterateSourceEvents(state, source);
+        persistJobProgress(state);
+    }
+    return identityFailure;
+}
+// ─── Public entrypoint ────────────────────────────────────────────────────────
+export async function* runIndexingJob(options) {
+    const capsule = resolveCapsule(options);
+    const sources = resolveSources(options, capsule);
+    const startedAt = (options.now ?? options.store._internal.now)();
+    const idSource = options.idSource ?? (() => randomUUID());
+    const jobId = idSource();
+    const state = buildInitialState(options, capsule, sources, jobId, startedAt);
+    persistStartedJob(state, sources);
+    yield emitJobStarted(state, sources);
+    if (cancellationRequested(state)) {
+        yield* finalize(state, undefined);
+        return;
+    }
+    const preflightFailure = await verifyEmbeddingPreflight(state);
+    if (cancellationRequested(state)) {
+        yield* finalize(state, undefined);
+        return;
+    }
+    if (preflightFailure !== undefined) {
+        state.lastError = preflightFailure;
+        yield* finalize(state, preflightFailure);
+        return;
+    }
+    const identityFailure = yield* runSourcesWithProgress(state, sources);
+    yield* finalize(state, identityFailure);
+}
+// Drains one source's event stream, yielding each event to the outer generator.
+// Returns the identity-failure error if encountered, undefined otherwise.
+async function* iterateSourceEvents(state, source) {
+    for await (const evt of runOneSource(state, source)) {
+        yield emit(state, evt);
+        if (evt.kind === "document-failed" && evt.error.code === "INCOMPATIBLE_EMBEDDING_IDENTITY") {
+            return evt.error;
+        }
+    }
+    return undefined;
+}
+function emit(state, event) {
+    emitProgress(state.options, event);
+    return event;
+}
+function resolveJobStatus(state, fatalFailure) {
+    if (fatalFailure !== undefined) {
+        state.lastError = fatalFailure;
+        return "failed";
+    }
+    if (cancellationRequested(state))
+        return "cancelled";
+    if (state.failedDocuments > 0 && state.processedDocuments === 0)
+        return "failed";
+    return "succeeded";
+}
+function* finalize(state, fatalFailure) {
+    const finishedAt = state.now();
+    const status = resolveJobStatus(state, fatalFailure);
+    finalizeJobRow(state.options.store._internal.db, {
+        id: state.jobId,
+        status,
+        finishedAt,
+        counters: buildCounters(state),
+        ...(state.lastError !== undefined ? { lastError: state.lastError } : {}),
+    });
+    try {
+        updateCapsuleState(state.options.store, state.capsule.id, status === "succeeded" ? "ready" : "error");
+    }
+    catch {
+        // informational only — see the started block for the rationale
+    }
+    const result = buildResult(state, status, finishedAt);
+    if (status === "cancelled") {
+        yield emit(state, { kind: "job-cancelled", jobId: state.jobId, result });
+        return;
+    }
+    if (status === "failed") {
+        const err = state.lastError ?? { code: "EMBEDDING_ADAPTER_FAILED", message: "indexing failed" };
+        state.options.auditSink?.emit({
+            kind: "indexing-job-failed",
+            capsuleId: state.capsule.id,
+            sourceIds: sourceIdsForState(state),
+            jobId: state.jobId,
+            errorCode: err.code,
+            occurredAt: finishedAt,
+        });
+        yield emit(state, { kind: "job-failed", jobId: state.jobId, error: err, result });
+        return;
+    }
+    state.options.auditSink?.emit({
+        kind: "indexing-job-completed",
+        capsuleId: state.capsule.id,
+        sourceIds: sourceIdsForState(state),
+        jobId: state.jobId,
+        processedDocuments: result.processedDocuments,
+        failedDocuments: result.failedDocuments,
+        occurredAt: finishedAt,
+    });
+    yield emit(state, { kind: "job-completed", jobId: state.jobId, result });
+}