@oscharko-dev/keiko 0.2.0-beta.3 → 0.2.0-beta.5

package/node_modules/@oscharko-dev/keiko-local-knowledge/dist/indexing/orchestrator.js

@@ -18,22 +18,25 @@
 // already persisted for completed documents are kept (the source-of-truth for resume is
 // the chunks/vectors tables, not the in-flight buffer).
 //
-// Force mode: deletes the capsule's pre-existing vector rows up front, then passes
-// `force=true` into the chunker so chunks are also re-emitted. Discovery's incremental
-// fast-path is bypassed by re-reading the bytes (the discovery layer already does the
-// content-hash compare for us — when force=true the orchestrator deletes the vectors but
-// the documents row stays valid, so chunk-and-embed still re-runs).
+// Force mode: passes `force=true` into the chunker per document so existing chunks are
+// replaced from the current source text, then re-embeds. Discovery's incremental
+// fast-path is bypassed (the skipped outcome is re-shaped to persisted in
+// handleFileExtracted) so chunk-and-embed re-runs even for unchanged file hashes.
+// Recovery mode (partial vector coverage, non-force): re-embeds using existing chunks
+// only — the chunker runs with force=false so it reuses the already-correct chunk rows.
 import { randomUUID } from "node:crypto";
-import { verifyEmbeddingCapability } from "@oscharko-dev/keiko-model-gateway";
+import { assertCompatibleEmbeddingIdentity, verifyEmbeddingCapability, } from "@oscharko-dev/keiko-model-gateway";
 import { chunkDocument } from "../chunking/chunker-runner.js";
-import { deleteChunksForDocument, hasStaleChunksForDocument } from "../chunking/chunker-persist.js";
+import { countChunksForDocument, deleteChunksForDocument, hasStaleChunksForDocument, } from "../chunking/chunker-persist.js";
+import { chunkingStrategyKey } from "../chunking/index.js";
 import { getCapsule, updateCapsuleState } from "../capsule-lifecycle.js";
 import { discoverAndExtract } from "../discovery/discovery-runner.js";
+import { DEFAULT_DISCOVERY_OPTIONS } from "../discovery/index.js";
 import { deleteDocumentRow, listPersistedDocumentsForSource, readDocumentTextRow, updateDocumentStatusRow, } from "../discovery/persist.js";
 import { listCapsuleSources } from "../source-lifecycle.js";
 import { finalizeJobRow, isJobCancellationRequested, insertJobRow, updateJobCounters, } from "./job-persist.js";
 import { embedChunkBatch } from "./embedding-batcher.js";
-import { countVectorsForDocument, deleteVectorsForCapsule, deleteVectorsForDocument, selectChunksForDocument, } from "./vector-persist.js";
+import { countVectorsForDocument, deleteVectorsForDocument, selectChunksForDocument, } from "./vector-persist.js";
 import { DEFAULT_INDEXING_BATCH_SIZE, DEFAULT_INDEXING_CONCURRENCY, IndexingError, } from "./types.js";
 // ─── Abort helper ─────────────────────────────────────────────────────────────
 // Reads `signal?.aborted` through a function call so TypeScript's control-flow analysis
@@ -55,6 +58,20 @@ function clampConcurrency(raw) {
     const v = raw ?? DEFAULT_INDEXING_CONCURRENCY;
     return Math.max(1, Math.min(DEFAULT_INDEXING_CONCURRENCY, Math.floor(v)));
 }
+function clampDiscoveryInteger(raw, fallback) {
+    if (raw === undefined || !Number.isFinite(raw))
+        return fallback;
+    return Math.max(1, Math.min(fallback, Math.floor(raw)));
+}
+function resolvedDiscoveryOptions(state) {
+    const raw = state.options.discoveryOptions;
+    const base = {
+        maxDepth: clampDiscoveryInteger(raw?.maxDepth, DEFAULT_DISCOVERY_OPTIONS.maxDepth),
+        maxFiles: clampDiscoveryInteger(raw?.maxFiles, DEFAULT_DISCOVERY_OPTIONS.maxFiles),
+    };
+    const signal = raw?.signal ?? state.options.signal;
+    return signal === undefined ? base : { ...base, signal };
+}
 // ─── Source resolution ────────────────────────────────────────────────────────
 function resolveSources(options, capsule) {
     const all = listCapsuleSources(options.store, capsule.id);
@@ -64,7 +81,14 @@ function resolveSources(options, capsule) {
     if (options.sourceIds === undefined)
         return all;
     const allow = new Set(options.sourceIds.map((s) => String(s)));
-    return all.filter((s) => allow.has(String(s.id)));
+    if (allow.size === 0) {
+        throw new IndexingError("INVALID_OPTIONS", "sourceIds must contain at least one source id.");
+    }
+    const selected = all.filter((s) => allow.has(String(s.id)));
+    if (selected.length !== allow.size) {
+        throw new IndexingError("INVALID_OPTIONS", "sourceIds must reference sources attached to the target capsule.");
+    }
+    return selected;
 }
 function buildCounters(state) {
     return {
@@ -103,7 +127,8 @@ function markDocumentFailed(state, documentId) {
 }
 const SELECT_CHUNKS_WITH_OFFSETS_SQL = [
     "SELECT c.id, c.capsule_id, c.source_id, c.document_id, c.parsed_unit_id, c.order_index,",
-    "  pu.character_start AS char_start, pu.character_end AS char_end",
+    "  COALESCE(c.character_start, pu.character_start) AS char_start,",
+    "  COALESCE(c.character_end, pu.character_end) AS char_end",
     "FROM chunks AS c",
     "JOIN parsed_units AS pu ON pu.capsule_id = c.capsule_id AND pu.id = c.parsed_unit_id",
     "WHERE c.capsule_id = :c AND c.document_id = :d",
@@ -164,12 +189,17 @@ function readSourceText(state, source, relativePath) {
         real = state.options.workspaceFs.realPath(abs);
     }
     catch (cause) {
-        throw new IndexingError("PERSISTENCE_FAILED", `source realPath failed before embedding: ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
+        throw new IndexingError("PERSISTENCE_FAILED", "source text could not be read before embedding", { cause });
     }
     if (!isContained(absoluteRoot, real)) {
         throw new IndexingError("PERSISTENCE_FAILED", `source realpath escapes scope root before embedding: ${relativePath}`);
     }
-    return state.options.workspaceFs.readFileUtf8(normaliseSep(real));
+    try {
+        return state.options.workspaceFs.readFileUtf8(normaliseSep(real));
+    }
+    catch (cause) {
+        throw new IndexingError("PERSISTENCE_FAILED", "source text could not be read before embedding", { cause });
+    }
 }
 function resolveChunkSourceText(state, documentId, source, relativePath) {
     const persistedText = readDocumentTextRow(state.options.store._internal.db, state.capsule.id, documentId);
@@ -188,6 +218,14 @@ function sliceIntoBatches(items, batchSize) {
     }
     return out;
 }
+function cancellationError() {
+    return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
+}
+function recordCancellationIfRequested(state, errors) {
+    if (cancellationRequested(state) && !errors.some((error) => error.code === "CANCELLED")) {
+        errors.push(cancellationError());
+    }
+}
 async function embedDocumentChunks(state, documentId, source, relativePath) {
     // Text-like documents are re-read from disk; binary parsers persist a normalized text
     // projection so chunk slicing stays aligned with extracted content.
@@ -227,6 +265,7 @@ async function embedDocumentChunks(state, documentId, source, relativePath) {
         if (cancellationRequested(state))
             break;
     }
+    recordCancellationIfRequested(state, errors);
     return { vectorCount, errors, lastChunkId };
 }
 // ─── Document handlers ────────────────────────────────────────────────────────
@@ -235,6 +274,8 @@ function handleExtractionSkipped(state, result) {
     return {
         kind: "document-skipped",
         jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
         documentId: result.outcome.kind === "skipped" ? result.outcome.document.id : "",
         reason: "unchanged",
     };
@@ -252,6 +293,8 @@ function handleExtractionFailed(state, result) {
     return {
         kind: "document-failed",
         jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId: result.sourceId,
         ...(result.outcome.kind === "failed" ? { documentId: result.outcome.document.id } : {}),
         relativePath: result.relativePath,
         error,
@@ -265,11 +308,24 @@ function resolveChunkCount(state, documentId, skippedExisting, freshChunkIds) {
     return selectChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId)
         .length;
 }
-function chunkPersistedDocument(state, result) {
+function embeddingCoverage(state, documentId) {
+    return {
+        chunkCount: countChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId),
+        vectorCount: countVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId),
+    };
+}
+function hasCompleteVectorCoverage(state, documentId) {
+    const coverage = embeddingCoverage(state, documentId);
+    return coverage.chunkCount > 0 && coverage.vectorCount === coverage.chunkCount;
+}
+function persistedDocumentId(result) {
     if (result.outcome.kind !== "persisted") {
         throw new IndexingError("INVALID_OPTIONS", "chunkPersistedDocument called with non-persisted result");
     }
-    const documentId = result.outcome.document.id;
+    return result.outcome.document.id;
+}
+function chunkPersistedDocument(state, result) {
+    const documentId = persistedDocumentId(result);
     const sourceText = resolveChunkSourceText(state, documentId, sourceForResult(state, result), result.relativePath);
     const chunkResult = chunkDocument(state.options.store, {
         capsuleId: state.capsule.id,
@@ -281,15 +337,29 @@ function chunkPersistedDocument(state, result) {
     }, state.options.chunkingOptions);
     const chunkCount = resolveChunkCount(state, documentId, chunkResult.skippedExisting, chunkResult.chunkIds);
     return {
-        events: chunkedDocumentEvents(state.jobId, documentId, result.relativePath, chunkCount),
+        events: chunkedDocumentEvents(state, result.sourceId, documentId, result.relativePath, chunkCount),
         documentId,
         chunkCount,
     };
 }
-function chunkedDocumentEvents(jobId, documentId, relativePath, chunkCount) {
+function chunkedDocumentEvents(state, sourceId, documentId, relativePath, chunkCount) {
     return [
-        { kind: "document-extracted", jobId, documentId, relativePath },
-        { kind: "document-chunked", jobId, documentId, chunkCount },
+        {
+            kind: "document-extracted",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId,
+            documentId,
+            relativePath,
+        },
+        {
+            kind: "document-chunked",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId,
+            documentId,
+            chunkCount,
+        },
     ];
 }
 function sourceForResult(state, result) {
@@ -305,27 +375,30 @@ function sourceForResult(state, result) {
 // Incremental fast-path: skips embedding when vectors already exist (non-force run), or
 // deletes prior vectors to prepare for a forced re-embed.
 // Returns a PersistedHandling to short-circuit when already-embedded, undefined to continue.
-function applyIncrementalFastPath(state, documentId) {
-    const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId);
+function applyIncrementalFastPath(state, sourceId, documentId) {
+    const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, documentId, chunkingStrategyKey(state.options.chunkingOptions));
     if (state.options.force !== true) {
-        // Incremental fast-path #2: if vectors already exist for this document AND not in force
-        // mode, skip the embedding step entirely. The chunker is also a no-op in this case.
-        const existing = countVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
-        if (existing > 0 && !staleChunks) {
+        const coverage = embeddingCoverage(state, documentId);
+        if (coverage.chunkCount > 0 && coverage.vectorCount === coverage.chunkCount && !staleChunks) {
             state.skippedDocuments += 1;
             return {
                 events: [
-                    { kind: "document-skipped", jobId: state.jobId, documentId, reason: "already-embedded" },
+                    {
+                        kind: "document-skipped",
+                        jobId: state.jobId,
+                        capsuleId: state.capsule.id,
+                        sourceId,
+                        documentId,
+                        reason: "already-embedded",
+                    },
                 ],
             };
         }
-        if (existing > 0 && staleChunks) {
+        if (coverage.vectorCount > 0) {
             deleteVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
         }
         return undefined;
     }
-    // Force mode: tear down prior vectors so the re-embed is the only surviving set.
-    deleteVectorsForDocument(state.options.store._internal.db, state.capsule.id, documentId);
     return undefined;
 }
 // Runs the chunker and returns its result, or a PersistedHandling failure event on throw.
@@ -333,7 +406,7 @@ function tryChunkDocument(state, result, documentId) {
     try {
         return { chunked: chunkPersistedDocument(state, result) };
     }
-    catch (cause) {
+    catch {
         if (cancellationRequested(state)) {
             clearDocumentArtifacts(state, documentId, { deleteChunks: true });
             return { events: [] };
@@ -343,7 +416,7 @@ function tryChunkDocument(state, result, documentId) {
         markDocumentFailed(state, documentId);
         const error = {
             code: "CHUNKING_FAILED",
-            message: cause instanceof Error ? cause.message : String(cause),
+            message: "document chunking failed",
         };
         state.lastError = error;
         return {
@@ -351,6 +424,8 @@ function tryChunkDocument(state, result, documentId) {
                 {
                     kind: "document-failed",
                     jobId: state.jobId,
+                    capsuleId: state.capsule.id,
+                    sourceId: result.sourceId,
                     documentId,
                     relativePath: result.relativePath,
                     error,
@@ -359,7 +434,7 @@ function tryChunkDocument(state, result, documentId) {
         };
     }
 }
-function appendDocumentFailure(state, events, documentId, relativePath, error, options) {
+function appendDocumentFailure(state, events, sourceId, documentId, relativePath, error, options) {
     state.failedDocuments += 1;
     clearDocumentArtifacts(state, documentId, options);
     markDocumentFailed(state, documentId);
@@ -367,13 +442,15 @@ function appendDocumentFailure(state, events, documentId, relativePath, error, o
     events.push({
         kind: "document-failed",
         jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId,
         documentId,
         relativePath,
         error,
     });
     return { events };
 }
-function completeEmbeddedDocument(state, events, documentId, embedResult) {
+function completeEmbeddedDocument(state, events, sourceId, documentId, embedResult) {
     state.processedDocuments += 1;
     state.vectorsPersisted += embedResult.vectorCount;
     if (embedResult.lastChunkId !== null)
@@ -381,6 +458,8 @@ function completeEmbeddedDocument(state, events, documentId, embedResult) {
     events.push({
         kind: "document-embedded",
         jobId: state.jobId,
+        capsuleId: state.capsule.id,
+        sourceId,
         documentId,
         vectorCount: embedResult.vectorCount,
         resumeToken: embedResult.lastChunkId ?? `${String(documentId)}#empty`,
@@ -393,16 +472,15 @@ function isCancellationOnlyEmbedResult(state, embedResult) {
         embedResult.errors.every((error) => error.code === "CANCELLED"));
 }
 // Maps an EmbedDocumentResult into PersistedHandling events, mutating run-state counters.
-function applyEmbedResult(state, documentId, relativePath, priorEvents, embedResult) {
+function applyEmbedResult(state, sourceId, documentId, relativePath, priorEvents, embedResult) {
     const events = [...priorEvents];
     if (isCancellationOnlyEmbedResult(state, embedResult)) {
-        clearDocumentArtifacts(state, documentId, { deleteChunks: false });
         return { events };
     }
     const identityErr = embedResult.errors.find((e) => e.code === "INCOMPATIBLE_EMBEDDING_IDENTITY");
     if (identityErr !== undefined) {
         return {
-            ...appendDocumentFailure(state, events, documentId, relativePath, identityErr, {
+            ...appendDocumentFailure(state, events, sourceId, documentId, relativePath, identityErr, {
                 deleteChunks: false,
             }),
             identityFailure: identityErr,
@@ -413,89 +491,78 @@ function applyEmbedResult(state, documentId, relativePath, priorEvents, embedRes
             code: "EMBEDDING_ADAPTER_FAILED",
             message: "embedding adapter failed",
         };
-        return appendDocumentFailure(state, events, documentId, relativePath, firstErr, {
+        return appendDocumentFailure(state, events, sourceId, documentId, relativePath, firstErr, {
             deleteChunks: false,
         });
     }
-    return completeEmbeddedDocument(state, events, documentId, embedResult);
+    return completeEmbeddedDocument(state, events, sourceId, documentId, embedResult);
 }
-// Wraps the chunk-then-embed pipeline for a single persisted document.
-async function handlePersistedDocument(state, result) {
+function* persistedEvents(handling) {
+    for (const event of handling.events) {
+        yield event;
+    }
+}
+// Wraps the chunk-then-embed pipeline for a single persisted document. Extraction/chunking
+// events are yielded before awaiting embeddings, so progress consumers see pre-model work
+// immediately instead of only after all embedding batches finish.
+async function* handlePersistedDocument(state, result) {
     const documentId = result.outcome.kind === "persisted" ? result.outcome.document.id : null;
     if (documentId === null)
-        return { events: [] };
+        return;
     if (result.outcome.document.status === "unsupported") {
         clearDocumentArtifacts(state, documentId, { deleteChunks: true });
         state.skippedDocuments += 1;
-        return {
-            events: [
-                {
-                    kind: "document-extracted",
-                    jobId: state.jobId,
-                    documentId,
-                    relativePath: result.relativePath,
-                },
-                { kind: "document-skipped", jobId: state.jobId, documentId, reason: "unsupported" },
-            ],
+        yield {
+            kind: "document-extracted",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: result.sourceId,
+            documentId,
+            relativePath: result.relativePath,
+        };
+        yield {
+            kind: "document-skipped",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: result.sourceId,
+            documentId,
+            reason: "unsupported",
         };
+        return;
+    }
+    const fastPath = applyIncrementalFastPath(state, result.sourceId, documentId);
+    if (fastPath !== undefined) {
+        yield* persistedEvents(fastPath);
+        return;
     }
-    const fastPath = applyIncrementalFastPath(state, documentId);
-    if (fastPath !== undefined)
-        return fastPath;
     const chunkStep = tryChunkDocument(state, result, documentId);
-    if (!("chunked" in chunkStep))
-        return chunkStep;
+    if (!("chunked" in chunkStep)) {
+        yield* persistedEvents(chunkStep);
+        return;
+    }
+    yield* chunkStep.chunked.events;
+    persistJobProgress(state);
     const embedResult = await embedDocumentChunks(state, documentId, sourceForResult(state, result), result.relativePath);
-    return applyEmbedResult(state, documentId, result.relativePath, chunkStep.chunked.events, embedResult);
+    yield* persistedEvents(applyEmbedResult(state, result.sourceId, documentId, result.relativePath, [], embedResult));
 }
-// ─── Per-source pipeline ──────────────────────────────────────────────────────
-async function* runOneSource(state, source) {
-    const stream = discoverAndExtract({
-        fs: state.options.workspaceFs,
-        store: state.options.store,
-        parserRegistry: state.options.parserRegistry,
-    }, sourceDiscoveryParams(state, source));
-    const progress = {
-        cancelled: false,
-        sawScopeError: false,
-        completed: false,
-        discoveredPaths: new Set(),
-    };
-    for await (const evt of stream) {
-        observeSourceEvent(progress, evt);
-        if (cancellationRequested(state)) {
-            progress.cancelled = true;
-            break;
-        }
-        const events = await handleDiscoveryEvent(state, evt);
-        if (events.length > 0) {
-            persistJobProgress(state);
-        }
-        for (const e of events) {
-            yield e;
-            if (e.kind === "document-failed" && e.error.code === "INCOMPATIBLE_EMBEDDING_IDENTITY") {
-                return;
-            }
-        }
-        // After yielding a batch we re-check the signal — the consumer's awaiting iterator
-        // may have aborted between events.
-        if (cancellationRequested(state)) {
-            progress.cancelled = true;
-            break;
-        }
-    }
-    finalizeSourceRun(state, source, progress);
+function* handleExtractionSkippedEvents(state, result) {
+    yield handleExtractionSkipped(state, result);
+}
+function* handleExtractionFailedEvents(state, result) {
+    yield handleExtractionFailed(state, result);
 }
 // Routes a file-extracted event: force-skipped docs are re-shaped to persisted so the
 // standard chunk-and-embed pipeline runs on them.
-async function handleFileExtracted(state, result) {
+async function* handleFileExtracted(state, result) {
     if (result.outcome.kind === "skipped") {
         // In force mode, an "unchanged" document still needs chunk-and-embed because the
-        // orchestrator deleted the vector rows at job-started. Re-shape the skipped outcome
-        // as a persisted outcome (the document row exists and is valid) so the standard
-        // pipeline runs. Outside force mode, surface the skip as-is.
-        const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, result.outcome.document.id);
-        if (state.options.force === true || staleChunks) {
+        // caller explicitly requested a fresh embedding pass. Re-shape the skipped outcome as
+        // a persisted outcome (the document row exists and is valid) so the standard pipeline
+        // runs. Outside force/recovery mode, surface the skip as-is.
+        const staleChunks = hasStaleChunksForDocument(state.options.store._internal.db, state.capsule.id, result.outcome.document.id, chunkingStrategyKey(state.options.chunkingOptions));
+        const missingVectors = result.outcome.document.status === "extracted" &&
+            !hasCompleteVectorCoverage(state, result.outcome.document.id);
+        if (state.options.force === true || staleChunks || missingVectors) {
             const synthetic = {
                 capsuleId: result.capsuleId,
                 sourceId: result.sourceId,
@@ -503,28 +570,30 @@ async function handleFileExtracted(state, result) {
                 outcome: { kind: "persisted", document: result.outcome.document },
                 diagnostics: result.diagnostics,
             };
-            const handled = await handlePersistedDocument(state, synthetic);
-            return handled.events;
+            yield* handlePersistedDocument(state, synthetic);
+            return;
         }
-        return [handleExtractionSkipped(state, result)];
+        yield* handleExtractionSkippedEvents(state, result);
+        return;
     }
     if (result.outcome.kind === "failed") {
-        return [handleExtractionFailed(state, result)];
+        yield* handleExtractionFailedEvents(state, result);
+        return;
     }
-    const handled = await handlePersistedDocument(state, result);
-    return handled.events;
+    yield* handlePersistedDocument(state, result);
 }
-async function handleDiscoveryEvent(state, evt) {
+async function* handleDiscoveryEvent(state, source, evt) {
     if (evt.kind === "file-discovered") {
         state.totalDocuments += 1;
-        return [
-            {
-                kind: "document-discovered",
-                jobId: state.jobId,
-                relativePath: evt.relativePath,
-                sizeBytes: evt.sizeBytes,
-            },
-        ];
+        yield {
+            kind: "document-discovered",
+            jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: source.id,
+            relativePath: evt.relativePath,
+            sizeBytes: evt.sizeBytes,
+        };
+        return;
     }
     if (evt.kind === "scope-error") {
         state.failedDocuments += 1;
@@ -533,30 +602,73 @@ async function handleDiscoveryEvent(state, evt) {
             message: evt.error.message,
         };
         state.lastError = err;
-        const failed = {
+        yield {
             kind: "document-failed",
             jobId: state.jobId,
+            capsuleId: state.capsule.id,
+            sourceId: source.id,
             ...(evt.error.relativePath !== undefined ? { relativePath: evt.error.relativePath } : {}),
             error: err,
         };
-        return [failed];
+        return;
     }
     if (evt.kind === "cancelled" || evt.kind === "completed") {
         // No-op at this level: the outer loop drives terminal events.
-        return [];
+        return;
     }
     // evt.kind === "file-extracted"
-    return handleFileExtracted(state, evt.result);
+    yield* handleFileExtracted(state, evt.result);
+}
+function shouldStopAfterEvent(event) {
+    return event.kind === "document-failed" && event.error.code === "INCOMPATIBLE_EMBEDDING_IDENTITY";
+}
+async function* streamDiscoveryEvent(state, source, evt) {
+    for await (const event of handleDiscoveryEvent(state, source, evt)) {
+        persistJobProgress(state);
+        yield event;
+        if (shouldStopAfterEvent(event)) {
+            return true;
+        }
+    }
+    return false;
+}
+// ─── Per-source pipeline ──────────────────────────────────────────────────────
+async function* runOneSource(state, source) {
+    const stream = discoverAndExtract({
+        fs: state.options.workspaceFs,
+        store: state.options.store,
+        parserRegistry: state.options.parserRegistry,
+    }, sourceDiscoveryParams(state, source));
+    const progress = {
+        cancelled: false,
+        sawScopeError: false,
+        completed: false,
+        discoveredPaths: new Set(),
+    };
+    for await (const evt of stream) {
+        observeSourceEvent(progress, evt);
+        if (cancellationRequested(state)) {
+            progress.cancelled = true;
+            break;
+        }
+        const shouldStop = yield* streamDiscoveryEvent(state, source, evt);
+        if (shouldStop) {
+            return;
+        }
+        // After yielding a batch we re-check the signal — the consumer's awaiting iterator
+        // may have aborted between events.
+        if (cancellationRequested(state)) {
+            progress.cancelled = true;
+            break;
+        }
+    }
+    finalizeSourceRun(state, source, progress);
 }
 function sourceDiscoveryParams(state, source) {
     return {
         capsuleId: state.capsule.id,
         source,
-        ...(state.options.discoveryOptions !== undefined
-            ? { discovery: state.options.discoveryOptions }
-            : state.options.signal !== undefined
-                ? { discovery: { maxDepth: 12, maxFiles: 5_000, signal: state.options.signal } }
-                : {}),
+        discovery: resolvedDiscoveryOptions(state),
     };
 }
 function observeSourceEvent(progress, evt) {
@@ -589,6 +701,8 @@ function finalizeSourceRun(state, source, progress) {
         return;
     if (!progress.completed || progress.sawScopeError)
         return;
+    if (progress.discoveredPaths.size >= resolvedDiscoveryOptions(state).maxFiles)
+        return;
     pruneDeletedSourceDocuments(state, source, progress.discoveredPaths);
 }
 // ─── Capsule resolution + job lifecycle ───────────────────────────────────────
@@ -646,8 +760,15 @@ async function verifyEmbeddingPreflight(state) {
             expectedDimensions: state.capsule.embeddingModelIdentity.vectorDimensions,
             ...(state.options.signal !== undefined ? { signal: state.options.signal } : {}),
         });
-        if (result.ok)
-            return undefined;
+        if (result.ok) {
+            const compatibility = assertCompatibleEmbeddingIdentity(state.capsule.embeddingModelIdentity, result.identity);
+            if (compatibility.ok)
+                return undefined;
+            return {
+                code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
+                message: compatibility.safeMessage,
+            };
+        }
         return {
             code: result.reason === "dimension-mismatch"
                 ? "INCOMPATIBLE_EMBEDDING_IDENTITY"
@@ -662,7 +783,7 @@ async function verifyEmbeddingPreflight(state) {
         }
         return {
             code: "EMBEDDING_ADAPTER_FAILED",
-            message: cause instanceof Error ? cause.message : String(cause),
+            message: "embedding capability preflight failed before indexing started",
         };
     }
 }
@@ -681,6 +802,9 @@ function persistStartedJob(state, sources) {
         // run. The events stream remains the source of truth.
     }
 }
+function sourceIdsForState(state) {
+    return [...state.sourcesById.values()].map((source) => source.id);
+}
 function emitJobStarted(state, sources) {
     const event = {
         kind: "job-started",
@@ -692,6 +816,7 @@ function emitJobStarted(state, sources) {
     state.options.auditSink?.emit({
         kind: "indexing-job-started",
         capsuleId: state.capsule.id,
+        sourceIds: sources.map((source) => source.id),
         jobId: state.jobId,
         occurredAt: state.startedAt,
     });
@@ -732,11 +857,6 @@ export async function* runIndexingJob(options) {
         yield* finalize(state, preflightFailure);
         return;
     }
-    // Force mode: tear down ALL vectors for the capsule up front. Per-document teardown
-    // still runs in handlePersistedDocument as a defence-in-depth measure.
-    if (options.force === true) {
-        deleteVectorsForCapsule(state.options.store._internal.db, capsule.id);
-    }
     const identityFailure = yield* runSourcesWithProgress(state, sources);
     yield* finalize(state, identityFailure);
 }
@@ -792,6 +912,7 @@ function* finalize(state, fatalFailure) {
         state.options.auditSink?.emit({
             kind: "indexing-job-failed",
             capsuleId: state.capsule.id,
+            sourceIds: sourceIdsForState(state),
             jobId: state.jobId,
             errorCode: err.code,
             occurredAt: finishedAt,
@@ -802,6 +923,7 @@ function* finalize(state, fatalFailure) {
     state.options.auditSink?.emit({
         kind: "indexing-job-completed",
         capsuleId: state.capsule.id,
+        sourceIds: sourceIdsForState(state),
         jobId: state.jobId,
         processedDocuments: result.processedDocuments,
         failedDocuments: result.failedDocuments,