@oscharko-dev/keiko-local-knowledge 0.2.0

package/dist/indexing/checkpoint-persist.js

@@ -0,0 +1,135 @@
+// Prepared-statement helpers for the `extraction_checkpoints` table (Epic #1160, Issue #1286).
+//
+// One row per (capsule_id, document_id). The progressive large-document path upserts the row as a
+// document advances through extraction → chunking → embedding so an interrupted job resumes from
+// durable progress. The row is content-free: hashes, cursors, counts, and redacted diagnostics —
+// never raw extracted text. The compatibility fingerprint columns let a resumed run refuse a
+// checkpoint produced under an incompatible source, parser, policy, chunking strategy, or
+// embedding identity.
+const UPSERT_SQL = [
+    "INSERT INTO extraction_checkpoints (",
+    "  capsule_id, document_id, job_id, strategy, phase, page_cursor, section_cursor,",
+    "  object_cursor, extracted_text_bytes, chunk_cursor, embedded_chunk_cursor,",
+    "  last_embedded_chunk_id, retry_count, coverage, source_content_hash, parser_version,",
+    "  policy_fingerprint, chunking_strategy_version, embedding_identity_json,",
+    "  terminal_diagnostics_json, created_at, updated_at",
+    ") VALUES (",
+    "  :capsule_id, :document_id, :job_id, :strategy, :phase, :page_cursor, :section_cursor,",
+    "  :object_cursor, :extracted_text_bytes, :chunk_cursor, :embedded_chunk_cursor,",
+    "  :last_embedded_chunk_id, :retry_count, :coverage, :source_content_hash, :parser_version,",
+    "  :policy_fingerprint, :chunking_strategy_version, :embedding_identity_json,",
+    "  :terminal_diagnostics_json, :created_at, :updated_at",
+    ") ON CONFLICT(capsule_id, document_id) DO UPDATE SET",
+    "  job_id = excluded.job_id,",
+    "  strategy = excluded.strategy,",
+    "  phase = excluded.phase,",
+    "  page_cursor = excluded.page_cursor,",
+    "  section_cursor = excluded.section_cursor,",
+    "  object_cursor = excluded.object_cursor,",
+    "  extracted_text_bytes = excluded.extracted_text_bytes,",
+    "  chunk_cursor = excluded.chunk_cursor,",
+    "  embedded_chunk_cursor = excluded.embedded_chunk_cursor,",
+    "  last_embedded_chunk_id = excluded.last_embedded_chunk_id,",
+    "  retry_count = excluded.retry_count,",
+    "  coverage = excluded.coverage,",
+    "  source_content_hash = excluded.source_content_hash,",
+    "  parser_version = excluded.parser_version,",
+    "  policy_fingerprint = excluded.policy_fingerprint,",
+    "  chunking_strategy_version = excluded.chunking_strategy_version,",
+    "  embedding_identity_json = excluded.embedding_identity_json,",
+    "  terminal_diagnostics_json = excluded.terminal_diagnostics_json,",
+    "  updated_at = excluded.updated_at",
+].join(" ");
+const SELECT_ONE_SQL = [
+    "SELECT * FROM extraction_checkpoints",
+    "WHERE capsule_id = :c AND document_id = :d",
+].join(" ");
+const SELECT_BY_CAPSULE_SQL = [
+    "SELECT * FROM extraction_checkpoints",
+    "WHERE capsule_id = :c",
+    "ORDER BY updated_at DESC, document_id ASC",
+].join(" ");
+const DELETE_ONE_SQL = "DELETE FROM extraction_checkpoints WHERE capsule_id = :c AND document_id = :d";
+function diagnosticsToJson(diagnostics) {
+    return JSON.stringify(diagnostics);
+}
+export function upsertExtractionCheckpoint(db, checkpoint) {
+    db.prepare(UPSERT_SQL).run({
+        capsule_id: String(checkpoint.capsuleId),
+        document_id: String(checkpoint.documentId),
+        job_id: checkpoint.jobId,
+        strategy: checkpoint.strategy,
+        phase: checkpoint.phase,
+        page_cursor: checkpoint.pageCursor,
+        section_cursor: checkpoint.sectionCursor,
+        object_cursor: checkpoint.objectCursor,
+        extracted_text_bytes: checkpoint.extractedTextBytes,
+        chunk_cursor: checkpoint.chunkCursor,
+        embedded_chunk_cursor: checkpoint.embeddedChunkCursor,
+        last_embedded_chunk_id: checkpoint.lastEmbeddedChunkId === undefined ? null : String(checkpoint.lastEmbeddedChunkId),
+        retry_count: checkpoint.retryCount,
+        coverage: checkpoint.coverage,
+        source_content_hash: checkpoint.fingerprint.sourceContentHash,
+        parser_version: checkpoint.fingerprint.parserVersion,
+        policy_fingerprint: checkpoint.fingerprint.policyFingerprint,
+        chunking_strategy_version: checkpoint.fingerprint.chunkingStrategyVersion,
+        embedding_identity_json: JSON.stringify(checkpoint.fingerprint.embeddingIdentity),
+        terminal_diagnostics_json: diagnosticsToJson(checkpoint.terminalDiagnostics),
+        created_at: checkpoint.createdAt,
+        updated_at: checkpoint.updatedAt,
+    });
+}
+function parseDiagnostics(json) {
+    try {
+        const parsed = JSON.parse(json);
+        return Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return [];
+    }
+}
+function parseEmbeddingIdentity(json) {
+    const parsed = JSON.parse(json);
+    return parsed;
+}
+function rowToCheckpoint(row) {
+    return {
+        capsuleId: row.capsule_id,
+        documentId: row.document_id,
+        jobId: row.job_id,
+        strategy: row.strategy,
+        phase: row.phase,
+        pageCursor: row.page_cursor,
+        sectionCursor: row.section_cursor,
+        objectCursor: row.object_cursor,
+        extractedTextBytes: row.extracted_text_bytes,
+        chunkCursor: row.chunk_cursor,
+        embeddedChunkCursor: row.embedded_chunk_cursor,
+        ...(row.last_embedded_chunk_id === null
+            ? {}
+            : { lastEmbeddedChunkId: row.last_embedded_chunk_id }),
+        retryCount: row.retry_count,
+        coverage: row.coverage,
+        fingerprint: {
+            sourceContentHash: row.source_content_hash,
+            parserVersion: row.parser_version,
+            policyFingerprint: row.policy_fingerprint,
+            chunkingStrategyVersion: row.chunking_strategy_version,
+            embeddingIdentity: parseEmbeddingIdentity(row.embedding_identity_json),
+        },
+        terminalDiagnostics: parseDiagnostics(row.terminal_diagnostics_json),
+        createdAt: row.created_at,
+        updatedAt: row.updated_at,
+    };
+}
+export function selectExtractionCheckpoint(db, capsuleId, documentId) {
+    const row = db.prepare(SELECT_ONE_SQL).get({ c: String(capsuleId), d: String(documentId) });
+    return row === undefined ? undefined : rowToCheckpoint(row);
+}
+export function listExtractionCheckpoints(db, capsuleId) {
+    const rows = db.prepare(SELECT_BY_CAPSULE_SQL).all({ c: String(capsuleId) });
+    return rows.map((row) => rowToCheckpoint(row));
+}
+export function deleteExtractionCheckpoint(db, capsuleId, documentId) {
+    db.prepare(DELETE_ONE_SQL).run({ c: String(capsuleId), d: String(documentId) });
+}

package/dist/indexing/checkpoint-resume.d.ts

@@ -0,0 +1,20 @@
+import type { CheckpointFingerprint, CheckpointIncompatibilityReason, DocumentId, ExtractionCheckpointRecord, KnowledgeCapsuleId, LargeDocumentJobProgress } from "@oscharko-dev/keiko-contracts";
+import type { KnowledgeStore } from "../store.js";
+export type CheckpointResumeDecision = {
+    readonly kind: "no-checkpoint";
+} | {
+    readonly kind: "resume";
+    readonly checkpoint: ExtractionCheckpointRecord;
+} | {
+    readonly kind: "complete";
+    readonly checkpoint: ExtractionCheckpointRecord;
+} | {
+    readonly kind: "incompatible";
+    readonly checkpoint: ExtractionCheckpointRecord;
+    readonly reasons: readonly CheckpointIncompatibilityReason[];
+};
+export declare function resolveExtractionResume(store: KnowledgeStore, capsuleId: KnowledgeCapsuleId, documentId: DocumentId, currentFingerprint: CheckpointFingerprint): CheckpointResumeDecision;
+export declare function isResumableCheckpoint(checkpoint: ExtractionCheckpointRecord): boolean;
+export declare function checkpointToProgress(checkpoint: ExtractionCheckpointRecord, safeDisplayName: string): LargeDocumentJobProgress;
+export declare function listResumableDocuments(store: KnowledgeStore, capsuleId: KnowledgeCapsuleId): readonly DocumentId[];
+//# sourceMappingURL=checkpoint-resume.d.ts.map

package/dist/indexing/checkpoint-resume.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint-resume.d.ts","sourceRoot":"","sources":["../../src/indexing/checkpoint-resume.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,qBAAqB,EACrB,+BAA+B,EAC/B,UAAU,EACV,0BAA0B,EAC1B,kBAAkB,EAClB,wBAAwB,EACzB,MAAM,+BAA+B,CAAC;AAIvC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,MAAM,wBAAwB,GAChC;IAAE,QAAQ,CAAC,IAAI,EAAE,eAAe,CAAA;CAAE,GAClC;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,0BAA0B,CAAA;CAAE,GAC5E;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,0BAA0B,CAAA;CAAE,GAC9E;IACE,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,0BAA0B,CAAC;IAChD,QAAQ,CAAC,OAAO,EAAE,SAAS,+BAA+B,EAAE,CAAC;CAC9D,CAAC;AAEN,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,cAAc,EACrB,SAAS,EAAE,kBAAkB,EAC7B,UAAU,EAAE,UAAU,EACtB,kBAAkB,EAAE,qBAAqB,GACxC,wBAAwB,CAa1B;AAID,wBAAgB,qBAAqB,CAAC,UAAU,EAAE,0BAA0B,GAAG,OAAO,CAGrF;AAED,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,0BAA0B,EACtC,eAAe,EAAE,MAAM,GACtB,wBAAwB,CAc1B;AAGD,wBAAgB,sBAAsB,CACpC,KAAK,EAAE,cAAc,EACrB,SAAS,EAAE,kBAAkB,GAC5B,SAAS,UAAU,EAAE,CAIvB"}

package/dist/indexing/checkpoint-resume.js

@@ -0,0 +1,50 @@
+// Resume decision for interrupted large-document jobs (Epic #1160, Issue #1286).
+//
+// Given a durable checkpoint and the current fingerprint (source hash, parser version, policy
+// fingerprint, chunking strategy, embedding identity), decide whether a document can continue from
+// where it stopped. A compatible non-terminal checkpoint resumes; an incompatible one restarts
+// cleanly with a precise reason; a completed checkpoint is skipped.
+import { checkpointCompatibility } from "@oscharko-dev/keiko-contracts";
+import { listExtractionCheckpoints, selectExtractionCheckpoint } from "./checkpoint-persist.js";
+export function resolveExtractionResume(store, capsuleId, documentId, currentFingerprint) {
+    const checkpoint = selectExtractionCheckpoint(store._internal.db, capsuleId, documentId);
+    if (checkpoint === undefined) {
+        return { kind: "no-checkpoint" };
+    }
+    const compat = checkpointCompatibility(checkpoint.fingerprint, currentFingerprint);
+    if (!compat.compatible) {
+        return { kind: "incompatible", checkpoint, reasons: compat.reasons };
+    }
+    if (checkpoint.phase === "complete") {
+        return { kind: "complete", checkpoint };
+    }
+    return { kind: "resume", checkpoint };
+}
+// A checkpoint is resumable for UI/BFF purposes when it carries progress that is neither complete
+// nor a hard failure (i.e. it stopped mid-flight or was cancelled).
+export function isResumableCheckpoint(checkpoint) {
+    if (checkpoint.phase === "complete" || checkpoint.phase === "failed")
+        return false;
+    return true;
+}
+export function checkpointToProgress(checkpoint, safeDisplayName) {
+    return {
+        documentId: checkpoint.documentId,
+        safeDisplayName,
+        strategy: checkpoint.strategy,
+        phase: checkpoint.phase,
+        processedPages: checkpoint.pageCursor,
+        extractedTextBytes: checkpoint.extractedTextBytes,
+        chunkCount: checkpoint.chunkCursor,
+        embeddedChunkCount: checkpoint.embeddedChunkCursor,
+        retryCount: checkpoint.retryCount,
+        coverage: checkpoint.coverage,
+        resumable: isResumableCheckpoint(checkpoint),
+    };
+}
+// Lists every document in the capsule that holds a resumable (mid-flight or cancelled) checkpoint.
+export function listResumableDocuments(store, capsuleId) {
+    return listExtractionCheckpoints(store._internal.db, capsuleId)
+        .filter(isResumableCheckpoint)
+        .map((checkpoint) => checkpoint.documentId);
+}

package/dist/indexing/embedding-batcher.d.ts

@@ -0,0 +1,3 @@
+import { type ChunkToEmbed, type EmbedBatchOptions, type EmbedBatchResult } from "./types.js";
+export declare function embedChunkBatch(chunks: readonly ChunkToEmbed[], options: EmbedBatchOptions): Promise<EmbedBatchResult>;
+//# sourceMappingURL=embedding-batcher.d.ts.map

package/dist/indexing/embedding-batcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-batcher.d.ts","sourceRoot":"","sources":["../../src/indexing/embedding-batcher.ts"],"names":[],"mappings":"AA8BA,OAAO,EAEL,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACtB,MAAM,YAAY,CAAC;AAsdpB,wBAAsB,eAAe,CACnC,MAAM,EAAE,SAAS,YAAY,EAAE,EAC/B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CA0B3B"}

package/dist/indexing/embedding-batcher.js

@@ -0,0 +1,390 @@
+// Embedding batcher (Epic #189, Issue #196). Given a batch of chunks already projected to
+// their excerpt text, this module:
+//
+//   1. Issues N concurrent requests through the injected `OpenAIEmbeddingAdapter`, bounded
+//      by `EmbedBatchOptions.concurrency` (hard-capped to 4 by the orchestrator).
+//   2. For EACH successful response, computes the adapter's reported identity and runs
+//      `assertCompatibleEmbeddingIdentity` against the capsule's pinned identity. The first
+//      structural mismatch aborts the batch with `INCOMPATIBLE_EMBEDDING_IDENTITY` and the
+//      orchestrator marks the job as failed — NO vectors from the batch are persisted.
+//   3. Persists the surviving chunks' embeddings inside a single transaction so a partial
+//      batch failure cannot leave vectors and chunks out of sync.
+//
+// The identity check is the load-bearing invariant from #192. Removing it would let a
+// capsule pinned to dim=1536 silently accept dim=768 rows — see test #5.
+import { assertCompatibleEmbeddingIdentity } from "@oscharko-dev/keiko-model-gateway";
+import { composeVectorRecord, insertVectorRow } from "./vector-persist.js";
+import { IndexingError, } from "./types.js";
+import { chunkDedupeKey } from "../chunking/chunker.js";
+// ─── Concurrency primitive ───────────────────────────────────────────────────
+// Hand-rolled bounded-concurrency runner. Avoids pulling in `p-limit` (the local-knowledge
+// package's runtime-deps surface stays narrow per ADR-0019-3e). Order of `inputs` is
+// preserved in `outputs` even though completion order may differ.
+async function runBounded(inputs, concurrency, work) {
+    const limit = Math.max(1, Math.floor(concurrency));
+    const outputs = new Array(inputs.length);
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < inputs.length) {
+            const i = nextIndex;
+            nextIndex += 1;
+            const input = inputs[i];
+            outputs[i] = await work(input, i);
+        }
+    }
+    const workers = [];
+    for (let i = 0; i < Math.min(limit, inputs.length); i += 1) {
+        workers.push(worker());
+    }
+    await Promise.all(workers);
+    return outputs;
+}
+// ─── Adapter → outcome plumbing ──────────────────────────────────────────────
+// `modelId` is required by the OpenAIEmbeddingRequest contract; the batcher fills it from
+// the capsule's pinned identity so a single batch never queries multiple models. The
+// adapter's `signal`, `apiKeyHeaderName`, and timeout defaults are honoured via the
+// optional-spread pattern (the strict `exactOptionalPropertyTypes` mode forbids passing
+// `undefined` for an optional property).
+async function embedSingleChunkWithModel(adapter, chunk, pinnedIdentity, signal) {
+    return adapter.request({
+        endpoint: adapter.endpoint,
+        apiKey: adapter.apiKey,
+        ...(adapter.apiKeyHeaderName !== undefined
+            ? { apiKeyHeaderName: adapter.apiKeyHeaderName }
+            : {}),
+        modelId: pinnedIdentity.modelId,
+        input: chunk.text,
+        ...(signal !== undefined ? { signal } : {}),
+    });
+}
+// ─── Transient-failure retry ─────────────────────────────────────────────────
+// Only network-flavoured failures are worth retrying. Auth (`wrong-header`),
+// `unsupported-model`, and `invalid-response` are deterministic — retrying them burns
+// the budget without any chance of recovery. `cancelled` is the caller's own abort.
+const TRANSIENT_EMBED_KINDS = new Set(["rate-limited", "timeout", "transport"]);
+const DEFAULT_EMBED_MAX_RETRIES = 6;
+const DEFAULT_EMBED_BASE_DELAY_MS = 500;
+const MAX_EMBED_BACKOFF_MS = 30_000;
+function isTransientOutcome(outcome) {
+    return !outcome.ok && TRANSIENT_EMBED_KINDS.has(outcome.kind);
+}
+function backoffMs(attempt, base) {
+    return Math.min(base * 2 ** (attempt - 1), MAX_EMBED_BACKOFF_MS);
+}
+// Cancellable default sleep. Rejects on abort so the retry loop can bail out of its backoff
+// the moment the caller cancels rather than waiting out the full delay.
+function defaultSleep(ms, signal) {
+    if (signal?.aborted === true) {
+        return Promise.reject(new DOMException("aborted", "AbortError"));
+    }
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        function onAbort() {
+            clearTimeout(timer);
+            reject(new DOMException("aborted", "AbortError"));
+        }
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+function resolveRetry(retry) {
+    return {
+        maxRetries: retry?.maxRetries ?? DEFAULT_EMBED_MAX_RETRIES,
+        baseDelayMs: retry?.baseDelayMs ?? DEFAULT_EMBED_BASE_DELAY_MS,
+        sleep: retry?.sleep ?? defaultSleep,
+    };
+}
+async function embedChunkWithRetry(options, chunk) {
+    const retry = resolveRetry(options.retry);
+    let outcome = await embedSingleChunkWithModel(options.adapter, chunk, options.pinnedIdentity, options.signal);
+    for (let attempt = 1; attempt <= retry.maxRetries; attempt += 1) {
+        if (!isTransientOutcome(outcome) || options.signal?.aborted === true) {
+            return outcome;
+        }
+        try {
+            await retry.sleep(backoffMs(attempt, retry.baseDelayMs), options.signal);
+        }
+        catch {
+            return outcome; // aborted mid-backoff; the abort gate converts this to CANCELLED
+        }
+        outcome = await embedSingleChunkWithModel(options.adapter, chunk, options.pinnedIdentity, options.signal);
+    }
+    return outcome;
+}
+// ─── Array-batch embedding (#189 GRD-004) ────────────────────────────────────
+// When the adapter exposes `requestBatch`, embed many unique chunks per HTTP round-trip.
+// Items per request are bounded so a single response stays well under the gateway's 10 MB
+// JSON cap (96 × 3072 float32 ≈ 4.4 MB) and inside provider per-request token limits.
+const BATCH_ITEM_CAP = 96;
+const BATCH_CHAR_CAP = 120_000;
+function groupIntoBatches(requests) {
+    const batches = [];
+    let current = [];
+    let currentChars = 0;
+    for (const request of requests) {
+        const len = request.representative.text.length;
+        if (current.length > 0 &&
+            (current.length >= BATCH_ITEM_CAP || currentChars + len > BATCH_CHAR_CAP)) {
+            batches.push(current);
+            current = [];
+            currentChars = 0;
+        }
+        current.push(request);
+        currentChars += len;
+    }
+    if (current.length > 0)
+        batches.push(current);
+    return batches;
+}
+function isTransientBatchOutcome(outcome) {
+    return !outcome.ok && TRANSIENT_EMBED_KINDS.has(outcome.kind);
+}
+function errorFromKind(kind) {
+    return { code: "EMBEDDING_ADAPTER_FAILED", message: `embedding adapter returned ${kind}` };
+}
+async function embedArrayBatchWithRetry(options, inputs) {
+    const adapter = options.adapter;
+    const requestBatch = adapter.requestBatch;
+    if (requestBatch === undefined) {
+        return { ok: false, kind: "transport" };
+    }
+    const base = {
+        endpoint: adapter.endpoint,
+        apiKey: adapter.apiKey,
+        ...(adapter.apiKeyHeaderName !== undefined
+            ? { apiKeyHeaderName: adapter.apiKeyHeaderName }
+            : {}),
+        modelId: options.pinnedIdentity.modelId,
+        inputs,
+        ...(options.signal !== undefined ? { signal: options.signal } : {}),
+    };
+    const retry = resolveRetry(options.retry);
+    let outcome = await requestBatch(base);
+    for (let attempt = 1; attempt <= retry.maxRetries; attempt += 1) {
+        if (!isTransientBatchOutcome(outcome) || options.signal?.aborted === true) {
+            return outcome;
+        }
+        try {
+            await retry.sleep(backoffMs(attempt, retry.baseDelayMs), options.signal);
+        }
+        catch {
+            return outcome; // aborted mid-backoff; the abort gate converts this to CANCELLED
+        }
+        outcome = await requestBatch(base);
+    }
+    return outcome;
+}
+// Apply the per-vector identity gate exactly as the scalar path does. Order-independent:
+// once `state.identityFailure` is set, embedChunkBatch persists nothing, so which concurrent
+// batch first observes the drift is irrelevant to the outcome.
+function gateVectorOutcome(representative, success, options, state) {
+    if (state.identityFailure !== undefined) {
+        return { ok: false, chunk: representative, error: state.identityFailure };
+    }
+    const observed = identityFromAdapter(options.pinnedIdentity, success);
+    const compat = assertCompatibleEmbeddingIdentity(options.pinnedIdentity, observed);
+    if (!compat.ok) {
+        state.identityFailure = {
+            code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
+            message: compat.safeMessage,
+        };
+        return { ok: false, chunk: representative, error: state.identityFailure };
+    }
+    return { ok: true, chunk: representative, success };
+}
+async function embedUniqueBatch(batch, options, state) {
+    if (state.identityFailure !== undefined) {
+        const failure = state.identityFailure;
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error: failure }));
+    }
+    const abortError = checkAbort(options.signal);
+    if (abortError !== undefined) {
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error: abortError }));
+    }
+    const outcome = await embedArrayBatchWithRetry(options, batch.map((r) => r.representative.text));
+    if (!outcome.ok) {
+        const error = errorFromKind(outcome.kind);
+        return batch.map((r) => ({ ok: false, chunk: r.representative, error }));
+    }
+    return batch.map((request, i) => {
+        const success = outcome.value[i];
+        if (success === undefined) {
+            return {
+                ok: false,
+                chunk: request.representative,
+                error: errorFromKind("invalid-response"),
+            };
+        }
+        return gateVectorOutcome(request.representative, success, options, state);
+    });
+}
+// ─── Identity verification ───────────────────────────────────────────────────
+function identityFromAdapter(pinned, success) {
+    // `provider` and `vectorMetric` are not echoed by the OpenAI API response — they come from
+    // the operator's pinned identity. Only `modelId`, `modelRevision`, and `vectorDimensions`
+    // are observed from the adapter's outcome. Identity-compatibility checks the structural
+    // tuple (provider+modelId+dims+metric), so the constructed identity only loses fidelity
+    // on `modelRevision` (which the compatibility check treats as a warning, not a failure).
+    return {
+        provider: pinned.provider,
+        modelId: success.modelId,
+        vectorDimensions: success.vector.length,
+        vectorMetric: pinned.vectorMetric,
+        ...(success.modelRevision !== undefined ? { modelRevision: success.modelRevision } : {}),
+    };
+}
+// ─── Float32 → byte serialisation ────────────────────────────────────────────
+// The schema column is BLOB; SQLite expects a Uint8Array. We copy the underlying
+// ArrayBuffer rather than aliasing it (Float32Array views can share buffers) so the
+// persisted row is a stable copy not affected by any later vector reuse.
+function floatToBytes(vector) {
+    return new Uint8Array(vector.buffer.slice(vector.byteOffset, vector.byteOffset + vector.byteLength));
+}
+function dedupeEmbeddingRequests(chunks) {
+    const byKey = new Map();
+    for (const chunk of chunks) {
+        const key = chunkDedupeKey(chunk.text) ?? `chunk:${String(chunk.id)}`;
+        const existing = byKey.get(key);
+        if (existing === undefined) {
+            byKey.set(key, { representative: chunk, chunks: [chunk] });
+        }
+        else {
+            existing.chunks.push(chunk);
+        }
+    }
+    return [...byKey.entries()].map(([key, value]) => ({ key, ...value }));
+}
+function outcomeForChunk(outcome, chunk) {
+    if (outcome.ok) {
+        return { ok: true, chunk, success: outcome.success };
+    }
+    return { ok: false, chunk, error: outcome.error };
+}
+function errorFromOutcome(outcome) {
+    return {
+        code: "EMBEDDING_ADAPTER_FAILED",
+        message: `embedding adapter returned ${outcome.kind}`,
+    };
+}
+function checkAbort(signal) {
+    if (signal?.aborted === true) {
+        return { code: "CANCELLED", message: "indexing aborted via AbortSignal" };
+    }
+    return undefined;
+}
+async function buildUniqueChunkOutcome(request, options, state) {
+    if (state.identityFailure !== undefined) {
+        return { ok: false, chunk: request.representative, error: state.identityFailure };
+    }
+    const abortError = checkAbort(options.signal);
+    if (abortError !== undefined) {
+        return { ok: false, chunk: request.representative, error: abortError };
+    }
+    const outcome = await embedChunkWithRetry(options, request.representative);
+    if (!outcome.ok) {
+        return { ok: false, chunk: request.representative, error: errorFromOutcome(outcome) };
+    }
+    const observed = identityFromAdapter(options.pinnedIdentity, outcome.value);
+    const compat = assertCompatibleEmbeddingIdentity(options.pinnedIdentity, observed);
+    if (!compat.ok) {
+        state.identityFailure = {
+            code: "INCOMPATIBLE_EMBEDDING_IDENTITY",
+            message: compat.safeMessage,
+        };
+        return { ok: false, chunk: request.representative, error: state.identityFailure };
+    }
+    return { ok: true, chunk: request.representative, success: outcome.value };
+}
+function expandUniqueOutcomes(uniqueRequests, uniqueOutcomes) {
+    const outcomes = [];
+    for (let i = 0; i < uniqueRequests.length; i += 1) {
+        const request = uniqueRequests[i];
+        const outcome = uniqueOutcomes[i];
+        if (request === undefined || outcome === undefined)
+            continue;
+        for (const chunk of request.chunks) {
+            outcomes.push(outcomeForChunk(outcome, chunk));
+        }
+    }
+    return outcomes;
+}
+// Build all per-chunk outcomes BEFORE we open a write transaction. The identity gate runs
+// after every successful response so we fail fast on dimension mismatch.
+async function buildChunkOutcomes(chunks, options) {
+    const state = { identityFailure: undefined };
+    const uniqueRequests = dedupeEmbeddingRequests(chunks);
+    let uniqueOutcomes;
+    if (typeof options.adapter.requestBatch === "function") {
+        // Array-batch path: collapse the unique requests into ceil(N / itemCap) HTTP calls,
+        // run those calls with bounded concurrency, then flatten back into request order.
+        const batches = groupIntoBatches(uniqueRequests);
+        const batchOutcomes = await runBounded(batches, options.concurrency, async (batch) => embedUniqueBatch(batch, options, state));
+        uniqueOutcomes = batchOutcomes.flat();
+    }
+    else {
+        // Scalar fallback (adapters/stubs without `requestBatch`): one HTTP call per unique chunk.
+        uniqueOutcomes = await runBounded(uniqueRequests, options.concurrency, async (request) => {
+            return buildUniqueChunkOutcome(request, options, state);
+        });
+    }
+    const outcomes = expandUniqueOutcomes(uniqueRequests, uniqueOutcomes);
+    return state.identityFailure === undefined
+        ? { outcomes }
+        : { outcomes, identityFailure: state.identityFailure };
+}
+// ─── Persistence boundary ─────────────────────────────────────────────────────
+// Wraps the row inserts in a single transaction so a partial INSERT failure rolls back the
+// whole batch. The orchestrator marks the document as failed; subsequent runs can retry.
+function persistOutcomes(store, outcomes, pinnedIdentity, idSource, now) {
+    const db = store._internal.db;
+    const persisted = [];
+    db.exec("BEGIN");
+    try {
+        for (const out of outcomes) {
+            if (!out.ok)
+                continue;
+            const observed = identityFromAdapter(pinnedIdentity, out.success);
+            const row = {
+                id: `vec:${String(out.chunk.id)}`,
+                capsuleId: out.chunk.capsuleId,
+                sourceId: out.chunk.sourceId,
+                documentId: out.chunk.documentId,
+                chunkId: out.chunk.id,
+                embedding: floatToBytes(out.success.vector),
+                identity: observed,
+                storageReference: idSource(),
+                createdAt: now(),
+            };
+            insertVectorRow(db, store._internal.contentCipher, row);
+            persisted.push(composeVectorRecord(row));
+        }
+        db.exec("COMMIT");
+    }
+    catch (cause) {
+        db.exec("ROLLBACK");
+        throw new IndexingError("PERSISTENCE_FAILED", "vector persistence failed mid-batch", cause === undefined ? undefined : { cause });
+    }
+    return persisted;
+}
+// ─── Public entrypoint ───────────────────────────────────────────────────────
+export async function embedChunkBatch(chunks, options) {
+    if (chunks.length === 0) {
+        return { vectors: [], errors: [] };
+    }
+    const { outcomes, identityFailure } = await buildChunkOutcomes(chunks, options);
+    const errors = outcomes
+        .filter((o) => !o.ok)
+        .map((o) => o.error);
+    // Identity drift OR cancellation: refuse to persist ANY row from this batch.
+    if (identityFailure !== undefined) {
+        return { vectors: [], errors };
+    }
+    const abortError = checkAbort(options.signal);
+    if (abortError !== undefined) {
+        return { vectors: [], errors: [...errors, abortError] };
+    }
+    const vectors = persistOutcomes(options.store, outcomes, options.pinnedIdentity, options.idSource, options.now);
+    return { vectors, errors };
+}

package/dist/indexing/index.d.ts

@@ -0,0 +1,7 @@
+export { runIndexingJob } from "./orchestrator.js";
+export { embedChunkBatch } from "./embedding-batcher.js";
+export { findResumableJob } from "./job-resume.js";
+export { upsertExtractionCheckpoint, selectExtractionCheckpoint, listExtractionCheckpoints, deleteExtractionCheckpoint, } from "./checkpoint-persist.js";
+export { resolveExtractionResume, isResumableCheckpoint, checkpointToProgress, listResumableDocuments, type CheckpointResumeDecision, } from "./checkpoint-resume.js";
+export { DEFAULT_INDEXING_BATCH_SIZE, DEFAULT_INDEXING_CONCURRENCY, IndexingError, type ChunkToEmbed, type EmbedBatchOptions, type EmbedBatchResult, type EmbedRetryOptions, type IndexingDocumentChunkedEvent, type IndexingDocumentDiscoveredEvent, type IndexingDocumentEmbeddedEvent, type IndexingDocumentExtractedEvent, type IndexingDocumentFailedEvent, type IndexingDocumentSkippedEvent, type IndexingErrorCode, type IndexingEvent, type IndexingJobCancelledEvent, type IndexingJobCompletedEvent, type IndexingJobFailedEvent, type IndexingJobStartedEvent, type IndexingOptions, type IndexingResult, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/indexing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/indexing/index.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,EACzB,0BAA0B,GAC3B,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,uBAAuB,EACvB,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACtB,KAAK,wBAAwB,GAC9B,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EACL,2BAA2B,EAC3B,4BAA4B,EAC5B,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,4BAA4B,EACjC,KAAK,+BAA+B,EACpC,KAAK,6BAA6B,EAClC,KAAK,8BAA8B,EACnC,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,iBAAiB,EACtB,KAAK,aAAa,EAClB,KAAK,yBAAyB,EAC9B,KAAK,yBAAyB,EAC9B,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,eAAe,EACpB,KAAK,cAAc,GACpB,MAAM,YAAY,CAAC"}

package/dist/indexing/index.js

@@ -0,0 +1,11 @@
+// Public surface of the indexing layer (Epic #189, Issue #196). Composed by the package
+// barrel in ../index.ts; consumers outside this package never import from this
+// subdirectory directly (ADR-0019 direction rule 3e + the trust-8 test-support naming
+// convention).
+export { runIndexingJob } from "./orchestrator.js";
+export { embedChunkBatch } from "./embedding-batcher.js";
+export { findResumableJob } from "./job-resume.js";
+// Bounded large-document ingestion checkpoints + resume (Epic #1160, Issue #1286).
+export { upsertExtractionCheckpoint, selectExtractionCheckpoint, listExtractionCheckpoints, deleteExtractionCheckpoint, } from "./checkpoint-persist.js";
+export { resolveExtractionResume, isResumableCheckpoint, checkpointToProgress, listResumableDocuments, } from "./checkpoint-resume.js";
+export { DEFAULT_INDEXING_BATCH_SIZE, DEFAULT_INDEXING_CONCURRENCY, IndexingError, } from "./types.js";