akm-cli 0.9.15 → 0.9.16-alpha.1

package/dist/core/config/schema/search.js

@@ -23,8 +23,9 @@ const SearchGraphBoostSchema = z
 })
     .passthrough();
 /**
- * `search.curateRerank` (#951) — an optional cross-encoder rerank pass over
- * `akm curate`'s already-selected candidates.
+ * `search.rerank` (#951, moved from `search.curateRerank` in 0.9.16 — the
+ * pass was always meant for `akm search`, not `akm curate`) — an optional
+ * cross-encoder rerank pass over search's already-ranked LOCAL stash hits.
  *
  * Deliberately its own small config arm rather than a third member of the
  * `engines` map (`EngineConfigSchema` in ./engines.ts): that union's "llm" /
@@ -37,26 +38,27 @@ const SearchGraphBoostSchema = z
  * connection shape as an LLM engine without inheriting that machinery.
  *
  * `curate_rerank` was removed as a dead `llm.features.*` key in 0.8.0 (no
- * implementation ever sent a request); this is a new, real implementation,
- * disabled by default.
+ * implementation ever sent a request); 0.9.15 shipped a real implementation
+ * wired to curate under `search.curateRerank`, disabled by default; 0.9.16
+ * moves it to search and renames the key (no compatibility alias — the old
+ * key shipped hours earlier, default-off, so nobody has it meaningfully set).
  */
-export const CurateRerankConfigSchema = z
+export const SearchRerankConfigSchema = z
     .object({
     enabled: z.boolean().optional(),
     /** Full URL of the reranker's rerank endpoint, e.g. `http://host:port/rerank`. */
     endpoint: httpUrl.optional(),
     model: nonEmptyString.optional(),
-    apiKey: symbolicOrWarnApiKey("search.curateRerank.apiKey").optional(),
+    apiKey: symbolicOrWarnApiKey("search.rerank.apiKey").optional(),
     timeoutMs: positiveInt.optional(),
-    /** How many of curate's already-ranked candidates to send to the reranker. Default 8. */
+    /** How many of search's already-ranked LOCAL hits to send to the reranker. Default 8. */
     topN: positiveInt.max(50).optional(),
 })
     .passthrough();
 export const SearchConfigSchema = z
     .object({
-    minScore: nonNegativeNumber.optional(),
     defaultExcludeTypes: z.array(nonEmptyString).optional(),
     graphBoost: SearchGraphBoostSchema.optional(),
-    curateRerank: CurateRerankConfigSchema.optional(),
+    rerank: SearchRerankConfigSchema.optional(),
 })
     .passthrough();

package/dist/core/errors.js

@@ -18,6 +18,7 @@ const CONFIG_HINTS = {
     UNKNOWN_IMPROVE_STRATEGY: "Pass one of the listed strategy names to `--strategy`, or define it under `improve.strategies`. Names are case-sensitive.",
     EXECUTION_NOT_AUTHORIZED: "Change the selected tools or update the machine/user execution policy, then retry.",
     SECRET_REFERENCE_UNRESOLVED: "Check the secret exists (`akm secret list`) and the name after `secret://` matches, or run `akm secret set <name> <value>` to store it.",
+    EMBEDDING_VEC_UNAVAILABLE: "Install sqlite-vec for unit-level semantic search, or rely on lexical search until it is available.",
 };
 // Code-review finding: COMPOSITION_INVALID covers several unrelated causes
 // (a rejected with:, a multi-job source, a composition cycle/depth/size
@@ -82,8 +83,10 @@ const USAGE_HINTS = {
 const TRANSIENT_HINTS = {
     RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
     STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
-    INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs.",
-    MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on scheduled index/improve/workflow runs.",
+    INDEX_DB_CONTENDED: "Another akm process is writing index.db right now. Wait a few seconds and retry — index runs take no rebuild " +
+        "lock, so this clears quickly; a scheduled run left alone will simply run again next time.",
+    MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on " +
+        "scheduled improve runs to skip gracefully instead — workflow run does not treat this code as skippable.",
     IMPROVE_LOCK_HELD: "Another akm improve run holds the whole-run lock right now. Wait for it to finish and retry, or pass --skip-if-locked on scheduled runs.",
 };
 /** Default hint for each NotFoundError code. */

package/dist/core/hash.js

@@ -0,0 +1,18 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Content addressing for text handed to an embedding provider.
+ *
+ * `hashEmbeddableText` is the ONE hash function every writer of and lookup
+ * against provider-bound text must agree on — a stable, deterministic sha256
+ * over the exact bytes sent, so a single-byte content change is never
+ * confused with unchanged text. Originally lived in
+ * `embedding-salvage-repository.ts` (#955); moved here so a pure
+ * text-processing module (`src/indexer/units/unit.ts`, #index-units) is not
+ * coupled to a storage repository just to reuse it.
+ */
+import { createHash } from "node:crypto";
+export function hashEmbeddableText(text) {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+}

package/dist/core/maintenance-barrier.js

@@ -27,16 +27,18 @@ const heldBarrierContext = new AsyncLocalStorage();
 const MAINTENANCE_BARRIER_STALE_AFTER_MS = 5 * 60 * 1000;
 /**
  * The barrier normally holds for one lock-file write — sub-millisecond on
- * any real filesystem. Two akm processes racing to register a lock in the
- * very same instant (e.g. two `akm index` runs a scheduler launched back to
- * back) can still collide on it; retrying briefly resolves that ordinary
- * case instead of failing a legitimate concurrent invocation outright
+ * any real filesystem. Two akm processes racing to register a lock/lease/
+ * activity in the very same instant (e.g. two `akm index` runs a scheduler
+ * launched back to back, both opening canonical state.db) can still collide
+ * on it; retrying briefly resolves that ordinary case instead of failing a
+ * legitimate concurrent invocation outright
  * (field follow-up to #956, G1). Bounded short so a genuinely wedged holder
  * still surfaces the busy error promptly rather than making a losing
  * process hang — comfortably above the barrier's normal hold time, well
  * below a length that would make this feel like the blocking lock #872
- * removed. Never applies to the rebuild lock itself, which stays
- * non-blocking (#872).
+ * removed. Never applies to whatever lock/lease/activity is registered
+ * after the barrier releases — that thing's own held/skip/throw/wait
+ * policy belongs to its caller, not to the barrier (#872).
  */
 const MAINTENANCE_BARRIER_BUSY_RETRY_BOUND_MS = 1_500;
 let busyRetryBoundMsForTests;

package/dist/core/paths.js

@@ -234,17 +234,6 @@ export function getDbPath(env = process.env) {
 export function getIndexWriterLockPath() {
     return path.join(getDataDir(), "index.db.write.lock");
 }
-/**
- * Path to the opt-in, PID-liveness-only rebuild lock an explicit `akm index`
- * run acquires (#956). Distinct from {@link getIndexWriterLockPath}'s
- * `index.db.write.lock`, which is the asset-mutation lease and unrelated to
- * indexing since #872 — this lock never blocks and is never required, it
- * only lets a scheduled/opportunistic `akm index --skip-if-locked` step
- * aside instead of contending with a run already in progress.
- */
-export function getIndexRebuildLockPath() {
-    return path.join(getDataDir(), "index.rebuild.lock");
-}
 export function getMaintenanceBarrierPath() {
     return path.join(getDataDir(), "maintenance.barrier.lock");
 }

package/dist/core/run-lock.js

@@ -3,8 +3,11 @@
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
  * PID-liveness-only run lock — the shared mechanics behind `akm improve`'s
- * whole-run lock (`commands/improve/locks.ts`) and the index rebuild lock
- * (`indexer/index-rebuild-lock.ts`, #956).
+ * whole-run lock (`commands/improve/locks.ts`). Originally also backed the
+ * index rebuild lock (`indexer/index-rebuild-lock.ts`, #956); index-redesign
+ * deleted that consumer (every index write is now a short, idempotent,
+ * content-addressed transaction, so there is no rebuild to serialize), and
+ * `improve`'s lock is this module's only caller today.
  *
  * No `staleAfterMs`: only a verifiably dead holder is ever reclaimed. This is
  * the #872 lesson encoded at the mechanics layer so every future caller gets

package/dist/core/state/migrations.js

@@ -1200,12 +1200,33 @@ export function getStateMigrationSafety(migrationId) {
  * Delegates to the shared SQLite migration engine; state.db has no
  * pre-versioning bootstrap step, so no `bootstrap` hook is passed.
  *
+ * `freshDatabase` is the one exception to "single transaction per migration":
+ * the entire registry is locked through the final migration and applied as
+ * ONE transaction (mirroring the existing-unversioned prefix lock below).
+ * Two akm processes can race to create the same brand-new state.db
+ * (`openStateDatabase`'s file-reservation only decides who creates the file,
+ * not who finishes initializing it first); with per-migration commits, the
+ * loser could open its own connection between two of the winner's commits,
+ * see a real-but-incomplete ledger, and — correctly believing this is an
+ * ordinary existing database needing to catch up on pending migrations —
+ * refuse historical-destructive migration 018 with the same message a
+ * genuine legacy database gets. Applying every migration for a fresh
+ * database atomically removes that partially-migrated state from view
+ * entirely: a concurrent opener now only ever observes "nothing committed
+ * yet" (treated as fresh) or "fully current" (nothing left to apply), never
+ * the in-between. Safe to lose on a crash mid-bootstrap: a fresh database
+ * has no prior data to preserve, so rolling back to nothing and letting the
+ * next open retry from scratch is strictly fine.
+ *
  * Called automatically by `openStateDatabase()`.
  */
 export function runMigrations(db, options) {
     const initialMigration = STATE_MIGRATIONS[0];
     if (!initialMigration)
         throw new Error("State migration registry has no initial migration.");
+    const finalMigration = STATE_MIGRATIONS[STATE_MIGRATIONS.length - 1];
+    if (!finalMigration)
+        throw new Error("State migration registry has no final migration.");
     let existingUnversionedSnapshotPrepared = false;
     const prepareExistingUnversionedState = (lockedDb) => {
         if (existingUnversionedSnapshotPrepared)
@@ -1228,7 +1249,11 @@ export function runMigrations(db, options) {
         existingUnversionedSnapshotPrepared = true;
     };
     runSqliteMigrations(db, STATE_MIGRATIONS, {
-        lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase ? "002-task-history-per-run" : undefined,
+        lockInitialMigrationPrefixThrough: options?.existingUnversionedDatabase
+            ? "002-task-history-per-run"
+            : options?.freshDatabase
+                ? finalMigration.id
+                : undefined,
         beforeLedgerInitializationLocked(lockedDb) {
             if (options?.freshDatabase)
                 return;

package/dist/core/state-db.js

@@ -442,10 +442,24 @@ export function openStateDatabase(dbPath, options) {
             });
             try {
                 preflight.exec("PRAGMA busy_timeout = 30000");
-                const ledger = assertMigrationLedger(preflight, STATE_MIGRATIONS);
+                // Both reads below must observe ONE WAL snapshot. Each is its own
+                // statement, and without an explicit transaction SQLite auto-commits
+                // each separately, so they can see two different snapshots. A sibling
+                // process's fresh bootstrap is now one all-or-nothing transaction, so
+                // a commit landing between the two reads showed an empty ledger to the
+                // first and the sibling's freshly created tables to the second — the
+                // signature of a genuine legacy unversioned database, which this
+                // preflight then refused. Pinned to one snapshot, either nothing the
+                // sibling did is visible (empty ledger AND no tables, correctly fresh)
+                // or all of it is (a current ledger, so the refusal is never reached).
+                // A real legacy database, racing no one, reads exactly as before.
+                const { ledger, hasNoOtherTables } = preflight.transaction(() => ({
+                    ledger: assertMigrationLedger(preflight, STATE_MIGRATIONS),
+                    hasNoOtherTables: unversionedDatabaseHasNoTables(preflight),
+                }))();
                 warnNewerStateLedger(ledger);
                 existingUnversionedDatabase = ledger.migrationIds.length === 0;
-                if (existingUnversionedDatabase && unversionedDatabaseHasNoTables(preflight)) {
+                if (existingUnversionedDatabase && hasNoOtherTables) {
                     existingUnversionedDatabase = false;
                     treatUnversionedAsFresh = true;
                 }
@@ -539,7 +553,21 @@ export function openStateDatabase(dbPath, options) {
         if (freshReservation)
             closeFileIdentity(freshReservation);
         releaseActivity?.();
-        throw error;
+        // The migration engine's own writer-lock retry (sqlite-migrations.ts's
+        // `withImmediateWriteLock`, used by every migration this open can run,
+        // including a from-empty first open) throws the raw driver error after
+        // its own retry budget, not a `TransientError` — only
+        // `beginImmediateTransaction` below does that reclassification, and nothing
+        // upstream of it re-wraps a raw SQLITE_BUSY/LOCKED that surfaces from
+        // deeper in the open/migrate sequence (field follow-up: a from-empty
+        // first open racing a concurrent opener can still hit this under load).
+        // Reclassify uniformly here so the contract this function promises --
+        // contention is reported as `STATE_DB_CONTENDED` (exit 75), never a raw
+        // driver message (exit 70) -- holds regardless of which internal step
+        // the contention was observed at. Anything not contention-shaped
+        // (a genuine legacy-database refusal, corruption, a real schema error)
+        // rethrows exactly as raised.
+        throwBeginFailure(error, "state");
     }
 }
 /**
@@ -690,35 +718,43 @@ function sleepSyncMs(ms) {
         return;
     sleepSync(ms);
 }
-/**
- * Open, but deliberately do not finish, an immediate transaction.
- *
- * This is the split-phase counterpart to {@link withImmediateTransaction} for
- * the source-update coordinator: index finalization must mutate state.db in a
- * transaction that remains pending until content, lockfile, and index
- * publication have all succeeded. The caller that asked for this split phase
- * owns the matching COMMIT/ROLLBACK.
- */
 /**
  * Reclassify an exhausted-retry BEGIN failure that is still contention-shaped
- * (#948) into a `TransientError("STATE_DB_CONTENDED")`, mirroring the
- * RUN_LEASE_HELD precedent (`WorkflowRunsRepository.acquireEngineLease`): the
- * driver text is accurate but unhelpful (`{"ok":false,"error":"database is
- * locked"}`, exit 70/INTERNAL) — this instead reads as a retryable-shortly
- * signal (exit 75, sysexits EX_TEMPFAIL — #948 addendum) with the original
- * error preserved as `cause` for `--verbose`/debugging. A genuinely unrelated
- * error (not contention-shaped) is rethrown exactly as raised, never
- * reclassified.
+ * (#948) into a `TransientError`, mirroring the RUN_LEASE_HELD precedent
+ * (`WorkflowRunsRepository.acquireEngineLease`): the driver text is accurate
+ * but unhelpful (`{"ok":false,"error":"database is locked"}`, exit
+ * 70/INTERNAL) — this instead reads as a retryable-shortly signal (exit 75,
+ * sysexits EX_TEMPFAIL — #948 addendum) with the original error preserved as
+ * `cause` for `--verbose`/debugging. A genuinely unrelated error (not
+ * contention-shaped) is rethrown exactly as raised, never reclassified.
+ *
+ * `dbKind` (field follow-up to #956) picks the reported identity: `"state"`
+ * (default, unchanged text) yields `STATE_DB_CONTENDED`; `"index"` yields
+ * `INDEX_DB_CONTENDED` with index.db's own message, mirroring
+ * `reclassifyIndexDbContention`'s text (`src/indexer/indexer.ts`) so a caller
+ * that reaches this helper directly and one that only reclassifies a raw
+ * driver error report the same thing.
  */
-function throwBeginFailure(err) {
+function throwBeginFailure(err, dbKind) {
     if (isSqliteContentionError(err)) {
-        const contended = new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
+        const contended = dbKind === "index"
+            ? new TransientError("akm's index database is busy (another akm process is writing it); retry shortly.", "INDEX_DB_CONTENDED")
+            : new TransientError("akm's state database is busy (another akm process is writing it); retry shortly.", "STATE_DB_CONTENDED");
         contended.cause = err;
         throw contended;
     }
     throw err;
 }
-export function beginImmediateTransaction(db) {
+/**
+ * Open, but deliberately do not finish, an immediate transaction.
+ *
+ * This is the split-phase counterpart to {@link withImmediateTransaction} for
+ * the source-update coordinator: index finalization must mutate state.db in a
+ * transaction that remains pending until content, lockfile, and index
+ * publication have all succeeded. The caller that asked for this split phase
+ * owns the matching COMMIT/ROLLBACK.
+ */
+export function beginImmediateTransaction(db, dbKind = "state") {
     if (db.inTransaction) {
         throw new Error("beginImmediateTransaction requires a connection with no active transaction");
     }
@@ -745,12 +781,12 @@ export function beginImmediateTransaction(db) {
                 sleepSyncMs(2 ** (attempt - 1));
                 continue;
             }
-            throwBeginFailure(err);
+            throwBeginFailure(err, dbKind);
         }
     }
-    throwBeginFailure(lastBeginErr);
+    throwBeginFailure(lastBeginErr, dbKind);
 }
-export function withImmediateTransaction(db, fn) {
+export function withImmediateTransaction(db, fn, dbKind = "state") {
     // Re-entrancy guard (issue #686): if a transaction is already open on this
     // connection (e.g. a nested withImmediateTransaction call inside an outer
     // frame's fn), join it — run fn directly with no BEGIN/COMMIT/ROLLBACK of
@@ -761,7 +797,7 @@ export function withImmediateTransaction(db, fn) {
     if (db.inTransaction) {
         return fn();
     }
-    beginImmediateTransaction(db);
+    beginImmediateTransaction(db, dbKind);
     try {
         const result = fn();
         if (!db.inTransaction) {

package/dist/indexer/drain.js

@@ -0,0 +1,306 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { getConfigPath } from "../core/paths.js";
+import { isVerbose } from "../core/warn.js";
+import { embedBatch } from "../llm/embedder.js";
+import { probeProviderLimits } from "../llm/embedders/provider-limits.js";
+import { describeEmbeddingCredential, hasRemoteEndpoint, normalizeEmbeddingEndpoint, } from "../llm/embedders/remote.js";
+import { getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
+import { SQLITE_CHUNK_SIZE } from "../storage/repositories/index-sql.js";
+import { isVecAvailable } from "../storage/repositories/index-vec-repository.js";
+import { dropOtherIdentities, listMissingHashes, upsertUnitVectors } from "../storage/repositories/units-repository.js";
+import { deriveObservedEmbeddingIdentity } from "./embedding-identity.js";
+/**
+ * Failure threshold, within the recent window below, that stops dispatching
+ * further provider batches. Mirrors materialize-embeddings.ts's own
+ * (unexported) `CIRCUIT_BREAKER_THRESHOLD`, #954 — reimplemented here at the
+ * same value rather than imported, since that file is private and slated for
+ * deletion by B5; the underlying stop-dispatch MECHANISM (`onSkip` returning
+ * `false`) is still the real `RemoteEmbedder`'s, reused unmodified. Two
+ * independent streaks share it: single-document failures (a multi-document
+ * timeout is not yet evidence of a dead endpoint — `RemoteEmbedder` retries
+ * and splits it smaller before ever reporting it this small), or network
+ * errors at ANY size (never retried, trusted immediately). Storage-write
+ * failures (E5b — `upsertUnitVectors`'s own per-row result) feed the SAME two
+ * streaks: a sustained STORAGE failure (contention, permissions, a full
+ * disk) must stop paying for provider requests just as surely as a
+ * sustained PROVIDER failure, even while the provider itself keeps
+ * succeeding.
+ */
+const CIRCUIT_BREAKER_THRESHOLD = 3;
+/**
+ * Recent-history window (in settled batch-starts) the two streaks above are
+ * evaluated over, in place of a plain "reset to zero on any success" counter
+ * (round-2 field finding): with concurrent dispatch (default 2, up to 16)
+ * outcomes settle out of dispatch order, so a degraded endpoint failing MOST
+ * requests never tripped the breaker as long as occasional successes
+ * interleaved — reproduced with a 67% failure rate dispatching the entire
+ * pending set. `CIRCUIT_BREAKER_THRESHOLD` failures within the last
+ * `CIRCUIT_BREAKER_WINDOW` settled batch-starts of a streak's own kind (see
+ * {@link pushBreakerOutcome}) trips it: a genuinely dead endpoint (no
+ * successes at all) still trips in exactly `CIRCUIT_BREAKER_THRESHOLD`
+ * batches, same as before; a single success now only AGES a failure out of
+ * the window over time rather than erasing the whole run's evidence at once.
+ */
+const CIRCUIT_BREAKER_WINDOW = CIRCUIT_BREAKER_THRESHOLD * 2;
+/** Record one settled batch-start's outcome into a breaker streak's window, capped at {@link CIRCUIT_BREAKER_WINDOW}. */
+function pushBreakerOutcome(window, isFailure) {
+    window.push(isFailure);
+    if (window.length > CIRCUIT_BREAKER_WINDOW)
+        window.shift();
+}
+/** Failures currently recorded in a breaker streak's window. */
+function breakerFailureCount(window) {
+    return window.reduce((n, isFailure) => n + (isFailure ? 1 : 0), 0);
+}
+/**
+ * Prefix of the per-committed-batch progress line (`"${DRAIN_BATCH_PROGRESS_PREFIX}N: …"`,
+ * emitted once per provider batch this call commits). Exported so a caller
+ * juggling several `onProgress` sources (`stash-cli.ts`'s `akm index`) can
+ * recognize — and, outside `--verbose`, suppress — this specific
+ * high-frequency line by prefix rather than re-deriving its own copy of the
+ * pattern (#954).
+ */
+export const DRAIN_BATCH_PROGRESS_PREFIX = "[drain] batch ";
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw signal.reason instanceof Error ? signal.reason : new Error("drain interrupted");
+    }
+}
+/** Every distinct unit hash the index currently knows about, regardless of identity. */
+function selectAllUnitHashes(db) {
+    return db.prepare("SELECT unit_hash FROM unit_texts ORDER BY unit_hash").all().map((row) => row.unit_hash);
+}
+/** `unit_hash -> text` for exactly `hashes`, chunked to respect SQLite's bound-parameter limit. */
+function fetchUnitTexts(db, hashes) {
+    const texts = new Map();
+    for (let offset = 0; offset < hashes.length; offset += SQLITE_CHUNK_SIZE) {
+        const chunk = hashes.slice(offset, offset + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        const rows = db
+            .prepare(`SELECT unit_hash, text FROM unit_texts WHERE unit_hash IN (${placeholders})`)
+            .all(...chunk);
+        for (const row of rows)
+            texts.set(row.unit_hash, row.text);
+    }
+    return texts;
+}
+/**
+ * Effective embedding config and request packing for this drain (index
+ * redesign, B5 — replaces the retired `embedding.maxTokens`/`batchSize`/
+ * `contextLength` config keys): `concurrency` (in-flight requests) defaults
+ * to the provider's OWN observed slot count when `embedding.concurrency`
+ * itself leaves it unset (`probeProviderLimits` already applies that same
+ * override to `slots`). The request TOKEN WINDOW, chars-per-token ratio, and
+ * Ollama `num_ctx` are no longer config fields at all — they are threaded
+ * into `RemoteEmbedder.embedBatch` as `packing`, sourced straight from the
+ * same probe: `windowTokens` for the per-request budget, `charsPerToken` for
+ * the calibrated per-text token estimate, and `windowTokens` again for
+ * Ollama's `num_ctx` when `source === "ollama"`. `windowIsKnown`
+ * (`source !== "default"`) gates `RemoteEmbedder`'s same-run adaptive
+ * shrink: a provider that reports nothing about its own context size still
+ * gets that corrective, but a probed, authoritative window does not need it
+ * second-guessed.
+ */
+async function resolveEmbeddingPacking(config, signal) {
+    const base = config.embedding ?? {};
+    const limits = await probeProviderLimits(base, { signal });
+    return {
+        embeddingConfig: { ...base, concurrency: base.concurrency ?? limits.slots },
+        packing: {
+            tokenBudget: limits.windowTokens,
+            charsPerToken: limits.charsPerToken,
+            windowIsKnown: limits.source !== "default",
+            ollamaNumCtx: limits.source === "ollama" ? limits.windowTokens : undefined,
+        },
+    };
+}
+/**
+ * #953 field gap, ported from the deleted `materialize-embeddings.ts`
+ * (`git show fc711fd6^:src/indexer/materialize-embeddings.ts`): a keyless
+ * request against a remote embedding endpoint could not be reproduced in the
+ * lab — every `RemoteEmbedder` path already resolves `secret://` through one
+ * boundary, so a keyless request can only mean `embedding.apiKey` was absent
+ * from the config THIS run loaded. The actionable outcome is a
+ * self-diagnosing run, not a fix: one default-level line, emitted once
+ * before the first provider request this call makes, naming the endpoint,
+ * model, and credential SOURCE (never the value) so a field run can compare
+ * it against what the gateway actually saw. A local (non-remote) endpoint,
+ * or a call with nothing pending, has nothing to diagnose and stays silent.
+ */
+function emitCredentialDiagnostic(config, onProgress) {
+    if (!onProgress || !hasRemoteEndpoint(config.embedding ?? {}))
+        return;
+    const endpoint = normalizeEmbeddingEndpoint(config.embedding?.endpoint ?? "");
+    const credential = describeEmbeddingCredential(config.embedding?.apiKey);
+    const configFileSuffix = isVerbose() ? `; config: ${getConfigPath()}` : "";
+    onProgress(`[embed] endpoint ${endpoint}, model ${config.embedding?.model ?? "unknown"}; credential: ${credential}${configFileSuffix}`);
+}
+function formatDoneLine(counts) {
+    return (`[drain] done: ${counts.pending} pending, ${counts.embedded} embedded, ${counts.failed} failed, ` +
+        `${counts.skipped} skipped (identity: ${counts.identity ?? "unknown"})`);
+}
+export async function drainEmbeddingQueue(db, config, opts = {}) {
+    throwIfAborted(opts.signal);
+    let identity = getMeta(db, "embeddingIdentity") ?? null;
+    if (config.semanticSearchMode === "off") {
+        return { pending: 0, embedded: 0, failed: 0, skipped: 0, identity };
+    }
+    const candidateHashes = opts.onlyHashes ? [...new Set(opts.onlyHashes)] : selectAllUnitHashes(db);
+    const missingHashes = identity ? listMissingHashes(db, candidateHashes, identity) : candidateHashes;
+    const pending = missingHashes.length;
+    const emitDone = (counts) => {
+        opts.onProgress?.(formatDoneLine(counts));
+        return counts;
+    };
+    // upsertUnitVectors is a no-op without sqlite-vec (units-repository.ts), so
+    // embedding the pending set here would just throw every vector away and
+    // leave it "missing" again for the next call — pure wasted provider
+    // traffic. `akmIndex`'s verification reports the missing extension as
+    // blocked; this stays silent. `pending` is computed above so the done line and `akm
+    // index status` stay truthful even though nothing was attempted.
+    if (!isVecAvailable(db)) {
+        return emitDone({ pending, embedded: 0, failed: 0, skipped: pending, identity });
+    }
+    if (pending === 0) {
+        return emitDone({ pending: 0, embedded: 0, failed: 0, skipped: 0, identity });
+    }
+    const boundedHashes = opts.limit !== undefined ? missingHashes.slice(0, opts.limit) : missingHashes;
+    const textByHash = fetchUnitTexts(db, boundedHashes);
+    // A hash in `unit_texts` should always resolve to a row (it was just read
+    // from that same table above), but a missing row is dropped rather than
+    // sent to the provider as `undefined` text.
+    const orderedHashes = boundedHashes.filter((hash) => textByHash.has(hash));
+    const texts = orderedHashes.map((hash) => textByHash.get(hash));
+    if (texts.length === 0) {
+        return emitDone({ pending, embedded: 0, failed: 0, skipped: 0, identity });
+    }
+    emitCredentialDiagnostic(config, opts.onProgress);
+    const { embeddingConfig, packing } = await resolveEmbeddingPacking(config, opts.signal);
+    let embedded = 0;
+    let failed = 0;
+    let batchNumber = 0;
+    // Two independent circuit-breaker streaks (single-document failures,
+    // network errors at any size) — see CIRCUIT_BREAKER_WINDOW above.
+    const singleDocFailureWindow = [];
+    const networkErrorFailureWindow = [];
+    // Whether this CALL has already decided the identity its first committed
+    // row observed (E1) — adoption happens at most once per call; see the
+    // module doc comment and the identity block in `onBatch` below.
+    let identityDecidedThisCall = false;
+    const onSkip = (skip) => {
+        failed++;
+        if (!skip.batchStart)
+            return undefined;
+        if (skip.reason === "context-window-exceeded") {
+            // Proves the provider IS reachable; not evidence of a dead endpoint.
+            singleDocFailureWindow.length = 0;
+            networkErrorFailureWindow.length = 0;
+            return undefined;
+        }
+        pushBreakerOutcome(singleDocFailureWindow, skip.batchSize === 1);
+        pushBreakerOutcome(networkErrorFailureWindow, skip.failureKind === "network-error");
+        if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD ||
+            breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
+            return false;
+        }
+        return undefined;
+    };
+    const onBatch = (indices, embeddings, model, outcome) => {
+        // "retrying"/"budget-lowered" are in-flight notices for a batch that has
+        // not settled yet (see EmbeddingBatchOutcome) — nothing to commit or
+        // count, and not a distinct "batch" for the one-line-per-batch contract.
+        if (outcome?.outcome === "retrying" || outcome?.outcome === "budget-lowered")
+            return;
+        const rows = [];
+        for (let k = 0; k < indices.length; k++) {
+            const embedding = embeddings[k];
+            if (!embedding)
+                continue;
+            const learned = deriveObservedEmbeddingIdentity(config.embedding, model, embedding.length);
+            if (!identityDecidedThisCall) {
+                // This call's FIRST committed row decides the identity it adopts
+                // (E1) — learned once, not re-derived per batch: a provider whose
+                // responses alternate between models WITHIN one call (a
+                // load-balanced gateway, a blue/green rollout behind one endpoint)
+                // must not thrash the store between them (embed, delete, re-embed,
+                // never converging). A genuine model change is still caught, just
+                // not until the NEXT call's own first batch observes it and purges
+                // whatever this call left behind.
+                identityDecidedThisCall = true;
+                if (learned && learned !== identity) {
+                    identity = learned;
+                    setMeta(db, "embeddingIdentity", identity);
+                    dropOtherIdentities(db, identity, embedding.length);
+                }
+            }
+            const currentIdentity = identity;
+            if (currentIdentity === null || learned !== currentIdentity) {
+                // Either nothing has ever been learned, or a LATER batch this same
+                // call reported an identity different from the one already adopted
+                // — left missing rather than switched to; it becomes "missing"
+                // again under whatever identity this call is using, and a later
+                // call, whose own first batch observes it, picks it up. Counted in
+                // `skipped` below (attempted minus embedded minus failed), not
+                // `embedded`.
+                continue;
+            }
+            const hash = orderedHashes[indices[k]];
+            if (hash)
+                rows.push({ hash, identity: currentIdentity, vector: embedding });
+        }
+        let storageBreakerTripped = false;
+        if (rows.length > 0) {
+            // upsertUnitVectors commits each row in its own transaction — this IS
+            // "each provider batch commits durably" (a wrapping db.transaction()
+            // here would only nest as an unobservable SAVEPOINT inside it, per the
+            // ambient-transaction hazard materialize-embeddings.ts's own drift
+            // guard documents), now made even finer-grained so one malformed
+            // vector in a batch (e.g. a width mismatch) can't roll back the rest
+            // of an otherwise-good response.
+            const result = upsertUnitVectors(db, rows);
+            embedded += result.inserted;
+            failed += result.failed;
+            if (result.failed > 0) {
+                // E5b: a write failure is just as much evidence of a broken run as
+                // a provider failure — a sustained STORAGE failure (contention,
+                // permissions, a full disk) must not keep dispatching every
+                // remaining batch to a perfectly healthy provider at full cost
+                // while every write silently fails. One event per committed batch
+                // (the "count batch starts, not documents" rule onSkip already
+                // applies to provider failures), fed into the SAME two streaks.
+                pushBreakerOutcome(singleDocFailureWindow, true);
+                pushBreakerOutcome(networkErrorFailureWindow, true);
+                if (breakerFailureCount(singleDocFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD ||
+                    breakerFailureCount(networkErrorFailureWindow) >= CIRCUIT_BREAKER_THRESHOLD) {
+                    storageBreakerTripped = true;
+                }
+            }
+        }
+        if (embeddings.some((embedding) => embedding !== undefined)) {
+            pushBreakerOutcome(singleDocFailureWindow, false);
+            pushBreakerOutcome(networkErrorFailureWindow, false);
+        }
+        batchNumber++;
+        if (opts.onProgress) {
+            const docCount = outcome?.docCount ?? indices.length;
+            const label = outcome && outcome.outcome !== "stored" ? `failed: ${outcome.reason ?? "unknown"}` : `${rows.length} stored`;
+            opts.onProgress(`${DRAIN_BATCH_PROGRESS_PREFIX}${batchNumber}: ${docCount} docs → ${label}`);
+        }
+        if (storageBreakerTripped) {
+            // onBatch has no `false`-return stop-dispatch contract the way onSkip
+            // does (a storage failure can trip this even when the provider itself
+            // keeps succeeding, so onSkip is never called at all) — this reuses
+            // RemoteEmbedder's own documented mechanism instead: a throw from
+            // onBatch stops the pool from dispatching any further provider
+            // request, and is rethrown once every in-flight batch has settled.
+            throw new Error(`Circuit breaker: ${CIRCUIT_BREAKER_THRESHOLD} storage write failures while embedding; stopping further provider requests this call.`);
+        }
+    };
+    await embedBatch(texts, embeddingConfig, opts.signal, onSkip, onBatch, packing);
+    throwIfAborted(opts.signal);
+    const attempted = texts.length;
+    const skipped = Math.max(0, attempted - embedded - failed);
+    return emitDone({ pending, embedded, failed, skipped, identity });
+}