akm-cli 0.9.15-beta.1 → 0.9.15-beta.3

package/dist/indexer/indexer.js

@@ -7,18 +7,21 @@ import { detectAdapterId } from "../core/adapter/detect-adapter.js";
 import { adapterForId } from "../core/adapter/registry.js";
 import { isHttpUrl, toErrorMessage } from "../core/common.js";
 import { concurrentMap } from "../core/concurrent.js";
-import { ConfigError } from "../core/errors.js";
+import { AkmError, ConfigError, TransientError } from "../core/errors.js";
+import { probeLock } from "../core/file-lock.js";
 import { defaultConcurrencyForEndpoint } from "../core/loopback.js";
 import { classifyPathAccess, describeInaccessiblePath } from "../core/path-access.js";
 import { getDbPath } from "../core/paths.js";
 import { SCRIPT_EXTENSIONS } from "../core/recognition-util.js";
-import { withStateDb } from "../core/state-db.js";
+import { formatLockHolderPid } from "../core/run-lock.js";
+import { isSqliteContentionError, withStateDb } from "../core/state-db.js";
 import { isVerbose, warn, warnOnce, warnVerbose } from "../core/warn.js";
 import { disposeLoweredExecutionDispatchLease, } from "../integrations/agent/execution-lowering.js";
 import { isLlmFeatureEnabled } from "../llm/feature-gate.js";
 import { resolveIndexPassExecution } from "../llm/index-passes.js";
 import { preflightStructuredLlmRunner } from "../llm/structured-call.js";
 import { resolveSourcesForOrigin } from "../registry/origin-resolve.js";
+import { salvageEmbeddingsBeforeDiscard } from "../storage/repositories/embedding-salvage-repository.js";
 import { closeDatabase, openExistingDatabase, openIndexDatabase, openReadonlyExistingDatabase, } from "../storage/repositories/index-connection.js";
 import { deleteAllEntries, deleteEntriesByBundle, deleteEntriesByDirAndBundle, deleteEntriesByDirExceptRefs, deleteEntriesByIds, deleteUsageEventsByEntryIds, findEntryIdByRef, getAllEntries, getEmbeddableEntryCount, getEntryCount, getIndexedBundleIdsByDir, getIndexedDirPathsByBundleId, relinkUsageEvents, upsertEntry, } from "../storage/repositories/index-entries-repository.js";
 import { clearStaleCacheEntries, computeBodyHash, getLlmCacheEntry, } from "../storage/repositories/index-llm-cache-repository.js";
@@ -27,6 +30,7 @@ import { upsertUtilityScore } from "../storage/repositories/index-utility-reposi
 import { getEmbeddingCount, isVecAvailable, isVecFastPathReady, warnIfVecMissing, } from "../storage/repositories/index-vec-repository.js";
 import { assertIndexedWorkflowSourceIdentity, WorkflowSourceIdentityError } from "../workflows/source-files.js";
 import { deleteStoredGraph } from "./db/graph-db.js";
+import { indexRebuildLockPath } from "./index-rebuild-lock.js";
 import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
 import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
 import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
@@ -184,21 +188,54 @@ async function runWalkPhase(ctx) {
     });
     ctx.timing.tLlmEnd = Date.now();
 }
+/**
+ * The ONE embedding-phase implementation (#954): generate and
+ * store vectors for every entry missing one, then compute the `hasEmbeddings`
+ * fact and the semantic-search verification off the result. `akmIndex`'s own
+ * (non-deferred) run calls this from {@link runEmbeddingPhase} below; `akm
+ * bundle update`'s coordinator calls it directly on its own connection AFTER
+ * its unified update transaction commits, since the ambient-transaction drift
+ * guard (and the whole point of per-batch commit, #954) requires `db` to have
+ * no ambient transaction open.
+ */
+export async function runEmbeddingPass(params) {
+    const { db, config, onProgress, signal, reembed } = params;
+    const embeddingResult = await generateEmbeddingsForDb(db, config, onProgress, signal, undefined, {
+        forceReembed: reembed,
+    });
+    setMeta(db, "hasEmbeddings", embeddingResult.success ? "1" : "0");
+    const semanticEntryCount = getEmbeddableEntryCount(db);
+    onProgress({ phase: "finalize", message: "Verifying semantic search state." });
+    const verification = verifyIndexState(db, config, semanticEntryCount, embeddingResult);
+    onProgress({ phase: "verify", message: verification.message });
+    return { embeddingResult, verification };
+}
 /**
  * Embedding phase: generate and store vector embeddings for all unembedded
- * entries. Writes `ctx.embeddingResult` for the finalize phase.
+ * entries. Writes `ctx.embeddingResult` and `ctx.verification` for the
+ * finalize phase / caller.
  */
 async function runEmbeddingPhase(ctx) {
-    const { db, config, signal, onProgress, reembed } = ctx;
+    const { db, config, signal, onProgress, reembed, deferredUpdateTransaction } = ctx;
     throwIfAborted(signal);
+    if (deferredUpdateTransaction) {
+        // `akm bundle update`'s deferred pass (#954): the embedding
+        // phase runs AFTER the coordinator's own commit, on its own connection,
+        // via the coordinator's direct `runEmbeddingPass` call — never here,
+        // inside the borrowed transaction (the ambient-transaction drift guard
+        // would reject it anyway). `runFinalizePhase` records semantic state as
+        // "pending".
+        ctx.timing.tEmbedEnd = Date.now();
+        return;
+    }
     // Forward the signal. Without it generateEmbeddingsForDb's abort machinery was
     // inert — its throwIfAborted checks and the signal it threads into embedBatch
     // (which RemoteEmbedder passes to every fetch and LocalEmbedder honours between
     // chunks) never saw a controller. Ctrl-C and the improve budget abort could not
     // stop the embedding phase, the longest phase of an index run.
-    ctx.embeddingResult = await generateEmbeddingsForDb(db, config, onProgress, signal, undefined, {
-        forceReembed: reembed,
-    });
+    const { embeddingResult, verification } = await runEmbeddingPass({ db, config, onProgress, signal, reembed });
+    ctx.embeddingResult = embeddingResult;
+    ctx.verification = verification;
     ctx.timing.tEmbedEnd = Date.now();
 }
 /**
@@ -206,8 +243,8 @@ async function runEmbeddingPhase(ctx) {
  * usage events, recompute utility scores, update index metadata, and emit the
  * verify event.
  */
-async function runFinalizePhase(ctx, deferredUpdateTransaction) {
-    const { db, config, sources, sourceDirs, stashDir, signal, onProgress } = ctx;
+async function runFinalizePhase(ctx) {
+    const { db, config, sources, sourceDirs, stashDir, signal, onProgress, deferredUpdateTransaction } = ctx;
     ctx.timing.tFinalizeStart = Date.now();
     // `upsertEntry` and every canonical delete own their FTS projection. This is
     // an observation point, not a second materialization pass.
@@ -249,22 +286,37 @@ async function runFinalizePhase(ctx, deferredUpdateTransaction) {
     // An incomplete run preserves the prior freshness watermark. Advancing it
     // could make a recovered source look unchanged even though this run never
     // persisted its files.
-    const embeddingResult = ctx.embeddingResult ?? { success: false };
     if (ctx.scanComplete) {
         setMeta(db, "builtAt", new Date().toISOString());
         setMeta(db, "stashDir", stashDir);
         setMeta(db, "stashDirs", JSON.stringify(sourceDirs));
         setMeta(db, "sourceOwners", JSON.stringify(sourceOwners(sources)));
     }
-    setMeta(db, "hasEmbeddings", embeddingResult.success ? "1" : "0");
     warnIfVecMissing(db);
     const totalEntries = getEntryCount(db);
-    const semanticEntryCount = getEmbeddableEntryCount(db);
-    onProgress({ phase: "finalize", message: "Verifying semantic search state." });
-    const verification = verifyIndexState(db, config, semanticEntryCount, embeddingResult);
-    onProgress({ phase: "verify", message: verification.message });
-    // Store verification result and totalEntries on ctx for the caller to use
-    ctx.verification = verification;
+    if (deferredUpdateTransaction) {
+        // #954: the embedding phase was skipped for this borrowed
+        // transaction — record semantic state as pending, never ready, until the
+        // coordinator's own post-commit `runEmbeddingPass` call reports the
+        // truth on a fresh connection.
+        setMeta(db, "hasEmbeddings", "0");
+        const semanticEntryCount = getEmbeddableEntryCount(db);
+        const message = "Semantic index update deferred until after the source-update commit.";
+        onProgress({ phase: "verify", message });
+        ctx.verification = {
+            ok: true,
+            message,
+            semanticSearchEnabled: config.semanticSearchMode === "auto",
+            semanticSearchMode: config.semanticSearchMode,
+            semanticStatus: config.semanticSearchMode === "off" ? "disabled" : "pending",
+            embeddingProvider: getEmbeddingProvider(config.embedding),
+            entryCount: semanticEntryCount,
+            embeddingCount: getEmbeddingCount(db),
+            vecAvailable: isVecAvailable(db),
+        };
+    }
+    // Non-deferred: ctx.verification was already populated by runEmbeddingPhase
+    // (via the shared runEmbeddingPass).
     ctx.totalEntries = totalEntries;
     ctx.timing.tFinalizeEnd = Date.now();
     // suppress unused warning — sources was previously used inline
@@ -322,6 +374,44 @@ let akmIndexOverride;
 export function _setAkmIndexForTests(fake) {
     akmIndexOverride = fake;
 }
+/**
+ * Read-only description of the rebuild lock's current holder, appended to a
+ * reclassified index.db contention message when known (field follow-up to
+ * #956). `probeLock` only inspects the sentinel — it never acquires or
+ * mutates it — so this is safe to call from inside an error path.
+ */
+function describeIndexRebuildLockHolder() {
+    const probe = probeLock(indexRebuildLockPath());
+    if (probe.state !== "held")
+        return "";
+    return ` The rebuild lock is currently held by pid ${formatLockHolderPid({
+        pid: probe.holderPid,
+        launcherPid: probe.launcherPid ?? null,
+    })}.`;
+}
+/**
+ * Reclassify a contention-shaped error escaping the walk, index, or
+ * embedding phase into a retryable-shortly `TransientError` (field
+ * follow-up to #956, dev-team field review 2026-09-10): a concurrent writer
+ * (another `akm index`, a source-update embedding pass, the per-command
+ * background reindex) can make index.db busy, and the raw SQLite driver
+ * error ("database is locked") used to escape as exit 70
+ * (internal/unclassified) instead of the "retry shortly" contract exit 75
+ * gives a scheduler to branch on — mirroring `STATE_DB_CONTENDED`'s
+ * precedent for state.db (`core/state-db.ts`). Reuses the ONE shared
+ * classifier, `isSqliteContentionError`, rather than a second one. An error
+ * that is already a classified akm error (e.g. a `STATE_DB_CONTENDED`
+ * TransientError from an inner state.db write) is never re-wrapped — only a
+ * raw, unclassified error matching the shared contention shape is
+ * reclassified. Every other error is rethrown unchanged.
+ */
+export function reclassifyIndexDbContention(error) {
+    if (error instanceof AkmError || !isSqliteContentionError(error))
+        return error;
+    const contended = new TransientError(`akm's index database is busy (another akm process is writing it); retry shortly.${describeIndexRebuildLockHolder()}`, "INDEX_DB_CONTENDED");
+    contended.cause = error;
+    return contended;
+}
 export async function akmIndex(options) {
     try {
         const override = akmIndexOverride;
@@ -338,7 +428,7 @@ export async function akmIndex(options) {
                 // rollback before closing its borrowed unified handle.
             }
         }
-        throw error;
+        throw reclassifyIndexDbContention(error);
     }
 }
 let indexTransactionHookForTests;
@@ -513,6 +603,11 @@ async function akmIndexReal(options) {
             force: full,
             materialize: options.hydrateSources !== false,
             secrets: storeSecretResolver,
+            // Same progress channel as every other phase (#954) — a
+            // stalled clone/fetch here runs BEFORE index.db is even opened, so
+            // without this it looked identical to "no database open, nothing
+            // written".
+            onProgress: (message) => onProgress({ phase: "preflight", message }),
         });
         const sourceCacheEnd = Date.now();
         const allSourceEntries = resolveSourceEntries(stashDir, config);
@@ -553,6 +648,7 @@ async function akmIndexReal(options) {
                 onProgress,
                 signal,
                 t0,
+                deferredUpdateTransaction: options.deferredUpdateTransaction,
             });
             indexRunContext = ctx;
             onProgress({
@@ -592,7 +688,7 @@ async function akmIndexReal(options) {
             }
             cleanEnd = Date.now();
             await runEmbeddingPhase(ctx);
-            await runFinalizePhase(ctx, options.deferredUpdateTransaction);
+            await runFinalizePhase(ctx);
             // ────────────────────────────────────────────────────────────────────────
             // runFinalizePhase always populates these before returning.
             const verification = ctx.verification;
@@ -1159,6 +1255,14 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
         // transaction so delete and re-insert are atomic — a concurrent reader
         // never observes an empty database between the two operations.
         if (fullDelete) {
+            // #955: copy every (search_text hash, embedding) pair about to be
+            // discarded wholesale into `embedding_salvage`, tagged with the
+            // fingerprint the discarded vectors were generated under, BEFORE the
+            // wipe below — inside the SAME transaction so the copy and the
+            // discard commit or roll back together. The embedding phase later in
+            // this run hands salvaged vectors back to unchanged content instead
+            // of re-embedding the whole corpus.
+            salvageEmbeddingsBeforeDiscard(db);
             // Entries and every child materialization share one deletion authority.
             // Usage events live in state.db and survive so finalize can relink them
             // to the replacement generation's row ids.