akm-cli 0.9.15-beta.2 → 0.9.15-beta.4

package/dist/llm/embedders/remote.js

@@ -30,10 +30,17 @@ export const DEFAULT_REMOTE_BATCH_SIZE = 100;
  * a batch of 100 small docs (~400 KB, ~100K tokens) took 14.8s against a
  * healthy local endpoint — half the 30s request timeout — and a single
  * 128 KB (~24K token) document alone was rejected by the endpoint as
- * exceeding its context size. 8000 tokens keeps a batch's estimated size
- * comfortably under both the timeout and common local-model context windows.
+ * exceeding its context size.
+ *
+ * Lowered from 8000 to 6000 (#954, field report on beta.1): the 4-chars-
+ * per-token estimator undercounts dense technical text by 7-55%, so 8000
+ * against an 8192-token llama.cpp embedder regularly landed real requests
+ * over the endpoint's context window. 6000 is the value the field confirmed
+ * stops that steady trickle of rejections; `embedBatch`'s run-scoped
+ * adaptive budget below still shrinks further, for an endpoint where even
+ * this is not enough.
  */
-export const DEFAULT_TOKEN_BUDGET = 8000;
+export const DEFAULT_TOKEN_BUDGET = 6000;
 /** Cheap token estimator: 4 chars ≈ 1 token. Used in verbose logging and error messages. */
 export function estimateTokenCount(text) {
     return Math.round(text.length / 4);
@@ -73,7 +80,7 @@ export function capEmbeddingText(text, maxTokens) {
 /**
  * Default per-request timeout when `embedding.timeoutMs` is unset (#954).
  * The prior fixed 30s cut off exactly the field-report case: a
- * local model server on an 8000-token (`DEFAULT_TOKEN_BUDGET`) batch
+ * local model server on a full-budget (`DEFAULT_TOKEN_BUDGET`) batch
  * legitimately takes longer than that, and the timeout fired mid-response
  * with no retry — every batch it hit was silently dropped for the rest of
  * an hours-long run. 120s comfortably covers a slow local batch while still
@@ -235,6 +242,21 @@ export function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
     flush();
     return batches;
 }
+/**
+ * Shrink factor applied to the effective request budget on the first
+ * context-size rejection of an `embedBatch` run (#954, field report on
+ * beta.1): one 25% cut absorbs the estimator's measured undercount without
+ * repeatedly re-shrinking mid-run — see the "shrink at most once" rule on
+ * {@link RemoteEmbedder.embedBatch}.
+ */
+const ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+/**
+ * Floor on the adaptive-budget shrink above, as a multiple of
+ * `maxInputTokens` (#954): a request budget below twice the per-document cap
+ * could no longer batch more than one document per request, defeating the
+ * point of batching at all.
+ */
+const ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 export class RemoteEmbedder {
     config;
     endpoint;
@@ -339,6 +361,18 @@ export class RemoteEmbedder {
      * {@link scaleEmbeddingTimeoutMs}, so a dead server is detected in seconds
      * on a small batch rather than always waiting out the full configured
      * `embedding.timeoutMs`.
+     *
+     * Run-scoped adaptive budget (#954, field report on beta.1): the FIRST
+     * context-size rejection of the run shrinks the effective request budget
+     * by {@link ADAPTIVE_BUDGET_SHRINK_FACTOR} (floored at
+     * {@link ADAPTIVE_BUDGET_FLOOR_MULTIPLIER} times `maxInputTokens`) for
+     * every batch not yet dispatched — the still-planned tail of `texts` is
+     * re-batched with `buildTokenBoundedBatches` at the smaller budget, and a
+     * `budget-lowered` `onBatch` event reports it once. This never touches the
+     * split-and-retry of the rejected batch itself (above), and never fires a
+     * second time in the same run even if a later batch is also rejected — a
+     * static configured budget that is simply too big for the endpoint should
+     * self-correct once, not ratchet down forever.
      */
     async embedBatch(texts, signal, onSkip, onBatch) {
         if (texts.length === 0)
@@ -351,10 +385,59 @@ export class RemoteEmbedder {
         // request budget too, so a config author setting it for one purpose
         // silently changed the other. `maxTokens` is the sole knob for the
         // request budget now; unset falls back to DEFAULT_TOKEN_BUDGET.
-        const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+        //
+        // `effectiveTokenBudget` (#954) starts at the configured/default value
+        // and MAY shrink once, on the run's first context-size rejection — see
+        // `maybeShrinkBudget` below. `textBatches` is mutated in place (spliced)
+        // by that shrink rather than reassigned, so the in-flight
+        // `concurrentMap` pool below (which reads this same array by reference)
+        // picks up the re-planned tail without restarting.
+        let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
         const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-        const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+        const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+        const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
         const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+        // How many of `textBatches` concurrentMap has already claimed (its own
+        // `nextIndex`, mirrored here so a budget shrink knows where the
+        // not-yet-dispatched tail begins). Assigned, not incremented, at the top
+        // of `runProviderBatch` — batches are claimed in strictly increasing
+        // order, so the highest `batchIndex` seen so far IS the claimed count.
+        let dispatchedBatchCount = 0;
+        // Set once the run's first context-size rejection has shrunk the budget
+        // (#954) — guards `maybeShrinkBudget` so it never fires twice.
+        let budgetShrunk = false;
+        // On the FIRST context-size rejection of this `embedBatch` call, shrink
+        // `effectiveTokenBudget` and re-plan every batch `concurrentMap` has not
+        // yet claimed from the smaller budget. Never touches `rejectedIndices`
+        // itself — the caller's own split-and-retry handles that batch — and is
+        // a no-op after the first call (`budgetShrunk`).
+        const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+            if (budgetShrunk)
+                return;
+            budgetShrunk = true;
+            const floor = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+            effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor);
+            const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+            const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+            if (remainingIndices.length > 0) {
+                const remainingTexts = remainingIndices.map((i) => texts[i]);
+                const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+                    indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+                    oversized: batch.oversized,
+                }));
+                textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+            }
+            warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+            commitBatch(rejectedIndices, rejectedIndices.map(() => undefined), undefined, {
+                batchIndex: rejectedBatchIndex,
+                batchCount: textBatches.length,
+                docCount: rejectedIndices.length,
+                requestTokens: rejectedRequestTokens,
+                elapsedMs: 0,
+                outcome: "budget-lowered",
+                reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`,
+            });
+        };
         // Stops the pool from claiming any FURTHER provider batch once the
         // caller's onBatch has failed once (the materializer's transaction
         // failed, so a subsequent commit would just fail again) — dispatching
@@ -437,7 +520,7 @@ export class RemoteEmbedder {
                 return;
             const batch = indices.map((i) => texts[i]);
             const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-            const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+            const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
             const requestStart = Date.now();
             let batchEmbeddings;
             let responseModel;
@@ -457,6 +540,15 @@ export class RemoteEmbedder {
                 // failed" condition, it means stop entirely.
                 if (signal?.aborted)
                     throw err;
+                if (err instanceof ContextExceededError) {
+                    // #954: the run's FIRST context-size rejection (any size) shrinks
+                    // the budget for everything not yet dispatched; a no-op after the
+                    // first call. Deliberately BEFORE the split below — it must fire
+                    // for a single-document rejection too (which never reaches the
+                    // `indices.length > 1` split branch), and it never touches this
+                    // batch's own split-and-retry.
+                    maybeShrinkBudget(indices, batchIndex, requestTokens);
+                }
                 if (err instanceof ContextExceededError && indices.length > 1) {
                     const mid = Math.ceil(indices.length / 2);
                     await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -554,13 +646,19 @@ export class RemoteEmbedder {
             });
         };
         const runProviderBatch = async (textBatch, batchIndex) => {
+            // Claimed in strictly increasing order by `concurrentMap` below, so
+            // the highest `batchIndex` seen so far is exactly how many batches it
+            // has claimed (#954) — see `maybeShrinkBudget`'s doc comment above.
+            // Assigned synchronously at entry, before any `await`, so this always
+            // matches `concurrentMap`'s own `nextIndex` at the moment of claim.
+            dispatchedBatchCount = batchIndex;
             if (textBatch.oversized) {
                 const idx = textBatch.indices[0];
                 const estTokens = estimateTokenCount(texts[idx]);
                 onSkip?.({
                     index: idx,
                     reason: "context-window-exceeded",
-                    message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+                    message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
                     batchStart: true,
                     batchSize: 1,
                 });

package/dist/llm/usage-persist.js

@@ -67,12 +67,18 @@ function toEventMetadata(record) {
  * opened for its other events. A getter is resolved for every append so a
  * caller can replace its context binding without replacing this owning sink.
  * When omitted, `appendEvent` falls back to its default open-insert-close path.
+ *
+ * `onRecord`, when supplied, runs synchronously for every terminal record
+ * BEFORE persistence — improve's first-engine-response heartbeat (#957) uses
+ * it to know the run is no longer silent, without this module taking on any
+ * dependency of its own on improve's lifecycle.
  */
-export function installLlmUsagePersistence(ctx) {
+export function installLlmUsagePersistence(ctx, onRecord) {
     let expectedTerminalRecords = 0;
     let disposed = false;
     setLlmUsageSink((record) => {
         expectedTerminalRecords += 1;
+        onRecord?.(record);
         appendEvent({ eventType: LLM_USAGE_EVENT, metadata: toEventMetadata(record) }, typeof ctx === "function" ? ctx() : ctx);
     });
     return () => {

package/dist/scripts/akm-migrate-node.js

@@ -7070,7 +7070,9 @@ var init_errors = __esm(() => {
   };
   TRANSIENT_HINTS = {
     RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
-    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing."
+    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
+    INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs.",
+    MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on scheduled index/improve/workflow runs."
   };
   NOT_FOUND_HINTS = {
     ASSET_NOT_FOUND: "Run `akm search <query>` or `akm index` to refresh the index.",
@@ -75920,7 +75922,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
   maxInputTokens: positiveInt.optional(),
   maxTokens: positiveInt.optional(),
   batchSize: positiveInt.optional(),
-  chunkSize: positiveInt.optional(),
   contextLength: positiveInt.optional(),
   ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
   timeoutMs: positiveInt.optional(),
@@ -77923,10 +77924,13 @@ import { AsyncLocalStorage } from "node:async_hooks";
 import { randomUUID as randomUUID2 } from "node:crypto";
 import fs7 from "node:fs";
 import path12 from "node:path";
+init_common();
 init_errors();
 init_paths();
 var heldBarrierContext = new AsyncLocalStorage;
 var MAINTENANCE_BARRIER_STALE_AFTER_MS = 5 * 60 * 1000;
+var MAINTENANCE_BARRIER_BUSY_RETRY_BOUND_MS = 1500;
+var busyRetryBoundMsForTests;
 function tryAcquireMaintenanceBarrier() {
   const lockPath = getMaintenanceBarrierPath();
   fs7.mkdirSync(path12.dirname(lockPath), { recursive: true });
@@ -77942,10 +77946,18 @@ function tryAcquireMaintenanceBarrier() {
   return;
 }
 function acquireMaintenanceBarrier() {
-  const release = tryAcquireMaintenanceBarrier();
-  if (release)
-    return release;
-  throw new ConfigError(`AKM maintenance is in progress (barrier ${getMaintenanceBarrierPath()}); retry after it completes. ` + `A sentinel older than ${MAINTENANCE_BARRIER_STALE_AFTER_MS / 60000} minute(s) is reclaimed automatically on the next attempt.`, "INVALID_CONFIG_FILE");
+  const boundMs = busyRetryBoundMsForTests ?? MAINTENANCE_BARRIER_BUSY_RETRY_BOUND_MS;
+  const deadline = Date.now() + boundMs;
+  for (let attempt = 0;; attempt += 1) {
+    const release = tryAcquireMaintenanceBarrier();
+    if (release)
+      return release;
+    const remainingMs = deadline - Date.now();
+    if (remainingMs <= 0)
+      break;
+    sleepSync(Math.min(backoffDelay(attempt), remainingMs));
+  }
+  throw new TransientError(`AKM maintenance is in progress (barrier ${getMaintenanceBarrierPath()}); retry shortly. ` + `A sentinel older than ${MAINTENANCE_BARRIER_STALE_AFTER_MS / 60000} minute(s) is reclaimed automatically on the next attempt.`, "MAINTENANCE_BARRIER_BUSY");
 }
 function withMaintenanceStartBarrier(run) {
   if (heldBarrierContext.getStore()?.active)
@@ -96987,7 +96999,7 @@ function defaultConcurrencyForEndpoint(endpoint) {
 // src/llm/embedders/remote.ts
 init_warn();
 var DEFAULT_REMOTE_BATCH_SIZE = 100;
-var DEFAULT_TOKEN_BUDGET = 8000;
+var DEFAULT_TOKEN_BUDGET = 6000;
 function estimateTokenCount(text) {
   return Math.round(text.length / 4);
 }
@@ -97069,6 +97081,8 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
   flush();
   return batches;
 }
+var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+var ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 
 class RemoteEmbedder {
   config;
@@ -97121,10 +97135,42 @@ class RemoteEmbedder {
     const results = new Array(texts.length).fill(undefined);
     const headers = this.buildHeaders();
     const ollamaOpts = resolveOllamaOptions(this.config);
-    const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+    let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
     const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-    const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+    const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+    const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
     const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+    let dispatchedBatchCount = 0;
+    let budgetShrunk = false;
+    const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+      if (budgetShrunk)
+        return;
+      budgetShrunk = true;
+      const floor2 = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+      effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
+      const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+      const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+      if (remainingIndices.length > 0) {
+        const remainingTexts = remainingIndices.map((i) => texts[i]);
+        const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+          indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+          oversized: batch.oversized
+        }));
+        textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+      }
+      warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+      commitBatch(rejectedIndices, rejectedIndices.map(() => {
+        return;
+      }), undefined, {
+        batchIndex: rejectedBatchIndex,
+        batchCount: textBatches.length,
+        docCount: rejectedIndices.length,
+        requestTokens: rejectedRequestTokens,
+        elapsedMs: 0,
+        outcome: "budget-lowered",
+        reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`
+      });
+    };
     const dispatchAbort = new AbortController;
     const stopDispatch = (reason) => {
       if (!dispatchAbort.signal.aborted)
@@ -97157,7 +97203,7 @@ class RemoteEmbedder {
         return;
       const batch = indices.map((i) => texts[i]);
       const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
       const requestStart = Date.now();
       let batchEmbeddings;
       let responseModel;
@@ -97174,6 +97220,9 @@ class RemoteEmbedder {
       } catch (err) {
         if (signal?.aborted)
           throw err;
+        if (err instanceof ContextExceededError) {
+          maybeShrinkBudget(indices, batchIndex, requestTokens);
+        }
         if (err instanceof ContextExceededError && indices.length > 1) {
           const mid = Math.ceil(indices.length / 2);
           await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -97239,13 +97288,14 @@ class RemoteEmbedder {
       });
     };
     const runProviderBatch = async (textBatch, batchIndex) => {
+      dispatchedBatchCount = batchIndex;
       if (textBatch.oversized) {
         const idx = textBatch.indices[0];
         const estTokens = estimateTokenCount(texts[idx]);
         onSkip?.({
           index: idx,
           reason: "context-window-exceeded",
-          message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+          message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
           batchStart: true,
           batchSize: 1
         });
@@ -97469,7 +97519,7 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
   }
   return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
 }
-async function runEmbeddingCanary(db, config, signal) {
+async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
   const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
   if (samples.length === 0) {
     return { outcome: "keep", verified: false, viaIdentityMatch: false };
@@ -97478,7 +97528,7 @@ async function runEmbeddingCanary(db, config, signal) {
   const skips = [];
   let canaryVectors;
   try {
-    canaryVectors = await embedBatch(samples.map((sample) => sample.searchText), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
+    canaryVectors = await embedBatch(samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
       if (model)
         observedModel = model;
     });
@@ -97548,6 +97598,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
   const storedFingerprint = getMeta(db, "embeddingFingerprint");
   let targetEntryIds = entryIds;
   let rebuildReason;
+  const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
   if (opts?.forceReembed) {
     db.transaction(() => {
       purgeEmbeddings(db, { dropVecTable: true });
@@ -97559,7 +97610,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
     targetEntryIds = undefined;
     rebuildReason = "forced by --reembed";
   } else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-    const decision = await runEmbeddingCanary(db, config, signal);
+    const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
     if (decision.outcome === "unverifiable") {
       warn(`[embed] ${decision.message}`);
       onProgress({ phase: "embeddings", message: decision.message });
@@ -97623,7 +97674,6 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
       purgeEmbeddingSalvage(db);
       return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
     }
-    const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     let truncatedCount = 0;
     const texts = [];
     const pendingEntries = [];
@@ -97718,6 +97768,15 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
           }
           return;
         }
+        if (outcome?.outcome === "budget-lowered") {
+          if (reportPerBatchLine) {
+            onProgress({
+              phase: "embeddings",
+              message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcome.reason}`
+            });
+          }
+          return;
+        }
         if (model)
           observedModel = model;
         if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
@@ -97726,7 +97785,8 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
         }
         db.transaction(() => {
           for (let k2 = 0;k2 < indices.length; k2++) {
-            const entry = pendingEntries[indices[k2]];
+            const index = indices[k2];
+            const entry = pendingEntries[index];
             if (!entry)
               continue;
             const embedding = batchEmbeddings[k2];
@@ -97739,7 +97799,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
             const result = upsertEmbedding(db, entry.id, embedding);
             if (result.stored) {
               storedCount++;
-              storedTokens += estimateTokenCount(entry.searchText);
+              storedTokens += estimateTokenCount(texts[index]);
             } else {
               skippedCount++;
             }

package/dist/scripts/akm-migrate.js

@@ -7069,7 +7069,9 @@ var init_errors = __esm(() => {
   };
   TRANSIENT_HINTS = {
     RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
-    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing."
+    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
+    INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs.",
+    MAINTENANCE_BARRIER_BUSY: "Another akm process is registering a lock or lease right now. Retry shortly, or pass --skip-if-locked on scheduled index/improve/workflow runs."
   };
   NOT_FOUND_HINTS = {
     ASSET_NOT_FOUND: "Run `akm search <query>` or `akm index` to refresh the index.",
@@ -75248,7 +75250,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
   maxInputTokens: positiveInt.optional(),
   maxTokens: positiveInt.optional(),
   batchSize: positiveInt.optional(),
-  chunkSize: positiveInt.optional(),
   contextLength: positiveInt.optional(),
   ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
   timeoutMs: positiveInt.optional(),
@@ -77251,10 +77252,13 @@ import { AsyncLocalStorage } from "async_hooks";
 import { randomUUID as randomUUID2 } from "crypto";
 import fs7 from "fs";
 import path12 from "path";
+init_common();
 init_errors();
 init_paths();
 var heldBarrierContext = new AsyncLocalStorage;
 var MAINTENANCE_BARRIER_STALE_AFTER_MS = 5 * 60 * 1000;
+var MAINTENANCE_BARRIER_BUSY_RETRY_BOUND_MS = 1500;
+var busyRetryBoundMsForTests;
 function tryAcquireMaintenanceBarrier() {
   const lockPath = getMaintenanceBarrierPath();
   fs7.mkdirSync(path12.dirname(lockPath), { recursive: true });
@@ -77270,10 +77274,18 @@ function tryAcquireMaintenanceBarrier() {
   return;
 }
 function acquireMaintenanceBarrier() {
-  const release = tryAcquireMaintenanceBarrier();
-  if (release)
-    return release;
-  throw new ConfigError(`AKM maintenance is in progress (barrier ${getMaintenanceBarrierPath()}); retry after it completes. ` + `A sentinel older than ${MAINTENANCE_BARRIER_STALE_AFTER_MS / 60000} minute(s) is reclaimed automatically on the next attempt.`, "INVALID_CONFIG_FILE");
+  const boundMs = busyRetryBoundMsForTests ?? MAINTENANCE_BARRIER_BUSY_RETRY_BOUND_MS;
+  const deadline = Date.now() + boundMs;
+  for (let attempt = 0;; attempt += 1) {
+    const release = tryAcquireMaintenanceBarrier();
+    if (release)
+      return release;
+    const remainingMs = deadline - Date.now();
+    if (remainingMs <= 0)
+      break;
+    sleepSync(Math.min(backoffDelay(attempt), remainingMs));
+  }
+  throw new TransientError(`AKM maintenance is in progress (barrier ${getMaintenanceBarrierPath()}); retry shortly. ` + `A sentinel older than ${MAINTENANCE_BARRIER_STALE_AFTER_MS / 60000} minute(s) is reclaimed automatically on the next attempt.`, "MAINTENANCE_BARRIER_BUSY");
 }
 function withMaintenanceStartBarrier(run) {
   if (heldBarrierContext.getStore()?.active)
@@ -96943,7 +96955,7 @@ function defaultConcurrencyForEndpoint(endpoint) {
 // src/llm/embedders/remote.ts
 init_warn();
 var DEFAULT_REMOTE_BATCH_SIZE = 100;
-var DEFAULT_TOKEN_BUDGET = 8000;
+var DEFAULT_TOKEN_BUDGET = 6000;
 function estimateTokenCount(text) {
   return Math.round(text.length / 4);
 }
@@ -97025,6 +97037,8 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
   flush();
   return batches;
 }
+var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+var ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 
 class RemoteEmbedder {
   config;
@@ -97077,10 +97091,42 @@ class RemoteEmbedder {
     const results = new Array(texts.length).fill(undefined);
     const headers = this.buildHeaders();
     const ollamaOpts = resolveOllamaOptions(this.config);
-    const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+    let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
     const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-    const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+    const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+    const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
     const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+    let dispatchedBatchCount = 0;
+    let budgetShrunk = false;
+    const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+      if (budgetShrunk)
+        return;
+      budgetShrunk = true;
+      const floor2 = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+      effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
+      const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+      const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+      if (remainingIndices.length > 0) {
+        const remainingTexts = remainingIndices.map((i) => texts[i]);
+        const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+          indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+          oversized: batch.oversized
+        }));
+        textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+      }
+      warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+      commitBatch(rejectedIndices, rejectedIndices.map(() => {
+        return;
+      }), undefined, {
+        batchIndex: rejectedBatchIndex,
+        batchCount: textBatches.length,
+        docCount: rejectedIndices.length,
+        requestTokens: rejectedRequestTokens,
+        elapsedMs: 0,
+        outcome: "budget-lowered",
+        reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`
+      });
+    };
     const dispatchAbort = new AbortController;
     const stopDispatch = (reason) => {
       if (!dispatchAbort.signal.aborted)
@@ -97113,7 +97159,7 @@ class RemoteEmbedder {
         return;
       const batch = indices.map((i) => texts[i]);
       const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
       const requestStart = Date.now();
       let batchEmbeddings;
       let responseModel;
@@ -97130,6 +97176,9 @@ class RemoteEmbedder {
       } catch (err) {
         if (signal?.aborted)
           throw err;
+        if (err instanceof ContextExceededError) {
+          maybeShrinkBudget(indices, batchIndex, requestTokens);
+        }
         if (err instanceof ContextExceededError && indices.length > 1) {
           const mid = Math.ceil(indices.length / 2);
           await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -97195,13 +97244,14 @@ class RemoteEmbedder {
       });
     };
     const runProviderBatch = async (textBatch, batchIndex) => {
+      dispatchedBatchCount = batchIndex;
       if (textBatch.oversized) {
         const idx = textBatch.indices[0];
         const estTokens = estimateTokenCount(texts[idx]);
         onSkip?.({
           index: idx,
           reason: "context-window-exceeded",
-          message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+          message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
           batchStart: true,
           batchSize: 1
         });
@@ -97425,7 +97475,7 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
   }
   return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
 }
-async function runEmbeddingCanary(db, config, signal) {
+async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
   const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
   if (samples.length === 0) {
     return { outcome: "keep", verified: false, viaIdentityMatch: false };
@@ -97434,7 +97484,7 @@ async function runEmbeddingCanary(db, config, signal) {
   const skips = [];
   let canaryVectors;
   try {
-    canaryVectors = await embedBatch(samples.map((sample) => sample.searchText), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
+    canaryVectors = await embedBatch(samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
       if (model)
         observedModel = model;
     });
@@ -97504,6 +97554,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
   const storedFingerprint = getMeta(db, "embeddingFingerprint");
   let targetEntryIds = entryIds;
   let rebuildReason;
+  const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
   if (opts?.forceReembed) {
     db.transaction(() => {
       purgeEmbeddings(db, { dropVecTable: true });
@@ -97515,7 +97566,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
     targetEntryIds = undefined;
     rebuildReason = "forced by --reembed";
   } else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-    const decision = await runEmbeddingCanary(db, config, signal);
+    const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
     if (decision.outcome === "unverifiable") {
       warn(`[embed] ${decision.message}`);
       onProgress({ phase: "embeddings", message: decision.message });
@@ -97579,7 +97630,6 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
       purgeEmbeddingSalvage(db);
       return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
     }
-    const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     let truncatedCount = 0;
     const texts = [];
     const pendingEntries = [];
@@ -97674,6 +97724,15 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
           }
           return;
         }
+        if (outcome?.outcome === "budget-lowered") {
+          if (reportPerBatchLine) {
+            onProgress({
+              phase: "embeddings",
+              message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens \u2192 ${outcome.reason}`
+            });
+          }
+          return;
+        }
         if (model)
           observedModel = model;
         if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
@@ -97682,7 +97741,8 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
         }
         db.transaction(() => {
           for (let k2 = 0;k2 < indices.length; k2++) {
-            const entry = pendingEntries[indices[k2]];
+            const index = indices[k2];
+            const entry = pendingEntries[index];
             if (!entry)
               continue;
             const embedding = batchEmbeddings[k2];
@@ -97695,7 +97755,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
             const result = upsertEmbedding(db, entry.id, embedding);
             if (result.stored) {
               storedCount++;
-              storedTokens += estimateTokenCount(entry.searchText);
+              storedTokens += estimateTokenCount(texts[index]);
             } else {
               skippedCount++;
             }