akm-cli 0.9.15-beta.2 → 0.9.15-beta.3

package/dist/llm/embedders/remote.js

@@ -30,10 +30,17 @@ export const DEFAULT_REMOTE_BATCH_SIZE = 100;
  * a batch of 100 small docs (~400 KB, ~100K tokens) took 14.8s against a
  * healthy local endpoint — half the 30s request timeout — and a single
  * 128 KB (~24K token) document alone was rejected by the endpoint as
- * exceeding its context size. 8000 tokens keeps a batch's estimated size
- * comfortably under both the timeout and common local-model context windows.
+ * exceeding its context size.
+ *
+ * Lowered from 8000 to 6000 (#954, field report on beta.1): the 4-chars-
+ * per-token estimator undercounts dense technical text by 7-55%, so 8000
+ * against an 8192-token llama.cpp embedder regularly landed real requests
+ * over the endpoint's context window. 6000 is the value the field confirmed
+ * stops that steady trickle of rejections; `embedBatch`'s run-scoped
+ * adaptive budget below still shrinks further, for an endpoint where even
+ * this is not enough.
  */
-export const DEFAULT_TOKEN_BUDGET = 8000;
+export const DEFAULT_TOKEN_BUDGET = 6000;
 /** Cheap token estimator: 4 chars ≈ 1 token. Used in verbose logging and error messages. */
 export function estimateTokenCount(text) {
     return Math.round(text.length / 4);
@@ -73,7 +80,7 @@ export function capEmbeddingText(text, maxTokens) {
 /**
  * Default per-request timeout when `embedding.timeoutMs` is unset (#954).
  * The prior fixed 30s cut off exactly the field-report case: a
- * local model server on an 8000-token (`DEFAULT_TOKEN_BUDGET`) batch
+ * local model server on a full-budget (`DEFAULT_TOKEN_BUDGET`) batch
  * legitimately takes longer than that, and the timeout fired mid-response
  * with no retry — every batch it hit was silently dropped for the rest of
  * an hours-long run. 120s comfortably covers a slow local batch while still
@@ -235,6 +242,21 @@ export function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
     flush();
     return batches;
 }
+/**
+ * Shrink factor applied to the effective request budget on the first
+ * context-size rejection of an `embedBatch` run (#954, field report on
+ * beta.1): one 25% cut absorbs the estimator's measured undercount without
+ * repeatedly re-shrinking mid-run — see the "shrink at most once" rule on
+ * {@link RemoteEmbedder.embedBatch}.
+ */
+const ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+/**
+ * Floor on the adaptive-budget shrink above, as a multiple of
+ * `maxInputTokens` (#954): a request budget below twice the per-document cap
+ * could no longer batch more than one document per request, defeating the
+ * point of batching at all.
+ */
+const ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 export class RemoteEmbedder {
     config;
     endpoint;
@@ -339,6 +361,18 @@ export class RemoteEmbedder {
      * {@link scaleEmbeddingTimeoutMs}, so a dead server is detected in seconds
      * on a small batch rather than always waiting out the full configured
      * `embedding.timeoutMs`.
+     *
+     * Run-scoped adaptive budget (#954, field report on beta.1): the FIRST
+     * context-size rejection of the run shrinks the effective request budget
+     * by {@link ADAPTIVE_BUDGET_SHRINK_FACTOR} (floored at
+     * {@link ADAPTIVE_BUDGET_FLOOR_MULTIPLIER} times `maxInputTokens`) for
+     * every batch not yet dispatched — the still-planned tail of `texts` is
+     * re-batched with `buildTokenBoundedBatches` at the smaller budget, and a
+     * `budget-lowered` `onBatch` event reports it once. This never touches the
+     * split-and-retry of the rejected batch itself (above), and never fires a
+     * second time in the same run even if a later batch is also rejected — a
+     * static configured budget that is simply too big for the endpoint should
+     * self-correct once, not ratchet down forever.
      */
     async embedBatch(texts, signal, onSkip, onBatch) {
         if (texts.length === 0)
@@ -351,10 +385,59 @@ export class RemoteEmbedder {
         // request budget too, so a config author setting it for one purpose
         // silently changed the other. `maxTokens` is the sole knob for the
         // request budget now; unset falls back to DEFAULT_TOKEN_BUDGET.
-        const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+        //
+        // `effectiveTokenBudget` (#954) starts at the configured/default value
+        // and MAY shrink once, on the run's first context-size rejection — see
+        // `maybeShrinkBudget` below. `textBatches` is mutated in place (spliced)
+        // by that shrink rather than reassigned, so the in-flight
+        // `concurrentMap` pool below (which reads this same array by reference)
+        // picks up the re-planned tail without restarting.
+        let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
         const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-        const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+        const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+        const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
         const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+        // How many of `textBatches` concurrentMap has already claimed (its own
+        // `nextIndex`, mirrored here so a budget shrink knows where the
+        // not-yet-dispatched tail begins). Assigned, not incremented, at the top
+        // of `runProviderBatch` — batches are claimed in strictly increasing
+        // order, so the highest `batchIndex` seen so far IS the claimed count.
+        let dispatchedBatchCount = 0;
+        // Set once the run's first context-size rejection has shrunk the budget
+        // (#954) — guards `maybeShrinkBudget` so it never fires twice.
+        let budgetShrunk = false;
+        // On the FIRST context-size rejection of this `embedBatch` call, shrink
+        // `effectiveTokenBudget` and re-plan every batch `concurrentMap` has not
+        // yet claimed from the smaller budget. Never touches `rejectedIndices`
+        // itself — the caller's own split-and-retry handles that batch — and is
+        // a no-op after the first call (`budgetShrunk`).
+        const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+            if (budgetShrunk)
+                return;
+            budgetShrunk = true;
+            const floor = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+            effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor);
+            const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+            const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+            if (remainingIndices.length > 0) {
+                const remainingTexts = remainingIndices.map((i) => texts[i]);
+                const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+                    indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+                    oversized: batch.oversized,
+                }));
+                textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+            }
+            warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+            commitBatch(rejectedIndices, rejectedIndices.map(() => undefined), undefined, {
+                batchIndex: rejectedBatchIndex,
+                batchCount: textBatches.length,
+                docCount: rejectedIndices.length,
+                requestTokens: rejectedRequestTokens,
+                elapsedMs: 0,
+                outcome: "budget-lowered",
+                reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`,
+            });
+        };
         // Stops the pool from claiming any FURTHER provider batch once the
         // caller's onBatch has failed once (the materializer's transaction
         // failed, so a subsequent commit would just fail again) — dispatching
@@ -437,7 +520,7 @@ export class RemoteEmbedder {
                 return;
             const batch = indices.map((i) => texts[i]);
             const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-            const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+            const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
             const requestStart = Date.now();
             let batchEmbeddings;
             let responseModel;
@@ -457,6 +540,15 @@ export class RemoteEmbedder {
                 // failed" condition, it means stop entirely.
                 if (signal?.aborted)
                     throw err;
+                if (err instanceof ContextExceededError) {
+                    // #954: the run's FIRST context-size rejection (any size) shrinks
+                    // the budget for everything not yet dispatched; a no-op after the
+                    // first call. Deliberately BEFORE the split below — it must fire
+                    // for a single-document rejection too (which never reaches the
+                    // `indices.length > 1` split branch), and it never touches this
+                    // batch's own split-and-retry.
+                    maybeShrinkBudget(indices, batchIndex, requestTokens);
+                }
                 if (err instanceof ContextExceededError && indices.length > 1) {
                     const mid = Math.ceil(indices.length / 2);
                     await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -554,13 +646,19 @@ export class RemoteEmbedder {
             });
         };
         const runProviderBatch = async (textBatch, batchIndex) => {
+            // Claimed in strictly increasing order by `concurrentMap` below, so
+            // the highest `batchIndex` seen so far is exactly how many batches it
+            // has claimed (#954) — see `maybeShrinkBudget`'s doc comment above.
+            // Assigned synchronously at entry, before any `await`, so this always
+            // matches `concurrentMap`'s own `nextIndex` at the moment of claim.
+            dispatchedBatchCount = batchIndex;
             if (textBatch.oversized) {
                 const idx = textBatch.indices[0];
                 const estTokens = estimateTokenCount(texts[idx]);
                 onSkip?.({
                     index: idx,
                     reason: "context-window-exceeded",
-                    message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+                    message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
                     batchStart: true,
                     batchSize: 1,
                 });

package/dist/scripts/akm-migrate-node.js

@@ -7070,7 +7070,8 @@ var init_errors = __esm(() => {
   };
   TRANSIENT_HINTS = {
     RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
-    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing."
+    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
+    INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs."
   };
   NOT_FOUND_HINTS = {
     ASSET_NOT_FOUND: "Run `akm search <query>` or `akm index` to refresh the index.",
@@ -75920,7 +75921,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
   maxInputTokens: positiveInt.optional(),
   maxTokens: positiveInt.optional(),
   batchSize: positiveInt.optional(),
-  chunkSize: positiveInt.optional(),
   contextLength: positiveInt.optional(),
   ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
   timeoutMs: positiveInt.optional(),
@@ -96987,7 +96987,7 @@ function defaultConcurrencyForEndpoint(endpoint) {
 // src/llm/embedders/remote.ts
 init_warn();
 var DEFAULT_REMOTE_BATCH_SIZE = 100;
-var DEFAULT_TOKEN_BUDGET = 8000;
+var DEFAULT_TOKEN_BUDGET = 6000;
 function estimateTokenCount(text) {
   return Math.round(text.length / 4);
 }
@@ -97069,6 +97069,8 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
   flush();
   return batches;
 }
+var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+var ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 
 class RemoteEmbedder {
   config;
@@ -97121,10 +97123,42 @@ class RemoteEmbedder {
     const results = new Array(texts.length).fill(undefined);
     const headers = this.buildHeaders();
     const ollamaOpts = resolveOllamaOptions(this.config);
-    const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+    let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
     const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-    const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+    const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+    const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
     const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+    let dispatchedBatchCount = 0;
+    let budgetShrunk = false;
+    const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+      if (budgetShrunk)
+        return;
+      budgetShrunk = true;
+      const floor2 = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+      effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
+      const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+      const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+      if (remainingIndices.length > 0) {
+        const remainingTexts = remainingIndices.map((i) => texts[i]);
+        const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+          indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+          oversized: batch.oversized
+        }));
+        textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+      }
+      warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+      commitBatch(rejectedIndices, rejectedIndices.map(() => {
+        return;
+      }), undefined, {
+        batchIndex: rejectedBatchIndex,
+        batchCount: textBatches.length,
+        docCount: rejectedIndices.length,
+        requestTokens: rejectedRequestTokens,
+        elapsedMs: 0,
+        outcome: "budget-lowered",
+        reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`
+      });
+    };
     const dispatchAbort = new AbortController;
     const stopDispatch = (reason) => {
       if (!dispatchAbort.signal.aborted)
@@ -97157,7 +97191,7 @@ class RemoteEmbedder {
         return;
       const batch = indices.map((i) => texts[i]);
       const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
       const requestStart = Date.now();
       let batchEmbeddings;
       let responseModel;
@@ -97174,6 +97208,9 @@ class RemoteEmbedder {
       } catch (err) {
         if (signal?.aborted)
           throw err;
+        if (err instanceof ContextExceededError) {
+          maybeShrinkBudget(indices, batchIndex, requestTokens);
+        }
         if (err instanceof ContextExceededError && indices.length > 1) {
           const mid = Math.ceil(indices.length / 2);
           await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -97239,13 +97276,14 @@ class RemoteEmbedder {
       });
     };
     const runProviderBatch = async (textBatch, batchIndex) => {
+      dispatchedBatchCount = batchIndex;
       if (textBatch.oversized) {
         const idx = textBatch.indices[0];
         const estTokens = estimateTokenCount(texts[idx]);
         onSkip?.({
           index: idx,
           reason: "context-window-exceeded",
-          message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+          message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
           batchStart: true,
           batchSize: 1
         });
@@ -97469,7 +97507,7 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
   }
   return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
 }
-async function runEmbeddingCanary(db, config, signal) {
+async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
   const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
   if (samples.length === 0) {
     return { outcome: "keep", verified: false, viaIdentityMatch: false };
@@ -97478,7 +97516,7 @@ async function runEmbeddingCanary(db, config, signal) {
   const skips = [];
   let canaryVectors;
   try {
-    canaryVectors = await embedBatch(samples.map((sample) => sample.searchText), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
+    canaryVectors = await embedBatch(samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
       if (model)
         observedModel = model;
     });
@@ -97548,6 +97586,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
   const storedFingerprint = getMeta(db, "embeddingFingerprint");
   let targetEntryIds = entryIds;
   let rebuildReason;
+  const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
   if (opts?.forceReembed) {
     db.transaction(() => {
       purgeEmbeddings(db, { dropVecTable: true });
@@ -97559,7 +97598,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
     targetEntryIds = undefined;
     rebuildReason = "forced by --reembed";
   } else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-    const decision = await runEmbeddingCanary(db, config, signal);
+    const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
     if (decision.outcome === "unverifiable") {
       warn(`[embed] ${decision.message}`);
       onProgress({ phase: "embeddings", message: decision.message });
@@ -97623,7 +97662,6 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
       purgeEmbeddingSalvage(db);
       return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
     }
-    const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     let truncatedCount = 0;
     const texts = [];
     const pendingEntries = [];
@@ -97718,6 +97756,15 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
           }
           return;
         }
+        if (outcome?.outcome === "budget-lowered") {
+          if (reportPerBatchLine) {
+            onProgress({
+              phase: "embeddings",
+              message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcome.reason}`
+            });
+          }
+          return;
+        }
         if (model)
           observedModel = model;
         if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
@@ -97726,7 +97773,8 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
         }
         db.transaction(() => {
           for (let k2 = 0;k2 < indices.length; k2++) {
-            const entry = pendingEntries[indices[k2]];
+            const index = indices[k2];
+            const entry = pendingEntries[index];
             if (!entry)
               continue;
             const embedding = batchEmbeddings[k2];
@@ -97739,7 +97787,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
             const result = upsertEmbedding(db, entry.id, embedding);
             if (result.stored) {
               storedCount++;
-              storedTokens += estimateTokenCount(entry.searchText);
+              storedTokens += estimateTokenCount(texts[index]);
             } else {
               skippedCount++;
             }

package/dist/scripts/akm-migrate.js

@@ -7069,7 +7069,8 @@ var init_errors = __esm(() => {
   };
   TRANSIENT_HINTS = {
     RUN_LEASE_HELD: "Wait for the named engine invocation to finish or for the lease to expire, then retry. `akm workflow status <id>` shows the current lease.",
-    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing."
+    STATE_DB_CONTENDED: "Another akm process is writing state.db right now. Wait a few seconds and retry; commands that support --skip-if-locked can skip instead of failing.",
+    INDEX_DB_CONTENDED: "Another akm process is writing index.db; retry shortly, or pass --skip-if-locked on scheduled runs."
   };
   NOT_FOUND_HINTS = {
     ASSET_NOT_FOUND: "Run `akm search <query>` or `akm index` to refresh the index.",
@@ -75248,7 +75249,6 @@ var EmbeddingConnectionConfigSchema = exports_external.object({
   maxInputTokens: positiveInt.optional(),
   maxTokens: positiveInt.optional(),
   batchSize: positiveInt.optional(),
-  chunkSize: positiveInt.optional(),
   contextLength: positiveInt.optional(),
   ollamaOptions: EmbeddingOllamaOptionsSchema.optional(),
   timeoutMs: positiveInt.optional(),
@@ -96943,7 +96943,7 @@ function defaultConcurrencyForEndpoint(endpoint) {
 // src/llm/embedders/remote.ts
 init_warn();
 var DEFAULT_REMOTE_BATCH_SIZE = 100;
-var DEFAULT_TOKEN_BUDGET = 8000;
+var DEFAULT_TOKEN_BUDGET = 6000;
 function estimateTokenCount(text) {
   return Math.round(text.length / 4);
 }
@@ -97025,6 +97025,8 @@ function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
   flush();
   return batches;
 }
+var ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+var ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 
 class RemoteEmbedder {
   config;
@@ -97077,10 +97079,42 @@ class RemoteEmbedder {
     const results = new Array(texts.length).fill(undefined);
     const headers = this.buildHeaders();
     const ollamaOpts = resolveOllamaOptions(this.config);
-    const tokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
+    let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
     const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-    const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+    const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+    const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
     const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+    let dispatchedBatchCount = 0;
+    let budgetShrunk = false;
+    const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+      if (budgetShrunk)
+        return;
+      budgetShrunk = true;
+      const floor2 = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+      effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor2);
+      const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+      const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+      if (remainingIndices.length > 0) {
+        const remainingTexts = remainingIndices.map((i) => texts[i]);
+        const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+          indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+          oversized: batch.oversized
+        }));
+        textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+      }
+      warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+      commitBatch(rejectedIndices, rejectedIndices.map(() => {
+        return;
+      }), undefined, {
+        batchIndex: rejectedBatchIndex,
+        batchCount: textBatches.length,
+        docCount: rejectedIndices.length,
+        requestTokens: rejectedRequestTokens,
+        elapsedMs: 0,
+        outcome: "budget-lowered",
+        reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`
+      });
+    };
     const dispatchAbort = new AbortController;
     const stopDispatch = (reason) => {
       if (!dispatchAbort.signal.aborted)
@@ -97113,7 +97147,7 @@ class RemoteEmbedder {
         return;
       const batch = indices.map((i) => texts[i]);
       const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
-      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, tokenBudget);
+      const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
       const requestStart = Date.now();
       let batchEmbeddings;
       let responseModel;
@@ -97130,6 +97164,9 @@ class RemoteEmbedder {
       } catch (err) {
         if (signal?.aborted)
           throw err;
+        if (err instanceof ContextExceededError) {
+          maybeShrinkBudget(indices, batchIndex, requestTokens);
+        }
         if (err instanceof ContextExceededError && indices.length > 1) {
           const mid = Math.ceil(indices.length / 2);
           await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
@@ -97195,13 +97232,14 @@ class RemoteEmbedder {
       });
     };
     const runProviderBatch = async (textBatch, batchIndex) => {
+      dispatchedBatchCount = batchIndex;
       if (textBatch.oversized) {
         const idx = textBatch.indices[0];
         const estTokens = estimateTokenCount(texts[idx]);
         onSkip?.({
           index: idx,
           reason: "context-window-exceeded",
-          message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+          message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
           batchStart: true,
           batchSize: 1
         });
@@ -97425,7 +97463,7 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
   }
   return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
 }
-async function runEmbeddingCanary(db, config, signal) {
+async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
   const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
   if (samples.length === 0) {
     return { outcome: "keep", verified: false, viaIdentityMatch: false };
@@ -97434,7 +97472,7 @@ async function runEmbeddingCanary(db, config, signal) {
   const skips = [];
   let canaryVectors;
   try {
-    canaryVectors = await embedBatch(samples.map((sample) => sample.searchText), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
+    canaryVectors = await embedBatch(samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
       if (model)
         observedModel = model;
     });
@@ -97504,6 +97542,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
   const storedFingerprint = getMeta(db, "embeddingFingerprint");
   let targetEntryIds = entryIds;
   let rebuildReason;
+  const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
   if (opts?.forceReembed) {
     db.transaction(() => {
       purgeEmbeddings(db, { dropVecTable: true });
@@ -97515,7 +97554,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
     targetEntryIds = undefined;
     rebuildReason = "forced by --reembed";
   } else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-    const decision = await runEmbeddingCanary(db, config, signal);
+    const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
     if (decision.outcome === "unverifiable") {
       warn(`[embed] ${decision.message}`);
       onProgress({ phase: "embeddings", message: decision.message });
@@ -97579,7 +97618,6 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
       purgeEmbeddingSalvage(db);
       return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
     }
-    const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     let truncatedCount = 0;
     const texts = [];
     const pendingEntries = [];
@@ -97674,6 +97712,15 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
           }
           return;
         }
+        if (outcome?.outcome === "budget-lowered") {
+          if (reportPerBatchLine) {
+            onProgress({
+              phase: "embeddings",
+              message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens \u2192 ${outcome.reason}`
+            });
+          }
+          return;
+        }
         if (model)
           observedModel = model;
         if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
@@ -97682,7 +97729,8 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
         }
         db.transaction(() => {
           for (let k2 = 0;k2 < indices.length; k2++) {
-            const entry = pendingEntries[indices[k2]];
+            const index = indices[k2];
+            const entry = pendingEntries[index];
             if (!entry)
               continue;
             const embedding = batchEmbeddings[k2];
@@ -97695,7 +97743,7 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds,
             const result = upsertEmbedding(db, entry.id, embedding);
             if (result.stored) {
               storedCount++;
-              storedTokens += estimateTokenCount(entry.searchText);
+              storedTokens += estimateTokenCount(texts[index]);
             } else {
               skippedCount++;
             }

package/docs/migration/release-notes/0.9.15.md

@@ -38,6 +38,17 @@ scheduled or opportunistic run step aside instead of contending with a rebuild
 already in progress; the shipped `index-refresh` scheduled task already passes
 it.
 
+`embedding.maxTokens`'s default (the per-request token budget) is now 6000,
+down from 8000: a field report on an 8192-token llama.cpp embedder showed the
+4-chars-per-token estimator undercounts dense technical text by 7-55%, so the
+old default regularly overshot the endpoint's real context window. If you
+already set `embedding.maxTokens` explicitly, this default change does not
+affect you — your configured value is unchanged. `akm index` also now
+recovers automatically within a run: on the first request rejected for
+exceeding the endpoint's context window, it lowers its effective budget for
+the rest of that run (reported with one line) rather than continuing to hit
+the same wall on every following batch.
+
 `embedding.concurrency` (positive integer, 1-16) overrides the number of
 embedding requests kept in flight at once, which otherwise defaults to 1 for
 a loopback endpoint and 2 for a remote one. Set it only for an endpoint that
@@ -131,3 +142,10 @@ effective config and another config file or bundle-relative file. `akm config
 unset` now refuses to unset a key whose value comes only from an
 `extends`-inherited base, naming the source, since there would be nothing local
 to remove.
+
+The scheduler runs the binary path `akm task sync` recorded at sync time, not
+whichever akm your shell now resolves to. After upgrading akm through a
+different installer than the one active at your last `task sync` (e.g.
+npm-global to a standalone download), run `akm task sync` again so the
+schedule points at the new binary; `akm health --probe` now warns via a new
+`scheduler-binary` advisory when the two diverge.

package/docs/reference/cli.md

@@ -117,7 +117,7 @@ Every command exits with one of the following codes:
 | 2 | Usage / bad input | `UsageError` |
 | 4 | Health warning (`akm health` only) | — |
 | 70 | Internal / unclassified error | unexpected throw |
-| 75 | Transient — retry shortly (sysexits `EX_TEMPFAIL`); another akm process holds a lock or is writing `state.db` right now, not a bad command line | `TransientError` |
+| 75 | Transient — retry shortly (sysexits `EX_TEMPFAIL`); another akm process holds a lock or is writing `state.db` or `index.db` right now, not a bad command line | `TransientError` |
 | 78 | Configuration error | `ConfigError` |
 
 Failures classified by akm emit a JSON error envelope on **stderr** before
@@ -279,7 +279,12 @@ opt-in, PID-liveness-only rebuild lock and releases it on exit — this is
 advisory, never the blocking lock #872 removed (see
 [Locks](https://github.com/itlackey/akm/blob/main/docs/architecture/internals/indexing.md#locks)). A human-typed
 `akm index` with no flag is never gated by it: if another run already holds
-the lock, it warns and proceeds anyway, contending with the existing run.
+the lock, it warns and proceeds anyway, contending with the existing run. If
+that contention makes index.db genuinely busy (SQLite `database is locked`)
+long enough to exhaust the driver's retry window, the run now fails with
+exit 75 (`TransientError`, code `INDEX_DB_CONTENDED`) instead of the raw
+driver error at exit 70 — the same retry-shortly contract as
+`STATE_DB_CONTENDED`, so a scheduler can branch on it instead of alerting.
 `--skip-if-locked` changes that only for the invocation that passes it: if
 the lock is already held by a live process, it skips gracefully (exit 0,
 `{ ok: true, skipped: { reason: "lock-held", pid, launcherPid, startedAt } }`
@@ -358,15 +363,17 @@ akm health --report --window-compare 7d --format html
 | `--window-compare` | Compare the current window against the prior window of the same duration (e.g. `24h`, `7d`). With `--report`, overrides the default trend window. |
 | `--group-by` | Group rows by `run` (one row per `improve_runs` entry). Omit for the default summary. |
 | `--windows` | Explicit comparison window(s) as `name=...,since=ISO,until=ISO` (repeatable, up to 4). Mutually exclusive with `--window-compare`. |
-| `--no-probe` | Skip the `default-llm-engine` / `configured-engines` reachability probes and the `cli-version` update check (for an offline or air-gapped host). |
+| `--no-probe` | Skip the `default-llm-engine` / `configured-engines` reachability probes, the `cli-version` update check, and the `scheduler-binary` version check (for an offline or air-gapped host). |
 
 The command reads `state.db`, verifies that the required tables exist, performs a
 write-read probe against the events stream, inspects `task_history`, checks the
 default agent engine, and summarizes recent `improve_*` events. Unless
 `--no-probe` is given, it also sends a bounded (3s timeout) reachability probe
 to the `default-llm-engine` and every `configured-engines` LLM connection (and
-an SDK engine's LLM fallback), one probe per distinct endpoint, and checks the
-installed akm-cli version against the latest GitHub release (`cli-version`).
+an SDK engine's LLM fallback), one probe per distinct endpoint, checks the
+installed akm-cli version against the latest GitHub release (`cli-version`),
+and runs the scheduler's recorded akm binary with `--version` to check it
+against the running CLI (`scheduler-binary`).
 
 Primary result fields:
 
@@ -1239,6 +1246,14 @@ Shipping akm inside your own product (a Docker image, a plugin's own
 `node_modules`)? See [Bundling akm](../integration/bundling-akm.md) for the
 full boot contract, JSON shapes, and exit codes.
 
+`akm upgrade` replaces the binary in place for its own install method, but a
+scheduler binding recorded by an earlier `akm task sync` under a *different*
+install method is not repointed automatically — the scheduler runs the
+binary path recorded at sync time, not whichever akm `upgrade` just
+installed. Run `akm task sync` after switching installers so scheduled runs
+pick up the new binary; see [`task sync`](#task) and `akm health`'s
+`scheduler-binary` advisory.
+
 ### clone
 
 Copy an asset from any source into a managed writable bundle or an unmanaged
@@ -2872,6 +2887,13 @@ task template (both the core set and the improve-schedule set) and asks once
 before changing task files or scheduler state; non-interactive setup changes
 neither.
 
+Because the scheduler runs the exact binary path recorded at the last `task
+sync`, upgrading akm through a different installer than the one active at
+that sync (npm-global to a standalone download, or vice versa) leaves
+scheduled runs invoking the old, now-stale binary — `task sync` re-resolves
+the current path and repoints them. `akm health --probe`'s `scheduler-binary`
+advisory warns when the two diverge, naming both versions.
+
 Setup reconfiguration preserves existing scheduler runtime bindings. Changing
 the AKM storage path or installed runtime path therefore requires an explicit
 `akm task sync --rebind`; setup does not silently migrate those entries.