chainlesschain 0.166.99 → 0.166.101

package/src/runtime/agent-core.js

@@ -156,6 +156,10 @@ import {
   isRetryableStreamError,
   STREAM_RETRY_BASE_MS,
   resolveStreamRetryMax,
+  captureStreamRetryMetadata,
+  attachStreamRetryMetadata,
+  resolveStreamRetryDelay,
+  waitForStreamRetry,
 } from "../lib/stream-retry.js";
 import {
   annotateLines,
@@ -184,7 +188,7 @@ import {
 } from "../lib/unattended-action-policy.js";
 import {
   formatProviderHttpError,
-  formatProviderResponseError,
+  createProviderResponseError,
 } from "../lib/provider-http-error.js";
 import { buildPermissionDecision } from "../lib/permission-decision.js";
 import { resolveSandboxPolicyPath } from "../lib/agent-sandbox.js";
@@ -194,6 +198,7 @@ import {
   settleManagedToolCheckpoint,
 } from "../lib/managed-tool-checkpoint.js";
 import { sessionBudgetAdmissionError } from "../lib/session-budget-production-root.js";
+import { projectRuntimeTokenUsage } from "../lib/runtime-usage-ledger.js";
 import {
   createMcpCallLedger,
   McpEffect,
@@ -11856,6 +11861,11 @@ export async function chatWithTools(rawMessages, options) {
 
   throwIfAborted(signal);
 
+  // Host accounting starts only after request admission. A rejected model
+  // projection never reaches the provider; failures after this boundary must
+  // still retain conservative usage accounting regardless of error metadata.
+  await options.onProviderCallStart?.();
+
   if (provider === "ollama") {
     const apiUrl = `${baseUrl}/api/chat`;
     // Multimodal (`cc agent --image`): ollama wants `{content, images:[base64]}`
@@ -11920,7 +11930,7 @@ export async function chatWithTools(rawMessages, options) {
       }),
     });
     if (!response.ok) {
-      throw new Error(await formatProviderResponseError("ollama", response));
+      throw await createProviderResponseError("ollama", response);
     }
     const data = await response.json();
     if (
@@ -12073,7 +12083,7 @@ export async function chatWithTools(rawMessages, options) {
     });
 
     if (!response.ok) {
-      throw new Error(await formatProviderResponseError("anthropic", response));
+      throw await createProviderResponseError("anthropic", response);
     }
 
     const data = await response.json();
@@ -12229,7 +12239,7 @@ export async function chatWithTools(rawMessages, options) {
       body: JSON.stringify(responseBody),
     });
     if (!response.ok) {
-      throw new Error(await formatProviderResponseError(provider, response));
+      throw await createProviderResponseError(provider, response);
     }
     const data = await response.json();
     const out = normalizeOpenAIResponsesResponse(data);
@@ -12310,7 +12320,7 @@ export async function chatWithTools(rawMessages, options) {
   });
 
   if (!response.ok) {
-    throw new Error(await formatProviderResponseError(provider, response));
+    throw await createProviderResponseError(provider, response);
   }
 
   const data = await response.json();
@@ -12397,9 +12407,11 @@ function _ollamaInitState() {
 
 function _streamUiCallbacks(onToken, onThinking) {
   let pending = [];
+  let visible = false;
   const capture = (callback) =>
     typeof callback === "function"
       ? (value) => {
+          if (typeof value === "string" && value.length > 0) visible = true;
           try {
             const result = callback(value);
             if (result && typeof result.then === "function") {
@@ -12413,6 +12425,7 @@ function _streamUiCallbacks(onToken, onThinking) {
   return {
     onToken: capture(onToken),
     onThinking: capture(onThinking),
+    hasVisibleOutput: () => visible,
     async settle() {
       if (pending.length === 0) return;
       const current = pending;
@@ -12516,11 +12529,17 @@ export function _accumulateOllamaStream(lines, onToken) {
  * Parity with Claude-Code 2.1.179 ("preserving partial responses instead of
  * showing raw errors"). Pure + exported for unit tests.
  */
-export function _streamErrorDisposition(err, signal, partialText) {
+export function _streamErrorDisposition(
+  err,
+  signal,
+  partialText,
+  visible = false,
+) {
   if (err?.isOutputBackpressureFailure === true) return "rethrow";
   if (isAbortError(err)) return "rethrow";
   if (signal && signal.aborted) return "rethrow";
-  if (typeof partialText === "string" && partialText.trim()) return "preserve";
+  if (visible || (typeof partialText === "string" && partialText.length > 0))
+    return "preserve";
   return "rethrow";
 }
 
@@ -12585,16 +12604,22 @@ export async function _retryStreamingChat(streamFn, opts = {}) {
   const retries = opts.retries ?? resolveStreamRetryMax();
   const base = opts.baseDelayMs ?? STREAM_RETRY_BASE_MS;
   const signal = opts.signal;
-  const sleep = opts.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
   const now = typeof opts.now === "function" ? opts.now : Date.now;
   let attempt = 0;
   for (;;) {
+    throwIfAborted(signal);
     const attemptStartedAt = now();
     try {
       return await streamFn(attempt);
     } catch (err) {
       if (attempt >= retries || !_isRetryableStreamError(err, signal))
         throw err;
+      const attemptEndedAt = now();
+      const delayMs = resolveStreamRetryDelay(err, attempt + 1, {
+        baseDelayMs: base,
+        now: attemptEndedAt,
+      });
+      if (delayMs === null) throw err;
       attempt++;
       if (
         opts.strictRetryObserver === true &&
@@ -12609,7 +12634,7 @@ export async function _retryStreamingChat(streamFn, opts = {}) {
       if (typeof opts.onRetry === "function") {
         try {
           const observation = opts.onRetry(attempt, err, {
-            durationMs: Math.max(0, now() - attemptStartedAt),
+            durationMs: Math.max(0, attemptEndedAt - attemptStartedAt),
           });
           if (observation && typeof observation.then === "function") {
             void Promise.resolve(observation).catch(() => {});
@@ -12638,8 +12663,7 @@ export async function _retryStreamingChat(streamFn, opts = {}) {
           }
         }
       }
-      await sleep(base * Math.pow(2, attempt - 1));
-      if (signal && signal.aborted) throw err; // user bailed during backoff
+      await waitForStreamRetry(delayMs, signal, { sleep: opts.sleep });
     }
   }
 }
@@ -12762,8 +12786,9 @@ async function _chatOllamaStreaming(
     body: JSON.stringify({ ...body, stream: true }),
   });
   if (!response.ok) {
-    throw new Error(await formatProviderResponseError("ollama", response));
+    throw await createProviderResponseError("ollama", response);
   }
+  const retryMetadata = captureStreamRetryMetadata(response);
   const state = _ollamaInitState();
   const ui = _streamUiCallbacks(onToken);
   const reader = response.body.getReader();
@@ -12788,7 +12813,17 @@ async function _chatOllamaStreaming(
       await ui.settle();
     }
   } catch (err) {
-    if (_streamErrorDisposition(err, signal, state.content) === "rethrow")
+    attachStreamRetryMetadata(err, retryMetadata);
+    if (!ui.hasVisibleOutput() && !state.content && state.toolCalls?.length)
+      err.noReplay = true;
+    if (
+      _streamErrorDisposition(
+        err,
+        signal,
+        state.content,
+        ui.hasVisibleOutput(),
+      ) === "rethrow"
+    )
       throw err;
     return _finalizeTruncatedStream(_ollamaFinalize, state);
   }
@@ -12813,8 +12848,9 @@ async function _chatAnthropicStreaming(
     body: JSON.stringify(body),
   });
   if (!response.ok) {
-    throw new Error(await formatProviderResponseError("anthropic", response));
+    throw await createProviderResponseError("anthropic", response);
   }
+  const retryMetadata = captureStreamRetryMetadata(response);
   const state = _anthropicInitState();
   const ui = _streamUiCallbacks(onToken, onThinking);
   const reader = response.body.getReader();
@@ -12839,7 +12875,21 @@ async function _chatAnthropicStreaming(
       await ui.settle();
     }
   } catch (err) {
-    if (_streamErrorDisposition(err, signal, state.text) === "rethrow")
+    attachStreamRetryMetadata(err, retryMetadata);
+    if (
+      !ui.hasVisibleOutput() &&
+      !state.text &&
+      Object.values(state.blocks).some((block) => block.type === "tool_use")
+    )
+      err.noReplay = true;
+    if (
+      _streamErrorDisposition(
+        err,
+        signal,
+        state.text,
+        ui.hasVisibleOutput(),
+      ) === "rethrow"
+    )
       throw err;
     return _finalizeTruncatedStream(_anthropicFinalize, state);
   }
@@ -12871,8 +12921,9 @@ async function _chatOpenAIStreaming(
     body: JSON.stringify(body),
   });
   if (!response.ok) {
-    throw new Error(await formatProviderResponseError(provider, response));
+    throw await createProviderResponseError(provider, response);
   }
+  const retryMetadata = captureStreamRetryMetadata(response);
   const state = _openaiInitState();
   const ui = _streamUiCallbacks(onToken);
   const reader = response.body.getReader();
@@ -12897,7 +12948,17 @@ async function _chatOpenAIStreaming(
       await ui.settle();
     }
   } catch (err) {
-    if (_streamErrorDisposition(err, signal, state.text) === "rethrow")
+    attachStreamRetryMetadata(err, retryMetadata);
+    if (!ui.hasVisibleOutput() && !state.text && state.tools.length)
+      err.noReplay = true;
+    if (
+      _streamErrorDisposition(
+        err,
+        signal,
+        state.text,
+        ui.hasVisibleOutput(),
+      ) === "rethrow"
+    )
       throw err;
     const out = _finalizeTruncatedStream(_openaiFinalize, state);
     const providerReceipt = _openAIProviderRequestReceipt(
@@ -12943,8 +13004,9 @@ async function _chatOpenAIResponsesStreaming(
     body: JSON.stringify(body),
   });
   if (!response.ok) {
-    throw new Error(await formatProviderResponseError("openai", response));
+    throw await createProviderResponseError("openai", response);
   }
+  const retryMetadata = captureStreamRetryMetadata(response);
   const state = createOpenAIResponsesStreamState();
   const ui = _streamUiCallbacks(onToken, onThinking);
   const reader = response.body.getReader();
@@ -12969,10 +13031,31 @@ async function _chatOpenAIResponsesStreaming(
       await ui.settle();
     }
   } catch (error) {
-    if (_streamErrorDisposition(error, signal, state.text) === "rethrow") {
+    if (state.terminalError) throw state.terminalError;
+    attachStreamRetryMetadata(error, retryMetadata);
+    if (
+      !ui.hasVisibleOutput() &&
+      !state.text &&
+      state.outputItems.some((item) => item?.type === "function_call")
+    )
+      error.noReplay = true;
+    if (
+      !state.normalized &&
+      _streamErrorDisposition(
+        error,
+        signal,
+        state.text,
+        ui.hasVisibleOutput(),
+      ) === "rethrow"
+    ) {
       throw error;
     }
-    const out = _finalizeTruncatedStream(finalizeOpenAIResponsesStream, state);
+    throwIfAborted(signal);
+    if (isAbortError(error) || error?.isOutputBackpressureFailure === true)
+      throw error;
+    const out = state.normalized
+      ? finalizeOpenAIResponsesStream(state)
+      : _finalizeTruncatedStream(finalizeOpenAIResponsesStream, state);
     const responseId = out.responseId || state.responseId;
     delete out.responseId;
     const providerReceipt = _openAIProviderRequestReceipt(
@@ -13394,6 +13477,19 @@ function _workflowNestedToolOutcomeUnknown(binding, toolName, result) {
   return error;
 }
 
+function _takeCompactionProviderReceipt(call, workflowEffectId) {
+  if (!call?.providerReceipt || call.providerReceiptEmitted) return null;
+  call.providerReceiptEmitted = true;
+  return {
+    type: "provider-request-receipt",
+    callId: call.callId,
+    workflowEffectId,
+    callSequence: call.callSequence,
+    source: "semantic-compaction",
+    ...call.providerReceipt,
+  };
+}
+
 function _workflowCompactionOutcomeUnknown(
   cause,
   code = "CC_WORKFLOW_COMPACTION_PROVIDER_OUTCOME_UNKNOWN",
@@ -13793,6 +13889,21 @@ function _instrumentAutoCompactorUsage(compactor, options) {
         this,
         requestBinding ? [...args, requestBinding] : args,
       );
+      // Keep validated provider-returned usage private to this actual call.
+      // It is not settled yet: the host consumes token-usage exactly once.
+      try {
+        call.returnedUsage = projectRuntimeTokenUsage({
+          callId: call.callId,
+          provider: result?.provider || call.provider,
+          model: result?.model || call.model,
+          source: call.source,
+          usage: result?.usage,
+        });
+        if (result?.callId && result.callId !== call.callId)
+          call.returnedUsage = null;
+      } catch {
+        call.returnedUsage = null;
+      }
       call.providerReceipt = providerRequestId
         ? _snapshotProviderRequestReceipt(
             result?.providerReceipt,
@@ -13802,6 +13913,7 @@ function _instrumentAutoCompactorUsage(compactor, options) {
         : null;
       return result;
     } catch (error) {
+      call.returnedUsage = null;
       if (workflowEffectId) {
         throw _workflowCompactionOutcomeUnknown(error);
       }
@@ -13949,8 +14061,8 @@ async function _getAutoCompactor(options) {
               compressOptions,
               sessionId: options.sessionId,
               operationId: `auto-${randomUUID()}`,
-              provider: options.provider,
-              model: options.model,
+              provider: options.provider || "ollama",
+              model: options.model || "unknown",
               env: options.contextMemoryEnv || process.env,
               persist: options.persistCompaction !== false,
               trigger: "auto",
@@ -14985,6 +15097,37 @@ export async function* agentLoop(messages, options) {
             !["committed", "degraded"].includes(stats.canonicalReceipt.status)
           ) {
             automaticCompactionSettlementBlocked = true;
+            const unsettledCall = compactionUsageState.calls.at(-1);
+            if (unsettledCall) {
+              const providerReceipt = _takeCompactionProviderReceipt(
+                unsettledCall,
+                workflowEffectId,
+              );
+              if (providerReceipt) yield providerReceipt;
+              if (
+                unsettledCall.returnedUsage &&
+                stats.summaryUsageUnknown !== true
+              ) {
+                yield {
+                  type: "token-usage",
+                  ...unsettledCall.returnedUsage,
+                  runId,
+                };
+              } else {
+                yield {
+                  type: "compaction-usage-unknown",
+                  callId: unsettledCall.callId,
+                  runId,
+                  provider: unsettledCall.provider,
+                  model: unsettledCall.model,
+                  source: "semantic-compaction",
+                  code: "provider_usage_unsettled",
+                  reason:
+                    stats.summaryUsageUnknownReason ||
+                    "provider_transport_outcome_unknown",
+                };
+              }
+            }
             yield {
               type: "compaction-degraded",
               runId,
@@ -15003,6 +15146,20 @@ export async function* agentLoop(messages, options) {
               backgroundSubAgents,
               backgroundUsageFailureState,
             );
+            if (workflowEffectId) {
+              throw _markWorkflowCompactionFailureReported(
+                _workflowCompactionOutcomeUnknown(
+                  null,
+                  "CC_WORKFLOW_COMPACTION_SETTLEMENT_UNKNOWN",
+                  "Workflow-bound semantic compaction requires reconciliation",
+                ),
+              );
+            }
+            yield {
+              type: "run-ended",
+              runId,
+              reason: "compaction-settlement-failed",
+            };
             return;
           }
           const observerFailure = compactionUsageState.calls.find(
@@ -15015,16 +15172,11 @@ export async function* agentLoop(messages, options) {
           }
           compactionUsageCall = compactionUsageState.calls.at(-1) || null;
           const compactionUsage = _compactionTokenUsage(stats);
-          if (compactionUsageCall?.providerReceipt) {
-            yield {
-              type: "provider-request-receipt",
-              callId: compactionUsageCall.callId,
-              workflowEffectId,
-              callSequence: compactionUsageCall.callSequence,
-              source: "semantic-compaction",
-              ...compactionUsageCall.providerReceipt,
-            };
-          }
+          const providerReceipt = _takeCompactionProviderReceipt(
+            compactionUsageCall,
+            workflowEffectId,
+          );
+          if (providerReceipt) yield providerReceipt;
           if (stats.degraded === true) {
             const usageOutcomeUnknown = stats.summaryUsageUnknown === true;
             const projectedStats = usageOutcomeUnknown
@@ -15100,6 +15252,11 @@ export async function* agentLoop(messages, options) {
                 ),
               );
             }
+            yield {
+              type: "run-ended",
+              runId,
+              reason: "compaction-usage-unknown",
+            };
             return;
           }
           if (
@@ -15162,6 +15319,11 @@ export async function* agentLoop(messages, options) {
                   ),
                 );
               }
+              yield {
+                type: "run-ended",
+                runId,
+                reason: "compaction-settlement-failed",
+              };
               return;
             }
             if (!hermeticExecution) {
@@ -15226,6 +15388,79 @@ export async function* agentLoop(messages, options) {
         }
         compactionUsageCall =
           compactionUsageCall || compactionUsageState?.calls?.at(-1) || null;
+        const { getRejectedCompactionUsage } =
+          await import("../lib/context-memory-kernel/live-compaction.js");
+        const rejectedUsage = getRejectedCompactionUsage(_e);
+        const returnedUsage = compactionUsageCall?.returnedUsage;
+        const providerReceipt = _takeCompactionProviderReceipt(
+          compactionUsageCall,
+          workflowEffectId,
+        );
+        if (providerReceipt) yield providerReceipt;
+        const knownPlanRejection =
+          rejectedUsage &&
+          returnedUsage &&
+          (!rejectedUsage.callId ||
+            rejectedUsage.callId === returnedUsage.callId) &&
+          (!rejectedUsage.receipt.callId ||
+            rejectedUsage.receipt.callId === returnedUsage.callId) &&
+          rejectedUsage.provider === returnedUsage.provider &&
+          rejectedUsage.model === returnedUsage.model &&
+          [
+            "input_tokens",
+            "output_tokens",
+            "cache_read_input_tokens",
+            "cache_creation_input_tokens",
+          ].every(
+            (field) =>
+              rejectedUsage.usage[field] === returnedUsage.usage[field],
+          );
+        if (knownPlanRejection || (!rejectedUsage && returnedUsage)) {
+          automaticCompactionSettlementBlocked = true;
+          // Yield through the existing host ledger/budget settlement path;
+          // never claim a callback or an error flag already paid this debt.
+          yield { type: "token-usage", ...returnedUsage, runId };
+          yield {
+            type: "compaction-degraded",
+            runId,
+            code: _e.code,
+            reason: knownPlanRejection
+              ? "canonical_compaction_plan_rejected"
+              : "canonical_compaction_settlement_failed",
+            summaryMode: "none",
+          };
+          await _awaitBackgroundUsageSettlement(
+            backgroundSubAgents,
+            backgroundUsageFailureState,
+          );
+          if (workflowEffectId) {
+            if (knownPlanRejection) {
+              const error = new Error(
+                "Workflow-bound compaction plan was rejected; the task is incomplete",
+                { cause: _e },
+              );
+              error.code = "CC_WORKFLOW_COMPACTION_PLAN_REJECTED";
+              error.contextOutcome = "aborted";
+              error.usageOutcome = "known";
+              throw error;
+            }
+            const error = _workflowCompactionOutcomeUnknown(
+              _e,
+              "CC_WORKFLOW_COMPACTION_SETTLEMENT_UNKNOWN",
+              "Workflow-bound semantic compaction settlement is unknown",
+            );
+            error.usageOutcome = "known";
+            throw _markWorkflowCompactionFailureReported(error);
+          }
+          yield {
+            type: "run-ended",
+            runId,
+            reason: knownPlanRejection
+              ? "compaction-plan-rejected"
+              : "compaction-settlement-failed",
+          };
+          return;
+        }
         if (compactionUsageCall) {
           automaticCompactionSettlementBlocked = true;
           yield {
@@ -15249,6 +15484,11 @@ export async function* agentLoop(messages, options) {
                 : _workflowCompactionOutcomeUnknown(_e),
             );
           }
+          yield {
+            type: "run-ended",
+            runId,
+            reason: "compaction-usage-unknown",
+          };
           return;
         }
         yield {