@vellumai/assistant 0.6.3 → 0.6.5

package/src/__tests__/workspace-policy.test.ts

@@ -212,17 +212,7 @@ describe("isWorkspaceScopedInvocation", () => {
   // ── Network tools ──────────────────────────────────────────────────
 
   describe("network tools", () => {
-    const networkTools = [
-      "web_search",
-      "web_fetch",
-      "browser_navigate",
-      "browser_click",
-      "browser_type",
-      "browser_scroll",
-      "browser_screenshot",
-      "browser_close",
-      "network_request",
-    ];
+    const networkTools = ["web_search", "web_fetch", "network_request"];
 
     for (const tool of networkTools) {
       test(`${tool} is NOT workspace-scoped`, () => {

package/src/acp/client-handler.ts

@@ -412,12 +412,11 @@ function mapDecisionToOptionId(
     decision === "allow_10m" ||
     decision === "allow_conversation" ||
     decision === "always_allow" ||
-    decision === "always_allow_high_risk" ||
     decision === "temporary_override";
 
   if (isAllow) {
     // Prefer allow_always for persistent decisions, fallback to allow_once
-    if (decision === "always_allow" || decision === "always_allow_high_risk") {
+    if (decision === "always_allow") {
       const alwaysOpt = options.find((o) => o.kind === "allow_always");
       if (alwaysOpt) return alwaysOpt.optionId;
     }

package/src/agent/image-optimize.ts

@@ -127,24 +127,36 @@ function runSips(inputBytes: Buffer): Buffer | null {
  * Results are cached on disk by content hash so repeated sends of the same
  * image (or daemon restarts) skip the sips call entirely.
  */
+/**
+ * Decide whether an image needs to be rescaled before sending.
+ *
+ * Anthropic rejects many-image requests when any image exceeds 2000 px on a
+ * side, so dimensions — not file size — are the authoritative gate. A sparse
+ * screenshot can be under 300 KB while still being 3000+ px wide, which the
+ * byte-size heuristic alone would let slip through.
+ *
+ * Exported for unit testing.
+ */
+export function shouldRescaleImage(
+  dims: { width: number; height: number } | null,
+  byteLength: number,
+): boolean {
+  if (dims) {
+    // Dimensions known — they are the authoritative check.
+    return dims.width > MAX_DIMENSION || dims.height > MAX_DIMENSION;
+  }
+  // Dimensions unparseable — fall back to file size as a rough proxy.
+  return byteLength > OPTIMIZE_THRESHOLD_BYTES;
+}
+
 export function optimizeImageForTransport(
   base64Data: string,
   mediaType: string,
 ): { data: string; mediaType: string } {
   const rawBytes = Buffer.from(base64Data, "base64");
-
-  // Small images don't need optimization.
-  if (rawBytes.length <= OPTIMIZE_THRESHOLD_BYTES) {
-    return { data: base64Data, mediaType };
-  }
-
-  // If dimensions are already within limits, skip.
   const dims = parseImageDimensions(base64Data, mediaType);
-  if (
-    dims &&
-    dims.width <= MAX_DIMENSION &&
-    dims.height <= MAX_DIMENSION
-  ) {
+
+  if (!shouldRescaleImage(dims, rawBytes.length)) {
     return { data: base64Data, mediaType };
   }
 

package/src/agent/loop.ts

@@ -1,6 +1,11 @@
 import * as Sentry from "@sentry/node";
 
-import { estimateToolsTokens } from "../context/token-estimator.js";
+import type { LLMCallSite } from "../config/schemas/llm.js";
+import {
+  estimatePromptTokensRaw,
+  estimateToolsTokens,
+  getCalibrationProviderKey,
+} from "../context/token-estimator.js";
 import { truncateOversizedToolResults } from "../context/tool-result-truncation.js";
 import { getHookManager } from "../hooks/manager.js";
 import type {
@@ -15,7 +20,9 @@ import {
   applyStreamingSubstitution,
   applySubstitutions,
 } from "../tools/sensitive-output-placeholders.js";
+import { ProviderError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
+import { isRetryableNetworkError } from "../util/retry.js";
 
 const log = getLogger("agent-loop");
 
@@ -23,7 +30,7 @@ export interface AgentLoopConfig {
   maxTokens: number;
   maxInputTokens?: number; // context window size for tool result truncation
   thinking?: { enabled: boolean };
-  effort: "low" | "medium" | "high" | "max";
+  effort: "low" | "medium" | "high" | "xhigh" | "max";
   speed?: "standard" | "fast";
   toolChoice?:
     | { type: "auto" }
@@ -100,6 +107,13 @@ export type AgentEvent =
       providerDurationMs: number;
       rawRequest?: unknown;
       rawResponse?: unknown;
+      /**
+       * Pre-send token estimate for the same call. Used by the estimator
+       * calibrator to learn how off the heuristic is versus provider
+       * ground truth. Omitted only when estimation genuinely was not run
+       * for this call (e.g. legacy/stubbed code paths).
+       */
+      estimatedInputTokens?: number;
     };
 
 const DEFAULT_CONFIG: AgentLoopConfig = {
@@ -109,6 +123,43 @@ const DEFAULT_CONFIG: AgentLoopConfig = {
 };
 
 const MAX_CONSECUTIVE_ERROR_NUDGES = 3;
+const MAX_EMPTY_RESPONSE_RETRIES = 1;
+
+/**
+ * User-config HTTP status codes that should never page the on-call: billing
+ * exhaustion (402), invalid credentials (401), and forbidden/plan-gated (403).
+ * The user-facing error path already surfaces an actionable message (e.g.
+ * credits_exhausted); a Sentry issue adds noise without engineering signal.
+ */
+const USER_CONFIG_STATUS_CODES = new Set([401, 402, 403]);
+
+/**
+ * Whether an agent-loop error should be reported to Sentry. Suppresses:
+ *
+ *  - `ProviderError` carrying a user-config status code (401/402/403) — these
+ *    are bad API keys, exhausted billing, or plan gates, not engineering bugs.
+ *  - Retry-exhausted transient network errors (`retriesExhausted === true` +
+ *    still categorized as retryable network) — the retry loop already tried
+ *    its best; the user's network was flaky, not our code.
+ *
+ * Everything else (5xx with no retry-exhaustion tag, surprise errors, tool
+ * failures, etc.) still pages.
+ */
+export function shouldCaptureAgentLoopError(err: Error): boolean {
+  if (
+    err instanceof ProviderError &&
+    err.statusCode !== undefined &&
+    USER_CONFIG_STATUS_CODES.has(err.statusCode)
+  ) {
+    return false;
+  }
+  const exhausted = (err as Error & { retriesExhausted?: boolean })
+    .retriesExhausted;
+  if (exhausted === true && isRetryableNetworkError(err)) {
+    return false;
+  }
+  return true;
+}
 
 export interface ResolvedSystemPrompt {
   systemPrompt: string;
@@ -203,10 +254,13 @@ export class AgentLoop {
     signal?: AbortSignal,
     requestId?: string,
     onCheckpoint?: (checkpoint: CheckpointInfo) => CheckpointDecision,
+    callSite?: LLMCallSite,
   ): Promise<Message[]> {
     const history = [...messages];
+    const initialHistoryLength = messages.length;
     let toolUseTurns = 0;
     let consecutiveErrorTurns = 0;
+    let emptyResponseRetries = 0;
     let lastLlmCallTime = 0;
     const rlog = requestId ? log.child({ requestId }) : log;
 
@@ -219,6 +273,11 @@ export class AgentLoop {
     while (true) {
       if (signal?.aborted) break;
 
+      rlog.info(
+        { turn: toolUseTurns, messageCount: history.length },
+        "Agent loop iteration start",
+      );
+
       let toolUseBlocks: Extract<ContentBlock, { type: "tool_use" }>[] = [];
 
       try {
@@ -233,25 +292,47 @@ export class AgentLoop {
           ? this.resolveSystemPrompt(history)
           : null;
         const turnSystemPrompt = resolved?.systemPrompt ?? this.systemPrompt;
-        const turnMaxTokens = resolved?.maxTokens ?? this.config.maxTokens;
         const turnModel = resolved?.model;
 
-        const providerConfig: Record<string, unknown> = {
-          max_tokens: turnMaxTokens,
-        };
-        if (turnModel) {
-          providerConfig.model = turnModel;
-        }
-        if (this.config.thinking?.enabled) {
-          providerConfig.thinking = { type: "adaptive" };
+        // Field precedence (highest wins):
+        //   1. Per-turn explicit (`resolved.maxTokens` / `resolved.model`)
+        //   2. Call-site resolved values (filled by
+        //      `RetryProvider.normalizeSendMessageOptions` from
+        //      `resolveCallSiteConfig(callSite, llm)`)
+        //   3. Conversation defaults (`this.config.*`, sourced from
+        //      `llm.default`)
+        //
+        // When `callSite` is present we deliberately leave
+        // `max_tokens`/`thinking`/`effort`/`speed` *unset* in `providerConfig`
+        // so the normalizer can fill them from the call-site resolution. The
+        // normalizer only writes these fields when they're undefined; if we
+        // pre-set them from `this.config` here, every per-call-site override
+        // for these knobs is silently ignored.
+        //
+        // `toolChoice` and `cacheTtl` are not part of the call-site schema, so
+        // they always come from `this.config` regardless of `callSite`.
+        const providerConfig: Record<string, unknown> = {};
+
+        if (resolved?.maxTokens !== undefined) {
+          providerConfig.max_tokens = resolved.maxTokens;
+        } else if (!callSite) {
+          providerConfig.max_tokens = this.config.maxTokens;
         }
 
-        if (this.config.effort) {
-          providerConfig.effort = this.config.effort;
+        if (turnModel) {
+          providerConfig.model = turnModel;
         }
 
-        if (this.config.speed && this.config.speed !== "standard") {
-          providerConfig.speed = this.config.speed;
+        if (!callSite) {
+          if (this.config.thinking?.enabled) {
+            providerConfig.thinking = { type: "adaptive" };
+          }
+          if (this.config.effort) {
+            providerConfig.effort = this.config.effort;
+          }
+          if (this.config.speed && this.config.speed !== "standard") {
+            providerConfig.speed = this.config.speed;
+          }
         }
 
         if (this.config.toolChoice) {
@@ -262,6 +343,17 @@ export class AgentLoop {
           providerConfig.cacheTtl = this.config.cacheTtl;
         }
 
+        // Per-call LLM call-site identifier. Surfaces on the per-call
+        // `config.callSite` so `RetryProvider.normalizeSendMessageOptions`
+        // can route through `resolveCallSiteConfig` against
+        // `llm.callSites.<id>` (falling back to `llm.default` when absent).
+        // User-initiated conversation turns default to `mainAgent` in the
+        // agent loop's caller; other invocation contexts (heartbeat, filing,
+        // analyze, etc.) pass their own `callSite`.
+        if (callSite) {
+          providerConfig.callSite = callSite;
+        }
+
         const preLlmResult = await getHookManager().trigger("pre-llm-call", {
           systemPrompt: turnSystemPrompt,
           messages: history,
@@ -290,6 +382,25 @@ export class AgentLoop {
         const providerStart = Date.now();
         lastLlmCallTime = providerStart;
 
+        // Compute the pre-send estimate against the full in-memory
+        // history — matching what upstream callers of
+        // `estimatePromptTokens` (preflight, mid-loop checkpoints, the
+        // window manager) see. We use the RAW estimate (before applying
+        // the existing correction) so the calibrator learns the true
+        // bias against provider ground truth instead of ratcheting a
+        // feedback loop against its own corrected output.
+        const toolTokenBudget =
+          currentTools.length > 0 ? estimateToolsTokens(currentTools) : 0;
+        const preSendEstimatedTokens = estimatePromptTokensRaw(
+          history,
+          turnSystemPrompt,
+          {
+            providerName: getCalibrationProviderKey(this.provider),
+            toolTokenBudget,
+          },
+        );
+        rlog.info({ turn: toolUseTurns }, "LLM call start");
+
         // Strip image contentBlocks from older tool results to prevent
         // screenshots from accumulating in the context window. The LLM
         // already saw each image on the turn it was captured; keeping
@@ -370,6 +481,7 @@ export class AgentLoop {
           providerDurationMs,
           rawRequest: response.rawRequest,
           rawResponse: response.rawResponse,
+          estimatedInputTokens: preSendEstimatedTokens,
         });
 
         void getHookManager().trigger("post-llm-call", {
@@ -398,9 +510,6 @@ export class AgentLoop {
           role: "assistant",
           content: response.content,
         };
-        history.push(assistantMessage);
-
-        await onEvent({ type: "message_complete", message: assistantMessage });
 
         // Check for tool use
         toolUseBlocks = response.content.filter(
@@ -408,6 +517,100 @@ export class AgentLoop {
             block.type === "tool_use",
         );
 
+        rlog.info(
+          {
+            turn: toolUseTurns,
+            stopReason: response.stopReason,
+            contentBlocks: response.content.length,
+            toolUseCount: toolUseBlocks.length,
+            durationMs: providerDurationMs,
+          },
+          "LLM call complete",
+        );
+
+        // Detect empty responses: no user-visible text and no tool calls.
+        // This can happen when the model fails to produce output after
+        // receiving a large tool result. Retry once with a nudge before
+        // the message is persisted.
+        //
+        // Only nudge when the model hasn't already delivered text to the user
+        // earlier in this tool-use chain. If a prior assistant turn in history
+        // contained visible text (e.g. the model said its piece before calling
+        // a side-effect tool like `remember`), an empty follow-up is the model
+        // correctly ending its turn — nudging would mislead it into thinking
+        // its earlier text didn't land and cause a verbatim re-send.
+        //
+        // Note: we check ANY prior assistant turn from this run()
+        // invocation, not just the most recent one. In multi-step tool-use
+        // chains (say-something → call-tool → call-another-tool → end),
+        // the "say-something" text lives on an earlier assistant turn while
+        // the most recent assistant turn is a pure tool_use with no text.
+        // Restricting the check to the most recent assistant turn would
+        // falsely nudge in that case and trigger a duplicate re-send of
+        // text the user already saw.
+        //
+        // Scope the scan to messages appended during this run() call only.
+        // Assistant text from prior conversation turns (earlier run()
+        // invocations passed in via `messages`) must NOT suppress the
+        // nudge — those turns completed long ago and have no bearing on
+        // whether the current tool-use chain has delivered text yet.
+        const hasVisibleText = response.content.some(
+          (block) => block.type === "text" && block.text.trim().length > 0,
+        );
+        const priorAssistantHadVisibleText = (() => {
+          for (let i = history.length - 1; i >= initialHistoryLength; i--) {
+            const msg = history[i];
+            if (msg.role !== "assistant") continue;
+            const hasText = msg.content.some(
+              (block) =>
+                block.type === "text" &&
+                typeof (block as { text?: unknown }).text === "string" &&
+                (block as { text: string }).text.trim().length > 0,
+            );
+            if (hasText) return true;
+          }
+          return false;
+        })();
+        if (
+          !hasVisibleText &&
+          toolUseBlocks.length === 0 &&
+          toolUseTurns > 0 &&
+          !priorAssistantHadVisibleText &&
+          emptyResponseRetries < MAX_EMPTY_RESPONSE_RETRIES
+        ) {
+          emptyResponseRetries++;
+          rlog.warn(
+            { turn: toolUseTurns, retry: emptyResponseRetries },
+            "Model returned empty response after tool results — retrying",
+          );
+          history.push({
+            role: "user",
+            content: [
+              {
+                type: "text",
+                text: "<system_notice>Your previous response was empty. You must respond to the user with a summary of what you found or did. Do not use any tools — just respond with text.</system_notice>",
+              },
+            ],
+          });
+          continue;
+        }
+
+        if (
+          !hasVisibleText &&
+          toolUseBlocks.length === 0 &&
+          toolUseTurns > 0 &&
+          !priorAssistantHadVisibleText
+        ) {
+          rlog.error(
+            { turn: toolUseTurns, retries: emptyResponseRetries },
+            "Model returned empty response after tool results — retries exhausted",
+          );
+        }
+
+        history.push(assistantMessage);
+
+        await onEvent({ type: "message_complete", message: assistantMessage });
+
         if (toolUseBlocks.length === 0 || !this.toolExecutor) {
           break;
         }
@@ -439,6 +642,15 @@ export class AgentLoop {
         // Execute all tools concurrently for reduced latency.
         // Race against the abort signal so cancellation isn't blocked by
         // stuck tools (e.g. a hung browser navigation).
+        const toolExecStart = Date.now();
+        rlog.info(
+          {
+            turn: toolUseTurns,
+            toolNames: toolUseBlocks.map((t) => t.name),
+          },
+          "Tool execution start",
+        );
+
         const toolExecutionPromise = Promise.all(
           toolUseBlocks.map(async (toolUse) => {
             const result = await this.toolExecutor!(
@@ -482,6 +694,15 @@ export class AgentLoop {
           toolResults = await toolExecutionPromise;
         }
 
+        rlog.info(
+          {
+            turn: toolUseTurns,
+            toolCount: toolResults.length,
+            durationMs: Date.now() - toolExecStart,
+          },
+          "Tool execution complete",
+        );
+
         // Merge sensitive output bindings from tool results into the
         // per-run substitution map. Bindings carry placeholder->value pairs
         // that are resolved in streamed text deltas and final message text.
@@ -613,12 +834,23 @@ export class AgentLoop {
           { err, turn: toolUseTurns, messageCount: history.length },
           "Agent loop error during turn processing",
         );
-        Sentry.captureException(err);
+        if (shouldCaptureAgentLoopError(err)) {
+          Sentry.captureException(err);
+        }
         onEvent({ type: "error", error: err });
         break;
       }
     }
 
+    rlog.info(
+      {
+        turns: toolUseTurns,
+        finalMessageCount: history.length,
+        aborted: signal?.aborted ?? false,
+      },
+      "Agent loop exited",
+    );
+
     return history;
   }
 }

package/src/avatar/resvg-lazy.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Tests for resvg-lazy and the writeTraitsAndRenderAvatar fallback it feeds.
+ *
+ * Covers the graceful-degradation path when the native @resvg/resvg-js binding
+ * is missing (e.g. bun install skipped the platform-specific optional
+ * dependency). The lazy loader must warn once and report unavailability, and
+ * writeTraitsAndRenderAvatar must return `native_unavailable` so the HTTP
+ * layer can respond 503 instead of 500.
+ *
+ * Bun's `mock.module` evaluates factories eagerly on re-registration, which
+ * makes it awkward to flip a module between "throws" and "returns fake
+ * export" across tests in the same file. We instead install a single
+ * throwing mock at module scope for the require-path test, and use the
+ * `__setResvgCacheForTests` / `__resetResvgCacheForTests` hooks to drive the
+ * rest of the cases deterministically.
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+type LogCall = { bindings: unknown; msg: string };
+const warnCalls: LogCall[] = [];
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () => ({
+    warn: (bindings: unknown, msg: string) => {
+      warnCalls.push({ bindings, msg });
+    },
+    info: () => {},
+    error: () => {},
+    debug: () => {},
+    trace: () => {},
+    fatal: () => {},
+  }),
+}));
+
+mock.module("../util/platform.js", () => ({
+  AVATAR_IMAGE_FILENAME: "avatar-image.png",
+  getAvatarDir: () => "/tmp/vellum-test-avatar-never-written",
+}));
+
+// Install a throwing @resvg/resvg-js mock at module scope. Bun only calls
+// this factory the first time the module is required within the test worker;
+// later `require` calls return whatever was produced by that first call (or
+// throw, if the factory threw). That's exactly what we want for the
+// "require fails" test below.
+mock.module("@resvg/resvg-js", () => {
+  throw new Error("Cannot require module @resvg/resvg-js-darwin-x64");
+});
+
+describe("resvg-lazy — require failure path", () => {
+  beforeEach(async () => {
+    warnCalls.length = 0;
+    const { __resetResvgCacheForTests } = await import("./resvg-lazy.js");
+    __resetResvgCacheForTests();
+  });
+
+  test("real require failure triggers warn log with platform context", async () => {
+    const { isResvgAvailable } = await import("./resvg-lazy.js");
+
+    // Call twice — caching must prevent a duplicate warn.
+    expect(isResvgAvailable()).toBe(false);
+    expect(isResvgAvailable()).toBe(false);
+
+    expect(warnCalls.length).toBe(1);
+    const call = warnCalls[0]!;
+    const bindings = call.bindings as Record<string, unknown>;
+    expect(bindings.platform).toBe(process.platform);
+    expect(bindings.arch).toBe(process.arch);
+    expect(String(bindings.module)).toContain("@resvg/resvg-js-");
+    expect(bindings.err).toBeInstanceOf(Error);
+    expect(call.msg).toContain("@resvg/resvg-js");
+  });
+});
+
+describe("resvg-lazy — API shape when unavailable", () => {
+  beforeEach(async () => {
+    warnCalls.length = 0;
+    const { __setResvgCacheForTests } = await import("./resvg-lazy.js");
+    __setResvgCacheForTests({
+      available: false,
+      error: new Error("Cannot require module @resvg/resvg-js-darwin-x64"),
+    });
+  });
+
+  test("isResvgAvailable returns false", async () => {
+    const { isResvgAvailable } = await import("./resvg-lazy.js");
+    expect(isResvgAvailable()).toBe(false);
+  });
+
+  test("getResvg throws the underlying error", async () => {
+    const { getResvg } = await import("./resvg-lazy.js");
+    expect(() => getResvg()).toThrow(
+      /Cannot require module @resvg\/resvg-js-darwin-x64/,
+    );
+  });
+});
+
+describe("writeTraitsAndRenderAvatar — native module missing", () => {
+  beforeEach(async () => {
+    warnCalls.length = 0;
+    const { __setResvgCacheForTests } = await import("./resvg-lazy.js");
+    __setResvgCacheForTests({
+      available: false,
+      error: new Error("Cannot require module @resvg/resvg-js-darwin-x64"),
+    });
+  });
+
+  test("returns native_unavailable without attempting disk writes", async () => {
+    const { writeTraitsAndRenderAvatar } = await import("./traits-png-sync.js");
+
+    const result = writeTraitsAndRenderAvatar({
+      bodyShape: "blob",
+      eyeStyle: "curious",
+      color: "green",
+    });
+
+    expect(result.ok).toBe(false);
+    if (result.ok) return; // narrow for TypeScript
+    expect(result.reason).toBe("native_unavailable");
+    expect(result.message).toContain("@resvg/resvg-js");
+  });
+
+  test("returns invalid_traits (not native_unavailable) when traits are bad", async () => {
+    const { writeTraitsAndRenderAvatar } = await import("./traits-png-sync.js");
+
+    // Empty traits object fails the shape check before we ever consult resvg.
+    const result = writeTraitsAndRenderAvatar({
+      bodyShape: "",
+      eyeStyle: "",
+      color: "",
+    });
+
+    expect(result.ok).toBe(false);
+    if (result.ok) return;
+    expect(result.reason).toBe("invalid_traits");
+  });
+});