@vellumai/assistant 0.4.55 → 0.4.57

package/src/providers/anthropic/client.ts

@@ -1,5 +1,6 @@
 import Anthropic from "@anthropic-ai/sdk";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/system-prompt.js";
 import { ProviderError } from "../../util/errors.js";
 import { getLogger } from "../../util/logger.js";
 import { extractRetryAfterMs } from "../../util/retry.js";
@@ -15,6 +16,45 @@ import type {
 
 const log = getLogger("anthropic-client");
 
+/**
+ * Validate an Anthropic API key by making a lightweight GET /v1/models call.
+ * Returns `{ valid: true }` on success or `{ valid: false, reason: string }` on failure.
+ */
+export async function validateAnthropicApiKey(
+  apiKey: string,
+): Promise<{ valid: true } | { valid: false; reason: string }> {
+  try {
+    const client = new Anthropic({ apiKey });
+    await client.models.list({ limit: 1 });
+    return { valid: true };
+  } catch (error) {
+    if (error instanceof Anthropic.APIError) {
+      if (error.status === 401) {
+        return { valid: false, reason: "API key is invalid or expired." };
+      }
+      if (error.status === 403) {
+        return {
+          valid: false,
+          reason: `Anthropic API error (${error.status}): ${error.message}`,
+        };
+      }
+      // Transient errors (429, 5xx, etc.) — validation is inconclusive,
+      // allow the key to be stored rather than blocking the user.
+      log.warn(
+        { status: error.status },
+        "Anthropic API returned a transient error during key validation — allowing key storage",
+      );
+      return { valid: true };
+    }
+    // Network errors — validation is inconclusive, allow key storage.
+    log.warn(
+      { error: error instanceof Error ? error.message : String(error) },
+      "Network error during Anthropic key validation — allowing key storage",
+    );
+    return { valid: true };
+  }
+}
+
 const TOOL_ID_RE = /[^a-wyzA-Z0-9_-]/g;
 
 const ANTHROPIC_SUPPORTED_IMAGE_TYPES = new Set([
@@ -234,6 +274,87 @@ function normalizeFollowingUserContent(
   };
 }
 
+/** Type-guard for server_tool_use blocks. */
+function isServerToolUseBlock(
+  block: unknown,
+): block is { type: "server_tool_use"; id: string; name: string } {
+  return (
+    typeof block === "object" &&
+    block != null &&
+    (block as { type: string }).type === "server_tool_use"
+  );
+}
+
+/** Type-guard for web_search_tool_result blocks. */
+function isWebSearchToolResultBlock(
+  block: unknown,
+): block is { type: "web_search_tool_result"; tool_use_id: string } {
+  return (
+    typeof block === "object" &&
+    block != null &&
+    (block as { type: string }).type === "web_search_tool_result"
+  );
+}
+
+/**
+ * Repair orphaned server_tool_use blocks within assistant messages.
+ * Server-side tools (e.g. web_search) are self-paired: the assistant message
+ * should contain both server_tool_use and its matching web_search_tool_result.
+ * When the stream is interrupted or the API returns a partial response, the
+ * web_search_tool_result may be missing. This function injects a synthetic
+ * error result for any orphaned server_tool_use block.
+ */
+function repairOrphanedServerToolUse(
+  messages: Anthropic.MessageParam[],
+): Anthropic.MessageParam[] {
+  return messages.map((msg) => {
+    if (msg.role !== "assistant") return msg;
+    const content = Array.isArray(msg.content) ? msg.content : [];
+
+    // Collect server_tool_use IDs and web_search_tool_result IDs in this message
+    const serverToolUseIds = new Set<string>();
+    const pairedResultIds = new Set<string>();
+    for (const block of content) {
+      if (isServerToolUseBlock(block)) {
+        serverToolUseIds.add(block.id);
+      }
+      if (isWebSearchToolResultBlock(block)) {
+        pairedResultIds.add(block.tool_use_id);
+      }
+    }
+
+    // Find orphaned server_tool_use blocks (no matching web_search_tool_result)
+    const orphanedIds: string[] = [];
+    for (const id of serverToolUseIds) {
+      if (!pairedResultIds.has(id)) {
+        orphanedIds.push(id);
+      }
+    }
+
+    if (orphanedIds.length === 0) return msg;
+
+    log.warn(
+      { orphanedIds, blockCount: content.length },
+      "Injecting synthetic web_search_tool_result for orphaned server_tool_use blocks",
+    );
+
+    // Insert a synthetic error web_search_tool_result after each orphaned server_tool_use
+    const repairedContent: Anthropic.ContentBlockParam[] = [];
+    for (const block of content) {
+      repairedContent.push(block);
+      if (isServerToolUseBlock(block) && orphanedIds.includes(block.id)) {
+        repairedContent.push({
+          type: "web_search_tool_result",
+          tool_use_id: block.id,
+          content: [],
+        } as unknown as Anthropic.ContentBlockParam);
+      }
+    }
+
+    return { role: msg.role, content: repairedContent };
+  });
+}
+
 /**
  * Last-line-of-defense fix that ensures every assistant message with tool_use
  * blocks has matching tool_result blocks in the immediately following user
@@ -538,7 +659,7 @@ export class AnthropicProvider implements Provider {
         }
       }
 
-      sentMessages = ensureToolPairing(formatted);
+      sentMessages = ensureToolPairing(repairOrphanedServerToolUse(formatted));
       const { effort, output_config, ...restConfig } = (config ?? {}) as Record<
         string,
         unknown
@@ -546,9 +667,15 @@ export class AnthropicProvider implements Provider {
         effort?: Anthropic.OutputConfig["effort"];
         output_config?: Record<string, unknown>;
       };
+      // Haiku does not support the effort / output_config parameter.
+      // Determine the effective model (per-call override or provider default)
+      // and strip effort when the model doesn't support it.
+      const effectiveModel =
+        (restConfig as Record<string, unknown>).model?.toString() ?? this.model;
+      const supportsEffort = !effectiveModel.includes("haiku");
       const mergedOutputConfig = {
         ...(output_config ?? {}),
-        ...(effort ? { effort } : {}),
+        ...(effort && supportsEffort ? { effort } : {}),
       };
       const params: Anthropic.MessageStreamParams = {
         model: this.model,
@@ -561,13 +688,37 @@ export class AnthropicProvider implements Provider {
       };
 
       if (systemPrompt) {
-        params.system = [
-          {
-            type: "text" as const,
-            text: systemPrompt,
-            cache_control: { type: "ephemeral" as const },
-          },
-        ];
+        const boundaryIdx = systemPrompt.indexOf(SYSTEM_PROMPT_CACHE_BOUNDARY);
+        if (boundaryIdx >= 0) {
+          // Split into two cache blocks: static instructions (stable across
+          // turns) and dynamic workspace content (changes when files are
+          // edited).  The static prefix stays cached even when workspace
+          // files change, saving ~8-10K tokens of cache creation per turn.
+          const staticBlock = systemPrompt.slice(0, boundaryIdx);
+          const dynamicBlock = systemPrompt.slice(
+            boundaryIdx + SYSTEM_PROMPT_CACHE_BOUNDARY.length,
+          );
+          params.system = [
+            {
+              type: "text" as const,
+              text: staticBlock,
+              cache_control: { type: "ephemeral" as const },
+            },
+            {
+              type: "text" as const,
+              text: dynamicBlock,
+              cache_control: { type: "ephemeral" as const },
+            },
+          ];
+        } else {
+          params.system = [
+            {
+              type: "text" as const,
+              text: systemPrompt,
+              cache_control: { type: "ephemeral" as const },
+            },
+          ];
+        }
       }
 
       if (tools && tools.length > 0) {
@@ -602,13 +753,25 @@ export class AnthropicProvider implements Provider {
         }
       }
 
-      // Place cache breakpoints on the last two user turns so the
+      // Place a cache breakpoint on the second-to-last user turn so the
       // conversation prefix is cached between agent-loop iterations.
+      //
+      // Why second-to-last, not last?  The last user message is always new
+      // (either the initial message with fresh temporal context, or a tool
+      // result appended by the agent loop) so its breakpoint never produces
+      // a cache hit.  The second-to-last user turn is stable between
+      // iterations and caching up to it saves re-processing the full
+      // conversation prefix.
+      //
+      // We use only 1 user-turn breakpoint to stay within the Anthropic
+      // API limit of 4 cache_control blocks total:
+      //   system-static (1) + system-dynamic (2) + last-tool (3) + user (4)
       const userIndices: number[] = [];
       for (let i = 0; i < params.messages.length; i++) {
         if (params.messages[i].role === "user") userIndices.push(i);
       }
-      for (const idx of userIndices.slice(-2)) {
+      // slice(-2, -1) gives the second-to-last; empty array if < 2 user turns.
+      for (const idx of userIndices.slice(-2, -1)) {
         const content = params.messages[idx].content;
         if (Array.isArray(content) && content.length > 0) {
           (
@@ -929,12 +1092,17 @@ export class AnthropicProvider implements Provider {
         };
       case "tool_result": {
         const toolUseId = sanitizeToolId(block.tool_use_id);
-        if (block.contentBlocks && block.contentBlocks.length > 0) {
+        // Anthropic API: when is_error is true, all content must be type "text".
+        // Filter out non-text blocks (e.g. images) for error results.
+        const usableBlocks = block.is_error
+          ? block.contentBlocks?.filter((cb) => cb.type === "text")
+          : block.contentBlocks;
+        if (usableBlocks && usableBlocks.length > 0) {
           // Build rich content array: text + images for Anthropic's native multi-part tool results
           const parts: Anthropic.ToolResultBlockParam["content"] = [
             { type: "text" as const, text: block.content },
           ];
-          for (const cb of block.contentBlocks) {
+          for (const cb of usableBlocks) {
             if (cb.type === "image") {
               parts.push({
                 type: "image" as const,

package/src/providers/gemini/client.ts

@@ -1,6 +1,7 @@
 import type * as genai from "@google/genai";
 import { ApiError, GoogleGenAI } from "@google/genai";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { ProviderError } from "../../util/errors.js";
 import { createStreamTimeout } from "../stream-timeout.js";
 import type {
@@ -16,10 +17,6 @@ export interface GeminiProviderOptions {
   streamTimeoutMs?: number;
   /** When set, routes requests through the managed proxy at this base URL. */
   managedBaseUrl?: string;
-  /** Vertex AI project placeholder (used with managed proxy). */
-  vertexProject?: string;
-  /** Vertex AI location placeholder (used with managed proxy). */
-  vertexLocation?: string;
 }
 
 export class GeminiProvider implements Provider {
@@ -35,12 +32,9 @@ export class GeminiProvider implements Provider {
   ) {
     this.client = options.managedBaseUrl
       ? new GoogleGenAI({
-          vertexai: true,
-          project: options.vertexProject ?? "proxy",
-          location: options.vertexLocation ?? "us-central1",
+          apiKey,
           httpOptions: {
             baseUrl: options.managedBaseUrl,
-            headers: { Authorization: `Bearer ${apiKey}` },
           },
         })
       : new GoogleGenAI({ apiKey });
@@ -65,7 +59,10 @@ export class GeminiProvider implements Provider {
       const geminiConfig: genai.GenerateContentConfig = {};
 
       if (systemPrompt) {
-        geminiConfig.systemInstruction = systemPrompt;
+        geminiConfig.systemInstruction = systemPrompt.replaceAll(
+          SYSTEM_PROMPT_CACHE_BOUNDARY,
+          "\n",
+        );
       }
       if (maxTokens) {
         geminiConfig.maxOutputTokens = maxTokens;

package/src/providers/managed-proxy/constants.ts

@@ -38,7 +38,7 @@ export const MANAGED_PROVIDER_META: Record<string, ManagedProviderMeta> = {
   gemini: {
     name: "gemini",
     managed: true,
-    proxyPath: "/v1/runtime-proxy/vertex",
+    proxyPath: "/v1/runtime-proxy/gemini",
   },
   fireworks: {
     name: "fireworks",
@@ -48,11 +48,5 @@ export const MANAGED_PROVIDER_META: Record<string, ManagedProviderMeta> = {
     name: "openrouter",
     managed: false,
   },
-  vertex: {
-    name: "vertex",
-    managed: true,
-    proxyPath: "/v1/runtime-proxy/vertex",
-  },
   ollama: { name: "ollama", managed: false },
 };
-

package/src/providers/managed-proxy/context.ts

@@ -29,6 +29,14 @@ export interface ManagedProxyContext {
   assistantApiKey: string;
 }
 
+/**
+ * Cached result of the last `resolveManagedProxyContext()` call.
+ * Updated every time the context is resolved (startup, credential changes).
+ * Defaults to `false` so that callers see a safe value before the first
+ * async resolution completes.
+ */
+let _managedProxyEnabled = false;
+
 /**
  * Resolve managed proxy context from environment and secure storage.
  *
@@ -39,8 +47,8 @@ export async function resolveManagedProxyContext(): Promise<ManagedProxyContext>
   const platformBaseUrl = getPlatformBaseUrl().replace(/\/+$/, "");
   const assistantApiKey =
     (await getSecureKeyAsync(ASSISTANT_API_KEY_STORAGE_KEY)) ?? "";
-
   const enabled = !!platformBaseUrl && !!assistantApiKey;
+  _managedProxyEnabled = enabled;
 
   return { enabled, platformBaseUrl, assistantApiKey };
 }
@@ -84,3 +92,20 @@ export async function managedFallbackEnabledFor(
   if (!meta?.managed) return false;
   return await hasManagedProxyPrereqs();
 }
+
+/**
+ * Synchronous check for whether managed proxy prerequisites were satisfied
+ * the last time `resolveManagedProxyContext()` ran.
+ *
+ * Returns `false` before the first async resolution or when the API key is
+ * missing. Safe for use in synchronous code paths (e.g. system prompt
+ * building) that cannot await the credential store.
+ */
+export function isManagedProxyEnabledSync(): boolean {
+  return _managedProxyEnabled;
+}
+
+/** @internal Test-only: reset the cached managed-proxy-enabled flag. */
+export function _resetManagedProxyEnabledCache(): void {
+  _managedProxyEnabled = false;
+}

package/src/providers/model-intents.ts

@@ -18,12 +18,12 @@ const PROVIDER_MODEL_INTENTS: Record<
   anthropic: {
     "latency-optimized": "claude-haiku-4-5-20251001",
     "quality-optimized": "claude-opus-4-6",
-    "vision-optimized": "claude-sonnet-4-6",
+    "vision-optimized": "claude-opus-4-6",
   },
   openai: {
-    "latency-optimized": "gpt-4o-mini",
-    "quality-optimized": "gpt-5.2",
-    "vision-optimized": "gpt-4o",
+    "latency-optimized": "gpt-5.4-nano",
+    "quality-optimized": "gpt-5.4",
+    "vision-optimized": "gpt-5.4",
   },
   gemini: {
     "latency-optimized": "gemini-3-flash",
@@ -42,7 +42,7 @@ const PROVIDER_MODEL_INTENTS: Record<
   },
   openrouter: {
     "latency-optimized": "x-ai/grok-4",
-    "quality-optimized": "x-ai/grok-4",
+    "quality-optimized": "x-ai/grok-4.20-beta",
     "vision-optimized": "x-ai/grok-4",
   },
 };

package/src/providers/openai/client.ts

@@ -1,5 +1,6 @@
 import OpenAI from "openai";
 
+import { SYSTEM_PROMPT_CACHE_BOUNDARY } from "../../prompts/cache-boundary.js";
 import { ProviderError } from "../../util/errors.js";
 import { extractRetryAfterMs } from "../../util/retry.js";
 import { escapeXmlAttr } from "../../util/xml.js";
@@ -18,6 +19,8 @@ export interface OpenAICompatibleProviderOptions {
   providerName?: string;
   providerLabel?: string;
   streamTimeoutMs?: number;
+  /** Extra params spread into every chat.completions.create call (e.g. reasoning). */
+  extraCreateParams?: Record<string, unknown>;
 }
 
 const OPENAI_SUPPORTED_IMAGE_TYPES = new Set([
@@ -33,6 +36,7 @@ export class OpenAIProvider implements Provider {
   private client: OpenAI;
   private model: string;
   private streamTimeoutMs: number;
+  private extraCreateParams: Record<string, unknown>;
 
   constructor(
     apiKey: string,
@@ -47,6 +51,7 @@ export class OpenAIProvider implements Provider {
     });
     this.model = model;
     this.streamTimeoutMs = options.streamTimeoutMs ?? 300_000;
+    this.extraCreateParams = options.extraCreateParams ?? {};
   }
 
   async sendMessage(
@@ -69,6 +74,7 @@ export class OpenAIProvider implements Provider {
           messages: openaiMessages,
           stream: true as const,
           stream_options: { include_usage: true },
+          ...this.extraCreateParams,
         };
 
       if (maxTokens) {
@@ -224,7 +230,10 @@ export class OpenAIProvider implements Provider {
     const result: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [];
 
     if (systemPrompt) {
-      result.push({ role: "system", content: systemPrompt });
+      result.push({
+        role: "system",
+        content: systemPrompt.replaceAll(SYSTEM_PROMPT_CACHE_BOUNDARY, "\n"),
+      });
     }
 
     for (const msg of messages) {

package/src/providers/openrouter/client.ts

@@ -19,6 +19,7 @@ export class OpenRouterProvider extends OpenAIProvider {
       providerName: "openrouter",
       providerLabel: "OpenRouter",
       streamTimeoutMs: options.streamTimeoutMs,
+      extraCreateParams: { reasoning: { enabled: true } },
     });
   }
 }

package/src/providers/provider-availability.ts

@@ -0,0 +1,39 @@
+/**
+ * Provider availability checks.
+ *
+ * Determines which LLM providers are usable by checking secure storage,
+ * environment variable fallbacks, and managed proxy availability.
+ */
+
+import { API_KEY_PROVIDERS } from "../config/loader.js";
+import { getProviderKeyAsync } from "../security/secure-keys.js";
+import { managedFallbackEnabledFor } from "./managed-proxy/context.js";
+
+/**
+ * Check whether a single provider is usable — via a user-provided key
+ * (secure storage or env var) or via the managed proxy fallback.
+ * Ollama is always considered available because it does not require an API key.
+ */
+export async function isProviderAvailable(provider: string): Promise<boolean> {
+  if (provider === "ollama") return true;
+  return !!(
+    (await getProviderKeyAsync(provider)) ||
+    (await managedFallbackEnabledFor(provider))
+  );
+}
+
+/**
+ * Build the list of providers that are usable — via a user-provided key
+ * (secure storage or env var) or via the managed proxy fallback.
+ * Ollama is always included because it does not require an API key.
+ */
+export async function getConfiguredProviders(): Promise<string[]> {
+  const configured: string[] = [];
+  for (const p of API_KEY_PROVIDERS) {
+    if (await isProviderAvailable(p)) {
+      configured.push(p);
+    }
+  }
+  if (!configured.includes("ollama")) configured.push("ollama");
+  return configured;
+}

package/src/providers/provider-send-message.ts

@@ -30,12 +30,20 @@ export interface ConfiguredProviderResult {
 const providerSelectionLog = getLogger("provider-selection");
 let fallbackWarningLogged = false;
 
+/**
+ * Cached promise for the lazy initialization path inside
+ * `resolveConfiguredProvider`. When multiple concurrent callers enter before
+ * providers are initialized, they all await the same promise instead of
+ * each triggering a redundant `initializeProviders` call.
+ */
+let lazyInitPromise: Promise<void> | null = null;
+
 /**
  * Resolve the configured provider with full selection metadata.
  * If providers haven't been initialized yet (e.g. non-daemon code paths),
  * performs a one-shot `initializeProviders(getConfig())`.
  *
- * Uses fail-open selection: if `config.provider` is unavailable but
+ * Uses fail-open selection: if the configured provider is unavailable but
  * alternates from `config.providerOrder` exist, selects the first available.
  *
  * Returns `null` when no providers are available at all.
@@ -44,8 +52,13 @@ export async function resolveConfiguredProvider(): Promise<ConfiguredProviderRes
   const config = getConfig();
 
   if (listProviders().length === 0) {
+    if (!lazyInitPromise) {
+      lazyInitPromise = initializeProviders(config).finally(() => {
+        lazyInitPromise = null;
+      });
+    }
     try {
-      await initializeProviders(config);
+      await lazyInitPromise;
     } catch {
       return null;
     }
@@ -54,7 +67,8 @@ export async function resolveConfiguredProvider(): Promise<ConfiguredProviderRes
   const providerOrder = Array.isArray(config.providerOrder)
     ? config.providerOrder
     : [];
-  const selection = resolveProviderSelection(config.provider, providerOrder);
+  const inferenceProvider = config.services.inference.provider;
+  const selection = resolveProviderSelection(inferenceProvider, providerOrder);
 
   if (!selection.selectedPrimary) {
     return null;
@@ -63,17 +77,17 @@ export async function resolveConfiguredProvider(): Promise<ConfiguredProviderRes
   if (selection.usedFallbackPrimary) {
     const level = fallbackWarningLogged ? "debug" : "warn";
     providerSelectionLog[level](
-      { configured: config.provider, selected: selection.selectedPrimary },
+      { configured: inferenceProvider, selected: selection.selectedPrimary },
       "Configured provider unavailable, using fallback",
     );
     fallbackWarningLogged = true;
   }
 
   try {
-    const provider = getFailoverProvider(config.provider, providerOrder);
+    const provider = getFailoverProvider(inferenceProvider, providerOrder);
     return {
       provider,
-      configuredProviderName: config.provider,
+      configuredProviderName: inferenceProvider,
       selectedProviderName: selection.selectedPrimary,
       usedFallbackPrimary: selection.usedFallbackPrimary,
     };

package/src/providers/ratelimit.ts

@@ -15,7 +15,6 @@ export class RateLimitProvider implements Provider {
   public readonly name: string;
 
   private requestTimestamps: number[];
-  private sessionTokens = 0;
 
   constructor(
     private readonly inner: Provider,
@@ -33,7 +32,6 @@ export class RateLimitProvider implements Provider {
     options?: SendMessageOptions,
   ): Promise<ProviderResponse> {
     this.enforceRequestRate();
-    this.enforceTokenBudget();
 
     // Record the request timestamp before the await to prevent concurrent
     // calls from bypassing the rate limit during the async gap.
@@ -46,8 +44,6 @@ export class RateLimitProvider implements Provider {
       options,
     );
 
-    this.recordTokens(response.usage.inputTokens + response.usage.outputTokens);
-
     return response;
   }
 
@@ -89,39 +85,8 @@ export class RateLimitProvider implements Provider {
     }
   }
 
-  private enforceTokenBudget(): void {
-    const limit = this.config.maxTokensPerSession;
-    if (limit <= 0) return;
-
-    if (this.sessionTokens >= limit) {
-      log.warn(
-        {
-          provider: this.name,
-          sessionTokens: this.sessionTokens,
-          limit,
-        },
-        `Session token budget exhausted for ${this.name}: ${this.sessionTokens.toLocaleString()}/${limit.toLocaleString()}`,
-      );
-      throw new RateLimitError(
-        `Session token budget exhausted: ${this.sessionTokens.toLocaleString()}/${limit.toLocaleString()} tokens used. Start a new session to continue.`,
-      );
-    }
-  }
-
   private recordRequest(): void {
     if (this.config.maxRequestsPerMinute <= 0) return;
     this.requestTimestamps.push(Date.now());
   }
-
-  private recordTokens(tokens: number): void {
-    if (this.config.maxTokensPerSession <= 0) return;
-    this.sessionTokens += tokens;
-    log.debug(
-      {
-        sessionTokens: this.sessionTokens,
-        limit: this.config.maxTokensPerSession,
-      },
-      "Token usage updated",
-    );
-  }
 }