@vellumai/assistant 0.6.4 → 0.6.5

package/src/providers/anthropic/client.ts

@@ -15,12 +15,50 @@ import type {
   SendMessageOptions,
   ToolDefinition,
 } from "../types.js";
+import {
+  ContextOverflowError,
+  extractOverflowTokensFromMessage,
+} from "../types.js";
 
 const log = getLogger("anthropic-client");
 
 /** Validation-specific timeout (10s) so a stalled network doesn't block key submission. */
 const VALIDATION_TIMEOUT_MS = 10_000;
 
+/**
+ * Detect Anthropic's `prompt_too_long` context-overflow signal.
+ *
+ * Anthropic returns HTTP 400 with a body shaped as
+ *   `{ type: "error", error: { type: "invalid_request_error", message: "..." } }`
+ * where the inner `message` carries the "prompt is too long: N tokens > M
+ * maximum" text. The SDK stores the body at `APIError.error` and formats a
+ * top-level `message` by JSON-stringifying the body when no top-level
+ * `.message` key exists — so we probe both the nested body and the
+ * formatted string.
+ */
+export function detectAnthropicContextOverflow(
+  error: InstanceType<typeof Anthropic.APIError>,
+): { actualTokens?: number; maxTokens?: number } | null {
+  // 413 is theoretically adjacent but Anthropic does not emit it today.
+  if (error.status !== 400) return null;
+  const body = error.error as
+    | {
+        type?: string;
+        message?: string;
+        error?: { type?: string; message?: string };
+      }
+    | undefined;
+  const innerMessage =
+    (typeof body === "object" && body != null
+      ? (body.error?.message ?? body.message)
+      : undefined) ?? "";
+  const topLevelMessage = error.message ?? "";
+  const combined = `${innerMessage} ${topLevelMessage}`;
+  if (!/prompt.?is.?too.?long|prompt_too_long/i.test(combined)) return null;
+  // Prefer the clean inner message over the JSON-stringified top-level string.
+  return extractOverflowTokensFromMessage(innerMessage || topLevelMessage);
+}
+
 /** Rate-limit the orphaned-surrogate warning so a single bad stream can't flood logs. */
 const ORPHAN_WARNING_THROTTLE_MS = 60_000;
 let lastOrphanWarningMs = 0;
@@ -132,6 +170,26 @@ export const PLACEHOLDER_EMPTY_TURN =
 export const PLACEHOLDER_BLOCKS_OMITTED =
   "\x00__PLACEHOLDER__[internal blocks omitted]";
 
+// Compared against the payload with any leading `\x00` stripped, so the check
+// matches both the prefixed sentinel we emit and any bare variant that lost
+// the null byte in transit (e.g. the model echoing the text back without
+// reproducing the control character).
+const PLACEHOLDER_SENTINEL_BARE: ReadonlySet<string> = new Set([
+  PLACEHOLDER_EMPTY_TURN.slice(1),
+  PLACEHOLDER_BLOCKS_OMITTED.slice(1),
+]);
+
+/**
+ * True when the text is one of the provider's internal alternation-preserving
+ * sentinels, with or without the null-byte prefix. These must never be
+ * persisted or rendered to users — they exist only in outbound Anthropic API
+ * request bodies.
+ */
+export function isPlaceholderSentinelText(text: string): boolean {
+  const normalized = text.startsWith("\x00") ? text.slice(1) : text;
+  return PLACEHOLDER_SENTINEL_BARE.has(normalized);
+}
+
 /**
  * Synthetic placeholder injected as user-message content when Anthropic API
  * alternation requires a user turn but no real user content exists. Uses the
@@ -607,16 +665,27 @@ export class AnthropicProvider implements Provider {
       authToken?: string;
     } = {},
   ) {
+    this.streamTimeoutMs = options.streamTimeoutMs ?? 1_800_000;
+    // Pass the same deadline to the SDK so its per-request timeout can't
+    // fire before `createStreamTimeout` does. The SDK's default is 10 min,
+    // which truncates any request we intend to run longer than that. We add
+    // a 60s buffer so `createStreamTimeout` always wins — its abort reason
+    // produces a clearer error message than the SDK's generic timeout.
+    const sdkTimeoutMs = this.streamTimeoutMs + 60_000;
     this.client = options.authToken
       ? new Anthropic({
           apiKey: null,
           authToken: options.authToken,
           baseURL: options.baseURL,
+          timeout: sdkTimeoutMs,
         })
-      : new Anthropic({ apiKey, baseURL: options.baseURL });
+      : new Anthropic({
+          apiKey,
+          baseURL: options.baseURL,
+          timeout: sdkTimeoutMs,
+        });
     this.model = model;
     this.useNativeWebSearch = options.useNativeWebSearch ?? false;
-    this.streamTimeoutMs = options.streamTimeoutMs ?? 1_800_000;
   }
 
   async sendMessage(
@@ -631,6 +700,11 @@ export class AnthropicProvider implements Provider {
         | "5m"
         | "1h") ?? "1h";
     let sentMessages: Anthropic.MessageParam[] | undefined;
+    const startedAt = Date.now();
+    // Hoisted so the catch block can distinguish our inner stream timeout
+    // (30 min default) from an external transport abort (bun fetch deadline,
+    // edge LB, NAT idle) — only the latter should be retried.
+    let innerTimeoutSignal: AbortSignal | undefined;
     try {
       const formatted = messages
         .map((m) => {
@@ -720,10 +794,7 @@ export class AnthropicProvider implements Provider {
             )
               return false;
             const text = (c[0] as { text?: string }).text;
-            return (
-              text === PLACEHOLDER_EMPTY_TURN ||
-              text === PLACEHOLDER_BLOCKS_OMITTED
-            );
+            return typeof text === "string" && isPlaceholderSentinelText(text);
           };
           if (isPlaceholder(iContent)) {
             formatted.splice(i, 1);
@@ -747,7 +818,11 @@ export class AnthropicProvider implements Provider {
         max_tokens: callerMaxTokens,
         ...restConfig
       } = (config ?? {}) as Record<string, unknown> & {
-        effort?: Anthropic.OutputConfig["effort"];
+        // "xhigh" is an intermediate tier between "high" and "max" supported
+        // by newer Anthropic models (e.g. Opus 4.7). The SDK's OutputConfig
+        // type doesn't yet include it, so we widen to the internal effort
+        // union and cast when building mergedOutputConfig.
+        effort?: "low" | "medium" | "high" | "xhigh" | "max";
         speed?: "standard" | "fast";
         output_config?: Record<string, unknown>;
       };
@@ -761,7 +836,9 @@ export class AnthropicProvider implements Provider {
       const supportsEffort = !isHaiku;
       const mergedOutputConfig = {
         ...(output_config ?? {}),
-        ...(effort && supportsEffort ? { effort } : {}),
+        ...(effort && supportsEffort
+          ? { effort: effort as Anthropic.OutputConfig["effort"] }
+          : {}),
       };
       // Build cache_control objects: Haiku doesn't support the extended
       // cache TTL beta, so omit the ttl field for Haiku models.
@@ -948,6 +1025,7 @@ export class AnthropicProvider implements Provider {
 
       const { signal: timeoutSignal, cleanup: cleanupTimeout } =
         createStreamTimeout(this.streamTimeoutMs, signal);
+      innerTimeoutSignal = timeoutSignal;
 
       /** Minimal stream interface shared by MessageStream and BetaMessageStream. */
       interface UnifiedStream {
@@ -999,8 +1077,30 @@ export class AnthropicProvider implements Provider {
                 signal: timeoutSignal,
               }) as unknown as UnifiedStream);
 
+        // Buffer streaming text until it's clear the accumulated text isn't
+        // going to form a placeholder sentinel. Sentinels are injected into
+        // outbound requests for role alternation and are sometimes echoed by
+        // the model; holding back partial prefixes prevents them from
+        // flashing on the live UI before cleanAssistantContent strips them
+        // at persist time. Buffer is bounded by the longest sentinel (~45
+        // chars) and resets on every content_block_start.
+        const SENTINEL_TEXTS: readonly string[] = [
+          PLACEHOLDER_EMPTY_TURN,
+          PLACEHOLDER_EMPTY_TURN.slice(1),
+          PLACEHOLDER_BLOCKS_OMITTED,
+          PLACEHOLDER_BLOCKS_OMITTED.slice(1),
+        ];
+        const couldBeSentinelPrefix = (s: string): boolean =>
+          SENTINEL_TEXTS.some((sentinel) => sentinel.startsWith(s));
+        const isCompleteSentinel = (s: string): boolean =>
+          SENTINEL_TEXTS.includes(s);
+        let textBuffer = "";
+
         stream.on("text", (text) => {
-          onEvent?.({ type: "text_delta", text });
+          textBuffer += text;
+          if (couldBeSentinelPrefix(textBuffer)) return;
+          onEvent?.({ type: "text_delta", text: textBuffer });
+          textBuffer = "";
         });
 
         stream.on("thinking", (thinking) => {
@@ -1015,6 +1115,13 @@ export class AnthropicProvider implements Provider {
         let pendingInputJsonFlush: ReturnType<typeof setTimeout> | undefined;
 
         stream.on("streamEvent", (event) => {
+          // Reset the text sentinel buffer at each content-block boundary.
+          // A new block starts fresh; at the end of a block, flush any
+          // buffered text that is NOT a complete sentinel, and drop it if
+          // it is one.
+          if (event.type === "content_block_start") {
+            textBuffer = "";
+          }
           if (
             event.type === "content_block_start" &&
             event.content_block.type === "tool_use"
@@ -1078,6 +1185,11 @@ export class AnthropicProvider implements Provider {
             currentStreamingToolName = undefined;
             currentStreamingToolUseId = undefined;
             accumulatedInputJson = "";
+            // Flush residual text buffer unless it's exactly a sentinel.
+            if (textBuffer.length > 0 && !isCompleteSentinel(textBuffer)) {
+              onEvent?.({ type: "text_delta", text: textBuffer });
+            }
+            textBuffer = "";
           }
         });
 
@@ -1143,10 +1255,18 @@ export class AnthropicProvider implements Provider {
     } catch (error) {
       // Propagate a tagged AbortReason (set by the daemon at controller.abort())
       // so wrapped errors can be classified as user cancellation downstream.
+      const callerAborted = signal?.aborted === true;
       const abortReason =
-        signal?.aborted && isAbortReason(signal.reason)
-          ? signal.reason
+        callerAborted && isAbortReason(signal!.reason)
+          ? signal!.reason
           : undefined;
+      const elapsedMs = Date.now() - startedAt;
+      // Inner-timeout means OUR 30-min stream deadline fired, not the caller
+      // and not an external transport cutoff. We rewrite the message so the
+      // retry layer can distinguish it from a transport abort (which should
+      // retry) — a timed-out stream would almost certainly time out again.
+      const innerTimeoutFired =
+        innerTimeoutSignal?.aborted === true && !callerAborted;
       if (error instanceof Anthropic.APIError) {
         // Log detailed message structure for tool_use/tool_result ordering errors
         if (
@@ -1162,30 +1282,79 @@ export class AnthropicProvider implements Provider {
             "Anthropic 400: tool_use/tool_result pairing error — dumping message structure",
           );
         }
+        const isAbortMessage =
+          error.status === undefined &&
+          /request was aborted/i.test(error.message);
         if (abortReason) {
           log.info(
-            { abortReason, message: error.message },
+            { abortReason, elapsedMs, message: error.message },
             "Anthropic request aborted by daemon",
           );
+        } else if (isAbortMessage) {
+          log.error(
+            {
+              elapsedMs,
+              cause: error.cause,
+              callerSignalAborted: callerAborted,
+              innerTimeoutFired,
+              message: error.message,
+            },
+            innerTimeoutFired
+              ? "Anthropic stream timed out (inner streamTimeoutMs fired)"
+              : "Anthropic stream aborted by transport — no daemon or inner-timeout abort; likely bun fetch deadline, edge LB, or network idle cutoff",
+          );
         } else {
           log.error(
             {
               status: error.status,
+              elapsedMs,
               message: error.message,
               headers: Object.fromEntries(error.headers?.entries() ?? []),
             },
             `Anthropic API error (${error.status})`,
           );
         }
+        const overflow = detectAnthropicContextOverflow(error);
+        if (overflow) {
+          throw new ContextOverflowError(
+            `Anthropic API error (${error.status}): ${error.message}`,
+            "anthropic",
+            {
+              actualTokens: overflow.actualTokens,
+              maxTokens: overflow.maxTokens,
+              statusCode: error.status,
+              cause: error,
+            },
+          );
+        }
         const retryAfterMs = extractRetryAfterMs(error.headers);
         const errorOptions: {
           retryAfterMs?: number;
           abortReason?: unknown;
+          cause?: unknown;
         } = {};
-        if (retryAfterMs !== undefined) errorOptions.retryAfterMs = retryAfterMs;
+        if (retryAfterMs !== undefined)
+          errorOptions.retryAfterMs = retryAfterMs;
         if (abortReason) errorOptions.abortReason = abortReason;
+        // Only preserve the original error as `cause` for transport aborts
+        // without a daemon-tagged reason — it's the diagnostic signal the
+        // retry layer and log reader rely on. Don't leak it through the
+        // caller-aborted path, which already carries `abortReason`.
+        if (!abortReason && isAbortMessage) errorOptions.cause = error;
+        // Rewrite the message only for inner-timeout, so the retry layer
+        // won't retry a request that already hit its 30-min deadline.
+        const rewrittenMessage =
+          isAbortMessage && innerTimeoutFired
+            ? `Anthropic stream timed out after ${Math.round(elapsedMs / 1000)}s (inner streamTimeoutMs)`
+            : error.message;
+        // Only include the `(status)` parenthetical when the SDK surfaced a
+        // real HTTP status. Abort paths and mid-stream protocol errors have
+        // `error.status === undefined`, and string-interpolating that produces
+        // a confusing "Anthropic API error (undefined): …" message.
+        const statusPart =
+          error.status !== undefined ? ` (${error.status})` : "";
         throw new ProviderError(
-          `Anthropic API error (${error.status}): ${error.message}`,
+          `Anthropic API error${statusPart}: ${rewrittenMessage}`,
           "anthropic",
           error.status,
           Object.keys(errorOptions).length > 0 ? errorOptions : undefined,

package/src/providers/call-site-routing.ts

@@ -0,0 +1,71 @@
+/**
+ * Provider wrapper that routes each `sendMessage` call to a different
+ * underlying provider transport when the per-call `options.config.callSite`
+ * resolves to a provider name that differs from the default.
+ *
+ * Without this wrapper the conversation-level provider transport is fixed at
+ * construction time, so a per-call-site `llm.callSites.<id>.provider`
+ * override only affects the request *metadata* the downstream client sees —
+ * the actual HTTP transport still belongs to `llm.default.provider`. That
+ * means routing decisions like "send `memoryRetrieval` calls to OpenAI even
+ * though the main agent runs on Anthropic" silently fail.
+ *
+ * `CallSiteRoutingProvider` consults `resolveCallSiteConfig` per call. When
+ * the resolved provider name differs from the default's name and the
+ * registry can produce a Provider for it, the call is delegated to that
+ * provider; otherwise it falls back to the default. Other Provider interface
+ * surface area (`name`, `tokenEstimationProvider`) is delegated to the
+ * default so wrappers further out (e.g. `RateLimitProvider`) still see a
+ * stable identity.
+ */
+
+import { resolveCallSiteConfig } from "../config/llm-resolver.js";
+import { getConfig } from "../config/loader.js";
+import type {
+  Message,
+  Provider,
+  ProviderResponse,
+  SendMessageOptions,
+  ToolDefinition,
+} from "./types.js";
+
+export class CallSiteRoutingProvider implements Provider {
+  public readonly name: string;
+  public readonly tokenEstimationProvider?: string;
+
+  constructor(
+    private readonly defaultProvider: Provider,
+    private readonly getProviderByName: (name: string) => Provider | undefined,
+  ) {
+    this.name = defaultProvider.name;
+    this.tokenEstimationProvider = defaultProvider.tokenEstimationProvider;
+  }
+
+  async sendMessage(
+    messages: Message[],
+    tools?: ToolDefinition[],
+    systemPrompt?: string,
+    options?: SendMessageOptions,
+  ): Promise<ProviderResponse> {
+    const target = this.selectProvider(options);
+    return target.sendMessage(messages, tools, systemPrompt, options);
+  }
+
+  /**
+   * Pick the provider to route this call through. The default provider wins
+   * unless the per-call `callSite` resolves to a different provider name and
+   * the registry can produce a Provider for it.
+   */
+  private selectProvider(options?: SendMessageOptions): Provider {
+    const callSite = options?.config?.callSite;
+    if (!callSite) return this.defaultProvider;
+
+    const resolved = resolveCallSiteConfig(callSite, getConfig().llm);
+    if (resolved.provider === this.defaultProvider.name) {
+      return this.defaultProvider;
+    }
+
+    const alternative = this.getProviderByName(resolved.provider);
+    return alternative ?? this.defaultProvider;
+  }
+}

package/src/providers/gemini/client.ts

@@ -14,6 +14,58 @@ import type {
   SendMessageOptions,
   ToolDefinition,
 } from "../types.js";
+import {
+  ContextOverflowError,
+  extractOverflowTokensFromMessage,
+} from "../types.js";
+
+/**
+ * Token/context-specific phrases that reliably indicate context-overflow
+ * regardless of status code. These never appear in quota/rate-limit error
+ * messages, so matching one of these is safe on any accepted status.
+ */
+const GEMINI_CONTEXT_OVERFLOW_TOKEN_PATTERNS =
+  /token.?count.*exceeds|exceeds.*maximum.*tokens|prompt.?is.?too.?long|too.?many.?(?:input.?)?tokens|input.?too.?long|context.?length.?exceeded/i;
+
+/**
+ * Detect Gemini's context-overflow signals on an `ApiError`. Gemini surfaces
+ * this condition via its "RESOURCE_EXHAUSTED" category. The Gemini SDK's
+ * `ApiError` only exposes `status` and `message`, so we match on both.
+ *
+ * On 400 (INVALID_ARGUMENT, the Generative Language API path), the
+ * `resource.?exhausted` phrase alone is sufficient — only context-overflow
+ * is surfaced with that status.
+ *
+ * On 429 (RESOURCE_EXHAUSTED, the Vertex path), the same status is used for
+ * BOTH rate-limit quota exhaustion AND context-overflow. To discriminate,
+ * we require a token/context-specific phrase; the bare `resource.?exhausted`
+ * signal is too broad and would misclassify quota errors as overflow and
+ * bypass the retry path in `providers/retry.ts`.
+ */
+export function detectGeminiContextOverflow(
+  error: ApiError,
+): { actualTokens?: number; maxTokens?: number } | null {
+  const status = error.status;
+  // 400 = INVALID_ARGUMENT (prompt too long), 413 occasional,
+  // 429 with RESOURCE_EXHAUSTED is the Vertex path.
+  if (status !== 400 && status !== 413 && status !== 429) return null;
+  const message = error.message ?? "";
+
+  // 429 has two meanings (quota vs context-overflow) — require a
+  // token/context-specific phrase to classify as overflow.
+  if (status === 429) {
+    if (!GEMINI_CONTEXT_OVERFLOW_TOKEN_PATTERNS.test(message)) return null;
+    return extractOverflowTokensFromMessage(message);
+  }
+
+  // 400/413: either a token/context-specific phrase or the broader
+  // `resource.?exhausted` signal is a reliable overflow indicator.
+  const matches =
+    /resource.?exhausted/i.test(message) ||
+    GEMINI_CONTEXT_OVERFLOW_TOKEN_PATTERNS.test(message);
+  if (!matches) return null;
+  return extractOverflowTokensFromMessage(message);
+}
 
 const log = getLogger("gemini-client");
 
@@ -238,6 +290,19 @@ export class GeminiProvider implements Provider {
           ? signal.reason
           : undefined;
       if (error instanceof ApiError) {
+        const overflow = detectGeminiContextOverflow(error);
+        if (overflow) {
+          throw new ContextOverflowError(
+            `Gemini API error (${error.status}): ${error.message}`,
+            "gemini",
+            {
+              actualTokens: overflow.actualTokens,
+              maxTokens: overflow.maxTokens,
+              statusCode: error.status,
+              cause: error,
+            },
+          );
+        }
         throw new ProviderError(
           `Gemini API error (${error.status}): ${error.message}`,
           "gemini",
@@ -330,7 +395,6 @@ export class GeminiProvider implements Provider {
         case "tool_use":
           parts.push({
             functionCall: {
-              id: block.id,
               name: block.name,
               args: block.input,
             },
@@ -363,7 +427,6 @@ export class GeminiProvider implements Provider {
           }
           parts.push({
             functionResponse: {
-              id: block.tool_use_id,
               name: toolCallNames.get(block.tool_use_id) ?? block.tool_use_id,
               response: { output: outputText },
             },

package/src/providers/managed-proxy/constants.ts

@@ -28,7 +28,8 @@ export interface ManagedProviderMeta {
 export const MANAGED_PROVIDER_META: Record<string, ManagedProviderMeta> = {
   openai: {
     name: "openai",
-    managed: false,
+    managed: true,
+    proxyPath: "/v1/runtime-proxy/openai",
   },
   anthropic: {
     name: "anthropic",