@vellumai/assistant 0.6.6 → 0.7.1

package/src/providers/__tests__/retry-callsite.test.ts

@@ -122,6 +122,117 @@ describe("RetryProvider — callSite resolution", () => {
     expect(config.modelIntent).toBeUndefined();
   });
 
+  test("attaches sanitized stable attribution headers only when enabled", async () => {
+    setLlmConfig({
+      default: {
+        provider: "anthropic",
+        model: "claude-default",
+      },
+      profiles: {
+        "conversation-profile": {
+          model: "claude-profile",
+          source: "user",
+        },
+      },
+      callSites: {
+        memoryRetrieval: {
+          provider: "openai",
+        },
+      },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("openai", (options) => {
+        seen = options;
+      }),
+      { forwardUsageAttributionHeaders: true },
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: {
+        callSite: "memoryRetrieval",
+        overrideProfile: "conversation-profile",
+      },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.usageAttributionHeaders).toEqual({
+      "X-Vellum-LLM-Call-Site": "memoryRetrieval",
+      "X-Vellum-Inference-Profile": "conversation-profile",
+      "X-Vellum-Inference-Profile-Source": "conversation",
+      "X-Vellum-Resolved-Provider": "openai",
+      "X-Vellum-Resolved-Model": "claude-profile",
+    });
+    expect(
+      (config.usageAttributionHeaders as Record<string, string>)[
+        "X-Vellum-LLM-Call-Site-Label"
+      ],
+    ).toBeUndefined();
+  });
+
+  test("omits attribution headers by default for direct provider transports", async () => {
+    setLlmConfig({
+      default: {
+        provider: "openai",
+        model: "gpt-default",
+      },
+      callSites: {
+        memoryRetrieval: {
+          provider: "openai",
+        },
+      },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("openai", (options) => {
+        seen = options;
+      }),
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: { callSite: "memoryRetrieval" },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.usageAttributionHeaders).toBeUndefined();
+    expect(config.callSite).toBeUndefined();
+  });
+
+  test("omits profile source attribution header when no profile is applied", async () => {
+    setLlmConfig({
+      default: {
+        provider: "openai",
+        model: "gpt-default",
+      },
+      callSites: {
+        memoryRetrieval: {
+          provider: "openai",
+        },
+      },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("openai", (options) => {
+        seen = options;
+      }),
+      { forwardUsageAttributionHeaders: true },
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: { callSite: "memoryRetrieval" },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.usageAttributionHeaders).toEqual({
+      "X-Vellum-LLM-Call-Site": "memoryRetrieval",
+      "X-Vellum-Resolved-Provider": "openai",
+      "X-Vellum-Resolved-Model": "gpt-default",
+    });
+  });
+
   test("falls back to llm.default when llm.callSites[id] is absent", async () => {
     setLlmConfig({
       default: {
@@ -179,16 +290,13 @@ describe("RetryProvider — callSite resolution", () => {
     expect(config.effort).toBe("high");
     expect(config.speed).toBe("fast");
     expect(config.temperature).toBe(0.7);
-    // Disabled thinking is omitted entirely so providers fall back to their
-    // default behavior — matches the legacy non-callSite path which only sets
-    // `providerConfig.thinking` when `enabled === true`.
-    expect(config.thinking).toBeUndefined();
+    expect(config.thinking).toEqual({ type: "disabled" });
     // `contextWindow` and `provider` are server-side concerns and must NOT
     // leak into the per-call provider config — Anthropic rejects unknown
     // fields with `{type:"invalid_request_error", message:"contextWindow:
     // Extra inputs are not permitted"}`. Provider routing is handled by
     // CallSiteRoutingProvider; contextWindow is consumed by the agent loop
-    // directly from `config.llm.default.contextWindow.*`.
+    // from the effective per-call-site/profile context resolver.
     expect(config.contextWindow).toBeUndefined();
     expect(config.provider).toBeUndefined();
   });
@@ -225,7 +333,7 @@ describe("RetryProvider — callSite resolution", () => {
     expect(config.thinking).toEqual({ type: "adaptive" });
   });
 
-  test("omits thinking when resolved config has thinking.enabled: false", async () => {
+  test("converts disabled thinking to Anthropic wire-format `{ type: 'disabled' }`", async () => {
     setLlmConfig({
       default: {
         provider: "anthropic",
@@ -249,7 +357,7 @@ describe("RetryProvider — callSite resolution", () => {
     });
 
     const config = seen?.config as Record<string, unknown>;
-    expect(config.thinking).toBeUndefined();
+    expect(config.thinking).toEqual({ type: "disabled" });
   });
 
   test("does NOT propagate temperature when resolved value is null (schema default)", async () => {
@@ -389,6 +497,32 @@ describe("RetryProvider — no callSite (pre-resolved config passes through)", (
     expect(config.model).not.toBe("MUST-NOT-LEAK");
     expect(config.model).not.toBe("ALSO-MUST-NOT-LEAK");
   });
+
+  test("does not forward caller-supplied attribution headers without callSite", async () => {
+    setLlmConfig({
+      default: { provider: "anthropic", model: "MUST-NOT-LEAK" },
+    });
+
+    let seen: SendMessageOptions | undefined;
+    const wrapped = new RetryProvider(
+      makeProvider("anthropic", (options) => {
+        seen = options;
+      }),
+    );
+
+    await wrapped.sendMessage(DUMMY_MESSAGES, undefined, undefined, {
+      config: {
+        model: "explicit-model",
+        usageAttributionHeaders: {
+          "X-Vellum-LLM-Call-Site": "injected",
+        },
+      },
+    });
+
+    const config = seen?.config as Record<string, unknown>;
+    expect(config.model).toBe("explicit-model");
+    expect(config.usageAttributionHeaders).toBeUndefined();
+  });
 });
 
 // ── getConfiguredProvider — call-site routing ──────────────────────────────

package/src/providers/anthropic/client.ts

@@ -771,22 +771,32 @@ export class AnthropicProvider implements Provider {
           return acc;
         }, []);
 
-      // Post-processing: fix consecutive same-role messages that can arise when
-      // dropping empty messages (e.g. empty assistant followed by empty user).
-      // Remove placeholder assistant messages that ended up adjacent to another
-      // assistant message, since the placeholder is no longer needed.
-      for (let i = formatted.length - 1; i > 0; i--) {
-        if (formatted[i].role !== formatted[i - 1].role) continue;
-        // For consecutive assistant messages, remove whichever one is a
-        // placeholder injected by the reduce above.
-        if (formatted[i].role === "assistant") {
-          const iContent = Array.isArray(formatted[i].content)
-            ? formatted[i].content
-            : [];
-          const prevContent = Array.isArray(formatted[i - 1].content)
-            ? formatted[i - 1].content
-            : [];
-          const isPlaceholder = (c: typeof iContent) => {
+      // Post-processing: merge consecutive same-role messages that violate
+      // Anthropic's strict user/assistant alternation requirement. These can
+      // arise from:
+      //   - Dropping empty messages in the reduce above (placeholder-adjacent)
+      //   - History reconstruction artifacts that bypass repairHistory
+      //
+      // Walk backwards so splice indices stay valid. After a merge+splice
+      // the element that was at i+1 shifts to i, potentially creating a
+      // new adjacent pair — bump i back up to recheck that position.
+      {
+        let i = formatted.length - 1;
+        while (i > 0 && i < formatted.length) {
+          if (formatted[i].role !== formatted[i - 1].role) {
+            i--;
+            continue;
+          }
+
+          const iContent = (
+            Array.isArray(formatted[i].content) ? formatted[i].content : []
+          ) as Anthropic.ContentBlockParam[];
+          const prevContent = (
+            Array.isArray(formatted[i - 1].content)
+              ? formatted[i - 1].content
+              : []
+          ) as Anthropic.ContentBlockParam[];
+          const isPlaceholder = (c: Anthropic.ContentBlockParam[]): boolean => {
             if (
               c.length !== 1 ||
               typeof c[0] === "string" ||
@@ -796,10 +806,50 @@ export class AnthropicProvider implements Provider {
             const text = (c[0] as { text?: string }).text;
             return typeof text === "string" && isPlaceholderSentinelText(text);
           };
+
           if (isPlaceholder(iContent)) {
             formatted.splice(i, 1);
+            // Removed the later element. The new formatted[i] (formerly
+            // i+1) may now be same-role as i-1, so decrement once to
+            // recheck from the correct position.
+            i--;
           } else if (isPlaceholder(prevContent)) {
             formatted.splice(i - 1, 1);
+            // Removed the earlier element — everything shifted down by 1.
+            // The element that was at i is now at i-1. Decrement so the
+            // next iteration compares the new i-1 with i-2 (or exits if
+            // i-1 is 0).
+            i--;
+          } else {
+            // Neither is a placeholder — merge content blocks into the
+            // earlier message and remove the later one. Skip the merge
+            // when either message carries tool_use or tool_result blocks;
+            // those require structural alternation for ensureToolPairing
+            // to inject the correct synthetic results downstream.
+            const hasToolBlock = (c: Anthropic.ContentBlockParam[]): boolean =>
+              c.some(
+                (b) =>
+                  typeof b !== "string" &&
+                  (b.type === "tool_use" || b.type === "tool_result"),
+              );
+            if (!hasToolBlock(prevContent) && !hasToolBlock(iContent)) {
+              formatted[i - 1] = {
+                ...formatted[i - 1],
+                content: [...prevContent, ...iContent],
+              };
+              formatted.splice(i, 1);
+              // Clamp i to the new last index — the splice may have put
+              // us past the end. If there's a new element at i (formerly
+              // i+1), it will be rechecked against the merged i-1.
+              if (i >= formatted.length) {
+                i = formatted.length - 1;
+              }
+            } else {
+              // Can't merge (tool blocks present) — leave for
+              // ensureToolPairing which handles tool_use/tool_result
+              // alternation in its own forward walk.
+              i--;
+            }
           }
         }
       }
@@ -816,18 +866,22 @@ export class AnthropicProvider implements Provider {
         output_config,
         cacheTtl: _cacheTtl,
         max_tokens: callerMaxTokens,
+        usageAttributionHeaders,
         ...restConfig
       } = (config ?? {}) as Record<string, unknown> & {
         // "xhigh" is an intermediate tier between "high" and "max" supported
         // by newer Anthropic models (e.g. Opus 4.7). The SDK's OutputConfig
         // type doesn't yet include it, so we widen to the internal effort
-        // union and cast when building mergedOutputConfig.
-        effort?: "low" | "medium" | "high" | "xhigh" | "max";
+        // union and cast when building mergedOutputConfig. "none" is a
+        // Vellum-wide value meaning "omit the effort param entirely" — we
+        // skip the output_config.effort field in that case.
+        effort?: "none" | "low" | "medium" | "high" | "xhigh" | "max";
         speed?: "standard" | "fast";
         output_config?: Record<string, unknown>;
+        usageAttributionHeaders?: Record<string, string>;
       };
-      // Haiku does not support the effort / output_config parameter,
-      // extended cache TTL betas, 1M context, or 64K output tokens.
+      // Haiku does not support the effort / output_config parameter or
+      // extended cache TTL betas.
       // Determine the effective model (per-call override or provider default)
       // and gate features accordingly.
       const effectiveModel =
@@ -836,7 +890,7 @@ export class AnthropicProvider implements Provider {
       const supportsEffort = !isHaiku;
       const mergedOutputConfig = {
         ...(output_config ?? {}),
-        ...(effort && supportsEffort
+        ...(effort && effort !== "none" && supportsEffort
           ? { effort: effort as Anthropic.OutputConfig["effort"] }
           : {}),
       };
@@ -939,24 +993,48 @@ export class AnthropicProvider implements Provider {
       // TTL is appropriate. Walk backwards to find the last user message
       // with a real text block (skipping tool_result-only messages and
       // synthetic continuation placeholders injected by ensureToolPairing).
-      let turnStartIdx = -1;
-      for (let i = sentMessages.length - 1; i >= 0; i--) {
-        const msg = sentMessages[i];
-        if (msg.role !== "user" || !Array.isArray(msg.content)) continue;
-        const hasText = msg.content.some(
-          (b) =>
-            typeof b !== "string" &&
-            b.type === "text" &&
-            b.text !== SYNTHETIC_CONTINUATION_TEXT,
-        );
-        if (!hasText) continue;
-        const lastBlock = msg.content[msg.content.length - 1];
+      const msgs = sentMessages;
+      const findUserTextMsgIdx = (startIdx: number): number => {
+        for (let i = startIdx; i >= 0; i--) {
+          const msg = msgs[i];
+          if (msg.role !== "user" || !Array.isArray(msg.content)) continue;
+          const hasText = msg.content.some(
+            (b) =>
+              typeof b !== "string" &&
+              b.type === "text" &&
+              b.text !== SYNTHETIC_CONTINUATION_TEXT,
+          );
+          if (hasText) return i;
+        }
+        return -1;
+      };
+      const applyCacheControlToLastBlock = (msgIdx: number): void => {
+        const content = msgs[msgIdx].content;
+        if (!Array.isArray(content) || content.length === 0) return;
+        const lastBlock = content[content.length - 1];
         if (typeof lastBlock !== "string") {
           (lastBlock as unknown as Record<string, unknown>).cache_control =
             cacheControl;
         }
-        turnStartIdx = i;
-        break;
+      };
+      const turnStartIdx = findUserTextMsgIdx(msgs.length - 1);
+      if (turnStartIdx >= 0) applyCacheControlToLastBlock(turnStartIdx);
+
+      // Previous-turn anchor: when this request is the first of a new turn
+      // (turn-start is the very last message — no tool-use loop yet), also
+      // place a 1h breakpoint on the *previous* turn-starting user message.
+      // Anthropic only matches the cache at cache_control points present in
+      // the current request, so without this anchor the breakpoint slides
+      // forward each new user turn and the prior cached prefix becomes
+      // unreachable — forcing a full re-creation of history (200K+
+      // cache_creation tokens per new turn). Skipped during tool-use loops
+      // where the current turn-start already covers the same prefix and a
+      // second anchor would blow the 4-breakpoint budget.
+      let prevTurnAnchorIdx = -1;
+      if (turnStartIdx === msgs.length - 1 && turnStartIdx > 0) {
+        prevTurnAnchorIdx = findUserTextMsgIdx(turnStartIdx - 1);
+        if (prevTurnAnchorIdx >= 0)
+          applyCacheControlToLastBlock(prevTurnAnchorIdx);
       }
 
       // Advancing tail: place a short-lived 5m cache breakpoint on the last
@@ -993,19 +1071,21 @@ export class AnthropicProvider implements Provider {
       }
 
       // Enforce Anthropic API maximum of 4 cache_control blocks.
-      // When the system prompt boundary splits into 2 cached blocks AND
-      // tools + turn-start + advancing-tail breakpoints are all present,
-      // we'd have 5.  Drop the static system block's breakpoint — it's
-      // small (<1K tokens) so the re-read cost is negligible, while the
-      // dynamic block (workspace context) rarely changes mid-session and
-      // benefits more from caching.
-      const hasTailBreakpoint = tailBreakpointApplied;
+      // With the system prompt boundary split into 2 cached blocks AND
+      // tools + turn-start + (tail OR prev-turn-anchor), we'd have 5.
+      // Drop the static system block's breakpoint — it's small (<1K
+      // tokens) so the re-read cost is negligible, while the dynamic
+      // block (workspace context) rarely changes mid-session and
+      // benefits more from caching. Tail and prev-turn-anchor are
+      // mutually exclusive (prev-turn-anchor only fires when turn-start
+      // is the last message, which is the exact condition that suppresses
+      // the tail), so we never exceed 5.
       const hasToolCacheBreakpoint =
         params.tools?.some(
           (t) => "cache_control" in t && t.cache_control != null,
         ) ?? false;
       if (
-        hasTailBreakpoint &&
+        (tailBreakpointApplied || prevTurnAnchorIdx >= 0) &&
         Array.isArray(params.system) &&
         params.system.length === 2 &&
         hasToolCacheBreakpoint
@@ -1039,15 +1119,13 @@ export class AnthropicProvider implements Provider {
         finalMessage(): Promise<Anthropic.Message>;
       }
 
-      // Fast mode: use the beta endpoint with speed: "fast" for Opus models (4.6, 4.7)
+      // Fast mode: use the beta endpoint with speed: "fast" for Opus models (4.6, 4.7).
       const useFastMode = speed === "fast" && effectiveModel.includes("opus");
 
-      // Collect required betas: extended cache TTL for 1h system prompt caching,
-      // 1M context window, and fast-mode when applicable.
-      // Haiku doesn't support the extended cache TTL or 1M context betas.
-      const betas: string[] = isHaiku
-        ? []
-        : ["extended-cache-ttl-2025-04-11", "context-1m-2025-08-07"];
+      // Collect request betas that are still explicit transport features.
+      // Current long-context Anthropic models expose their larger windows by
+      // model capability in the catalog/resolver, not by this generic header.
+      const betas: string[] = isHaiku ? [] : ["extended-cache-ttl-2025-04-11"];
       if (useFastMode) {
         betas.push("fast-mode-2026-02-01");
       }
@@ -1062,7 +1140,12 @@ export class AnthropicProvider implements Provider {
                 betas,
               } as Anthropic.Beta.Messages.MessageCreateParamsNonStreaming &
                 Anthropic.Beta.Messages.MessageCreateParamsStreaming,
-              { signal: timeoutSignal },
+              {
+                signal: timeoutSignal,
+                ...(usageAttributionHeaders
+                  ? { headers: usageAttributionHeaders }
+                  : {}),
+              },
             ) as unknown as UnifiedStream)
           : betas.length > 0
             ? (this.client.beta.messages.stream(
@@ -1071,10 +1154,18 @@ export class AnthropicProvider implements Provider {
                   betas,
                 } as Anthropic.Beta.Messages.MessageCreateParamsNonStreaming &
                   Anthropic.Beta.Messages.MessageCreateParamsStreaming,
-                { signal: timeoutSignal },
+                {
+                  signal: timeoutSignal,
+                  ...(usageAttributionHeaders
+                    ? { headers: usageAttributionHeaders }
+                    : {}),
+                },
               ) as unknown as UnifiedStream)
             : (this.client.messages.stream(params, {
                 signal: timeoutSignal,
+                ...(usageAttributionHeaders
+                  ? { headers: usageAttributionHeaders }
+                  : {}),
               }) as unknown as UnifiedStream);
 
         // Buffer streaming text until it's clear the accumulated text isn't

package/src/providers/call-site-routing.ts

@@ -19,6 +19,8 @@
  * stable identity.
  */
 
+import { AsyncLocalStorage } from "node:async_hooks";
+
 import { resolveCallSiteConfig } from "../config/llm-resolver.js";
 import { getConfig } from "../config/loader.js";
 import type {
@@ -30,14 +32,28 @@ import type {
 } from "./types.js";
 
 export class CallSiteRoutingProvider implements Provider {
-  public readonly name: string;
   public readonly tokenEstimationProvider?: string;
 
+  // Per-call async context that tracks which provider is currently executing.
+  // Using AsyncLocalStorage instead of a plain instance field means concurrent
+  // sendMessage calls (e.g. the main agent turn and a title-generation call
+  // both in-flight at the same time on the same provider instance) each see
+  // their own value — no clobbering, no premature clear.
+  //
+  // During sendMessage, emitLlmCallStartedIfNeeded reads provider.name on the
+  // first text_delta (before the response completes). The getter below returns
+  // the async-context value so streaming trace events carry the routed
+  // provider's name, not the default's.
+  private readonly _activeProviderContext = new AsyncLocalStorage<string>();
+
+  get name(): string {
+    return this._activeProviderContext.getStore() ?? this.defaultProvider.name;
+  }
+
   constructor(
     private readonly defaultProvider: Provider,
     private readonly getProviderByName: (name: string) => Provider | undefined,
   ) {
-    this.name = defaultProvider.name;
     this.tokenEstimationProvider = defaultProvider.tokenEstimationProvider;
   }
 
@@ -48,19 +64,46 @@ export class CallSiteRoutingProvider implements Provider {
     options?: SendMessageOptions,
   ): Promise<ProviderResponse> {
     const target = this.selectProvider(options);
-    return target.sendMessage(messages, tools, systemPrompt, options);
+    const isRouted = target !== this.defaultProvider;
+
+    const doSend = async (): Promise<ProviderResponse> => {
+      const response = await target.sendMessage(
+        messages,
+        tools,
+        systemPrompt,
+        options,
+      );
+      // Also stamp actualProvider on the response so that handleUsage /
+      // llm_call_finished (which read event.actualProvider, not provider.name)
+      // attribute the call to the right provider.
+      if (isRouted && response.actualProvider == null) {
+        return { ...response, actualProvider: target.name };
+      }
+      return response;
+    };
+
+    // Run inside the async context so that any code reading provider.name
+    // during streaming (e.g. emitLlmCallStartedIfNeeded on text_delta) sees
+    // the routed provider's name for this specific call, not the default.
+    return isRouted
+      ? this._activeProviderContext.run(target.name, doSend)
+      : doSend();
   }
 
   /**
    * Pick the provider to route this call through. The default provider wins
-   * unless the per-call `callSite` resolves to a different provider name and
-   * the registry can produce a Provider for it.
+   * unless the per-call `callSite` (layered with any `overrideProfile`)
+   * resolves to a different provider name and the registry can produce a
+   * Provider for it.
    */
   private selectProvider(options?: SendMessageOptions): Provider {
     const callSite = options?.config?.callSite;
     if (!callSite) return this.defaultProvider;
 
-    const resolved = resolveCallSiteConfig(callSite, getConfig().llm);
+    const overrideProfile = options?.config?.overrideProfile;
+    const resolved = resolveCallSiteConfig(callSite, getConfig().llm, {
+      overrideProfile,
+    });
     if (resolved.provider === this.defaultProvider.name) {
       return this.defaultProvider;
     }

package/src/providers/fireworks/client.ts

@@ -19,6 +19,9 @@ export class FireworksProvider extends OpenAIChatCompletionsProvider {
       providerName: "fireworks",
       providerLabel: "Fireworks",
       streamTimeoutMs: options.streamTimeoutMs,
+      // Fireworks' OpenAI-compatible chat-completions API documents only
+      // low|medium|high for reasoning_effort; sending "xhigh" 4xxs upstream.
+      maxReasoningEffort: "high",
     });
   }
 }