@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, test } from "bun:test";
 
-import type { SendMessageOptions } from "../../types.js";
+import OpenAI from "openai";
+
+import { ProviderError } from "../../../util/errors.js";
+import { ContextOverflowError, type SendMessageOptions } from "../../types.js";
 import { OpenAIChatCompletionsProvider } from "../chat-completions-provider.js";
 
 class ExtraBodyProvider extends OpenAIChatCompletionsProvider {
@@ -48,4 +51,94 @@ describe("chat-completions extraBody", () => {
     expect(seenParams?.directions).toEqual({ companion: 0.4 });
     expect(seenOptions?.extraBody).toBeUndefined();
   });
+
+  test("attaches the SDK create params as ProviderError.rawRequest on 4xx", async () => {
+    const provider = new ExtraBodyProvider("test-key", "qwen/qwen3-8b", {
+      providerName: "vellum",
+      providerLabel: "Vellum",
+    });
+    (
+      provider as unknown as {
+        client: {
+          chat: {
+            completions: {
+              create: () => Promise<AsyncIterable<unknown>>;
+            };
+          };
+        };
+      }
+    ).client.chat.completions.create = async () => {
+      throw new OpenAI.APIError(
+        400,
+        { detail: "directions are not loaded" },
+        undefined,
+        new Headers(),
+      );
+    };
+
+    try {
+      await provider.sendMessage([
+        { role: "user", content: [{ type: "text", text: "hi" }] },
+      ]);
+      throw new Error("expected ProviderError");
+    } catch (error) {
+      expect(error).toBeInstanceOf(ProviderError);
+      const rejected = error as ProviderError;
+      expect(rejected.provider).toBe("vellum");
+      expect(rejected.statusCode).toBe(400);
+      expect(rejected.rawRequest).toEqual(
+        expect.objectContaining({
+          model: "qwen/qwen3-8b",
+          directions: { companion: 0.4 },
+        }),
+      );
+    }
+  });
+
+  test("attaches the SDK create params as ContextOverflowError.rawRequest", async () => {
+    const provider = new ExtraBodyProvider("test-key", "qwen/qwen3-8b", {
+      providerName: "vellum",
+      providerLabel: "Vellum",
+    });
+    (
+      provider as unknown as {
+        client: {
+          chat: {
+            completions: {
+              create: () => Promise<AsyncIterable<unknown>>;
+            };
+          };
+        };
+      }
+    ).client.chat.completions.create = async () => {
+      throw new OpenAI.APIError(
+        400,
+        {
+          message:
+            "This model's maximum context length is 128000 tokens. However, your messages resulted in 150000 tokens.",
+          type: "invalid_request_error",
+          code: "context_length_exceeded",
+        },
+        undefined,
+        new Headers(),
+      );
+    };
+
+    try {
+      await provider.sendMessage([
+        { role: "user", content: [{ type: "text", text: "hi" }] },
+      ]);
+      throw new Error("expected ContextOverflowError");
+    } catch (error) {
+      expect(error).toBeInstanceOf(ContextOverflowError);
+      const rejected = error as ContextOverflowError;
+      expect(rejected.provider).toBe("vellum");
+      expect(rejected.rawRequest).toEqual(
+        expect.objectContaining({
+          model: "qwen/qwen3-8b",
+          directions: { companion: 0.4 },
+        }),
+      );
+    }
+  });
 });

package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts

@@ -1,12 +1,14 @@
-import { describe, expect, test } from "bun:test";
+import { beforeEach, describe, expect, test } from "bun:test";
 
 import OpenAI from "openai";
 
+import { ProviderError } from "../../../util/errors.js";
 import { isPlaceholderSentinelText } from "../../placeholder-sentinels.js";
 import {
   EMPTY_ASSISTANT_TURN_PLACEHOLDER,
   OpenAIChatCompletionsProvider,
   type OpenAIChatCompletionsProviderOptions,
+  resetReasoningOptOutRejectersForTests,
 } from "../chat-completions-provider.js";
 
 type ReasoningDetail = {
@@ -777,6 +779,65 @@ function rejection(message: string, status = 400): Error {
 }
 
 describe("reasoning opt-out rejection fallback", () => {
+  beforeEach(() => {
+    resetReasoningOptOutRejectersForTests();
+  });
+
+  test("remembers a rejecting model and skips the opt-out on later requests", async () => {
+    const { provider, requests } = stubProviderWithErrors(
+      [rejection("reasoning_effort 'none' is not supported for this model")],
+      OK_CHUNKS,
+    );
+    const send = () =>
+      provider.sendMessage(
+        [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+        { config: { effort: "none" } },
+      );
+
+    await send();
+    await send();
+
+    expect(requests).toHaveLength(3);
+    expect(
+      (requests[2] as { reasoning_effort?: string }).reasoning_effort,
+    ).toBeUndefined();
+  });
+
+  test("does not remember when success needed a further compat retry", async () => {
+    // The broad /reasoning/ match misreads the reasoning_content round-trip
+    // error as an opt-out rejection; the request only succeeds after the
+    // backfill retry, so the opt-out itself was never proven unsupported.
+    const { provider, requests } = stubProviderWithErrors(
+      [
+        rejection("reasoning_content must be passed back to the API"),
+        rejection("reasoning_content must be passed back to the API"),
+      ],
+      OK_CHUNKS,
+    );
+    const messages = [
+      {
+        role: "user" as const,
+        content: [{ type: "text" as const, text: "hi" }],
+      },
+      {
+        role: "assistant" as const,
+        content: [{ type: "text" as const, text: "hello" }],
+      },
+      {
+        role: "user" as const,
+        content: [{ type: "text" as const, text: "again" }],
+      },
+    ];
+
+    await provider.sendMessage(messages, { config: { effort: "none" } });
+    await provider.sendMessage(messages, { config: { effort: "none" } });
+
+    expect(requests).toHaveLength(4);
+    expect(
+      (requests[3] as { reasoning_effort?: string }).reasoning_effort,
+    ).toBe("none");
+  });
+
   test("retries once without reasoning params when a model rejects the explicit opt-out", async () => {
     const { provider, requests } = stubProviderWithErrors(
       [rejection("reasoning_effort 'none' is not supported for this model")],
@@ -928,12 +989,56 @@ describe("thinking-mode tool_choice rejection fallback", () => {
     expect(first.reasoning_effort).toBe("high");
     expect(second.tool_choice).toBeUndefined();
     expect(second.reasoning_effort).toBe("high");
+    expect(
+      (response.rawRequest as { tool_choice?: unknown }).tool_choice,
+    ).toBeUndefined();
     const text = response.content.find((b) => b.type === "text") as
       | { type: "text"; text: string }
       | undefined;
     expect(text?.text).toBe("ok");
   });
 
+  test("ProviderError.rawRequest is the post-retry params when the retried call fails", async () => {
+    const { provider } = stubProviderWithErrors(
+      [
+        rejection("Thinking mode does not support this tool_choice"),
+        Object.assign(
+          new Error("Thinking mode does not support this tool_choice"),
+          { status: 400 },
+        ),
+      ],
+      OK_CHUNKS,
+    );
+
+    try {
+      await provider.sendMessage(
+        [{ role: "user", content: [{ type: "text", text: "hi" }] }],
+        {
+          tools: [
+            {
+              name: "bash",
+              description: "Run a shell command",
+              input_schema: { type: "object", properties: {} },
+            },
+          ],
+          config: { tool_choice: { type: "none" }, effort: "high" },
+        },
+      );
+      throw new Error("expected ProviderError");
+    } catch (error) {
+      expect(error).toBeInstanceOf(ProviderError);
+      const rejected = error as ProviderError;
+      expect(
+        (rejected.rawRequest as { tool_choice?: unknown } | undefined)
+          ?.tool_choice,
+      ).toBeUndefined();
+      expect(
+        (rejected.rawRequest as { reasoning_effort?: string } | undefined)
+          ?.reasoning_effort,
+      ).toBe("high");
+    }
+  });
+
   test("retries once when Kimi rejects a specified tool_choice in thinking mode", async () => {
     const { provider, requests } = stubProviderWithErrors(
       [

package/src/providers/openai/__tests__/chat-template-flatten.test.ts

@@ -57,6 +57,10 @@ function stubProviderWithErrors(errors: unknown[]): {
 const CHAT_TEMPLATE_400 =
   "Failed to apply chat template: invalid operation: object is not callable (in chat:22)";
 
+// Fireworks' renderer error for DeepSeek, verbatim.
+const FIREWORKS_STR_JOIN_400 =
+  "sequence item 0: expected str instance, list found";
+
 function rejection(message: string, status = 400): Error {
   return Object.assign(new Error(message), { status });
 }
@@ -103,6 +107,26 @@ describe("chat-template rejection flatten fallback", () => {
     expect(text?.text).toBe("ok");
   });
 
+  test("retries with flattened content on Fireworks' str-join template error", async () => {
+    const { provider, requests } = stubProviderWithErrors([
+      rejection(FIREWORKS_STR_JOIN_400),
+    ]);
+
+    await provider.sendMessage([
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "[Image: a cat]" },
+          { type: "text", text: "what is this?" },
+        ],
+      },
+    ]);
+
+    expect(requests).toHaveLength(2);
+    const retry = requests[1] as { messages: Array<{ content: unknown }> };
+    expect(retry.messages[0].content).toBe("[Image: a cat]\n\nwhat is this?");
+  });
+
   test("does not retry when a content-parts array carries media (never silently drop an image)", async () => {
     // GIVEN an endpoint whose chat template 400s on structured message content
     const { provider, requests } = stubProviderWithErrors([

package/src/providers/openai/__tests__/google-thought-signature.test.ts

@@ -7,7 +7,6 @@ import {
   attachGoogleThoughtSignature,
   attachGoogleThoughtSignatureIfNeeded,
   backfillGoogleThoughtSignatures,
-  backfillUnsignedGoogleThoughtSignatures,
   geminiThoughtSignaturesByToolCallId,
   googleThoughtSignatureFromUnknown,
   type GoogleToolCallExtraContent,
@@ -177,7 +176,7 @@ describe("thought signature params helpers", () => {
 
   test("backfills the dummy signature onto unsigned tool_calls", () => {
     const params = structuredClone(unsignedParams);
-    expect(backfillUnsignedGoogleThoughtSignatures(params)).toBe(true);
+    expect(backfillGoogleThoughtSignatures(params)).toBe(true);
     expect(messagesCarryGoogleThoughtSignature(params)).toBe(true);
     expect(
       params.messages[0].tool_calls[0].extra_content?.google?.thought_signature,

package/src/providers/openai/chat-completions-provider.ts

@@ -355,6 +355,41 @@ function protectJsonSchemaToolResult(payload: string): string {
     : payload;
 }
 
+function carriesReasoningOptOut(params: unknown): boolean {
+  const p = params as {
+    reasoning_effort?: unknown;
+    reasoning?: { effort?: unknown } | null;
+  };
+  return (
+    p.reasoning_effort === "none" ||
+    (typeof p.reasoning === "object" &&
+      p.reasoning !== null &&
+      p.reasoning.effort === "none")
+  );
+}
+
+function stripReasoningParams(params: unknown): void {
+  const p = params as Record<string, unknown>;
+  delete p.reasoning_effort;
+  delete p.reasoning;
+}
+
+/**
+ * `baseURL|model|routing` keys whose request succeeded on the attempt directly
+ * after the opt-out was stripped. Later requests skip the opt-out up front
+ * instead of paying a rejected round-trip on every call. A success that needed
+ * a further compat retry proves nothing about the opt-out, so it is not
+ * recorded. Routing (OpenRouter's `provider` body field) is part of the key
+ * because opt-out support belongs to the upstream backend, not the model slug.
+ * Process-lifetime only: a restart re-learns with one rejected request per key.
+ */
+const reasoningOptOutRejecters = new Set<string>();
+
+/** Test-only: forget learned reasoning opt-out rejections. */
+export function resetReasoningOptOutRejectersForTests(): void {
+  reasoningOptOutRejecters.clear();
+}
+
 /**
  * True when the request carried an explicit reasoning opt-out (`"none"` sent
  * as flat `reasoning_effort` or nested `reasoning.effort`) and the provider
@@ -364,16 +399,7 @@ function protectJsonSchemaToolResult(payload: string): string {
  * model-default reasoning beats a hard failure.
  */
 function isReasoningOptOutRejection(error: unknown, params: unknown): boolean {
-  const p = params as {
-    reasoning_effort?: unknown;
-    reasoning?: { effort?: unknown } | null;
-  };
-  const optedOut =
-    p.reasoning_effort === "none" ||
-    (typeof p.reasoning === "object" &&
-      p.reasoning !== null &&
-      p.reasoning.effort === "none");
-  if (!optedOut) {
+  if (!carriesReasoningOptOut(params)) {
     return false;
   }
   if (!isClientErrorStatus(error)) {
@@ -685,10 +711,7 @@ function classifyOpenAICompatRetry(
       kind: "reasoning-opt-out",
       message:
         "Model rejected the explicit reasoning opt-out; retrying without reasoning params",
-      apply: () => {
-        delete params.reasoning_effort;
-        delete (params as unknown as Record<string, unknown>).reasoning;
-      },
+      apply: () => stripReasoningParams(params),
     };
   }
   if (isThinkingModeToolChoiceRejection(error, params)) {
@@ -788,6 +811,24 @@ const OPENAI_SUPPORTED_IMAGE_TYPES = new Set([
   "image/webp",
 ]);
 
+/**
+ * Persistable view of the chat.completions create params. A `logit_bias`
+ * preset (e.g. `suppress-cjk`) is ~5.3k deterministic entries (~68KB);
+ * summarize it so inspector rows don't balloon. The full map still went
+ * out on the wire.
+ */
+function inspectableChatCompletionsRequest(
+  params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming,
+): unknown {
+  if (!params.logit_bias) {
+    return params;
+  }
+  return {
+    ...params,
+    logit_bias: `<${Object.keys(params.logit_bias).length} token biases omitted>`,
+  };
+}
+
 // Think-tag scanning primitives are shared with the TTS reasoning filter
 // (util/think-tag-stream.ts) so the two stream parsers cannot drift. This
 // provider keeps its exact historical behavior: case-sensitive, <think> only.
@@ -883,6 +924,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
     // wire, to be decoded back on the response. Empty unless
     // `coerceObjectArgsToJsonString` is enabled.
     const coercedObjectKeys = new Map<string, string[]>();
+    let inspectableRequest: unknown | undefined;
 
     try {
       const thoughtSignaturesByCallId =
@@ -1125,18 +1167,35 @@ export class OpenAIChatCompletionsProvider implements Provider {
         if (extraBody) {
           Object.assign(params, extraBody);
         }
-        const createStream = () =>
-          this.client.chat.completions.create(params, {
+        const optOutKey = `${this.client.baseURL}|${params.model}|${JSON.stringify(
+          (params as { provider?: unknown }).provider ?? null,
+        )}`;
+        if (
+          reasoningOptOutRejecters.has(optOutKey) &&
+          carriesReasoningOptOut(params)
+        ) {
+          stripReasoningParams(params);
+        }
+        const createStream = () => {
+          // Snapshot after extra-body merge and any in-place compat retries
+          // so inspector rows match the params that actually went on the wire.
+          inspectableRequest = inspectableChatCompletionsRequest(params);
+          return this.client.chat.completions.create(params, {
             signal: timeoutSignal,
             ...(Object.keys(requestHeaders).length > 0
               ? { headers: requestHeaders }
               : {}),
           });
+        };
         const attemptedCompatRetries = new Set<OpenAICompatRetryKind>();
+        let lastCompatRetry: OpenAICompatRetryKind | undefined;
         let stream: Awaited<ReturnType<typeof createStream>>;
         for (;;) {
           try {
             stream = await createStream();
+            if (lastCompatRetry === "reasoning-opt-out") {
+              reasoningOptOutRejecters.add(optOutKey);
+            }
             break;
           } catch (error) {
             const retry = classifyOpenAICompatRetry(
@@ -1148,6 +1207,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
               throw error;
             }
             attemptedCompatRetries.add(retry.kind);
+            lastCompatRetry = retry.kind;
             log.warn(
               {
                 provider: this.name,
@@ -1447,15 +1507,9 @@ export class OpenAIChatCompletionsProvider implements Provider {
         },
         stopReason: finishReason,
         // `rawRequest` is persisted to the request-log DB and inspector on every
-        // call. A `logit_bias` preset (e.g. `suppress-cjk`) is ~5.3k deterministic
-        // entries (~68KB); summarize it here so logs don't balloon. The full map
-        // still went out on the wire above.
-        rawRequest: params.logit_bias
-          ? {
-              ...params,
-              logit_bias: `<${Object.keys(params.logit_bias).length} token biases omitted>`,
-            }
-          : params,
+        // call, including 4xx throws below. A `logit_bias` preset is summarized
+        // so logs don't balloon. The full map still went out on the wire above.
+        rawRequest: inspectableRequest,
         rawResponse,
       };
     } catch (error) {
@@ -1482,6 +1536,9 @@ export class OpenAIChatCompletionsProvider implements Provider {
             maxTokens: overflow.maxTokens,
             statusCode: error.status,
             cause: error,
+            ...(inspectableRequest !== undefined
+              ? { rawRequest: inspectableRequest }
+              : {}),
           });
         }
         if (detectVisionNotSupported(error, normalized.message)) {
@@ -1492,9 +1549,13 @@ export class OpenAIChatCompletionsProvider implements Provider {
             error.status,
             // Stamp the reason so classification is status-independent (a vision
             // rejection returned as 401/403 must not read as an invalid key).
-            abortReason
-              ? { abortReason, reason: "vision_unsupported" }
-              : { reason: "vision_unsupported" },
+            {
+              reason: "vision_unsupported" as const,
+              ...(abortReason ? { abortReason } : {}),
+              ...(inspectableRequest !== undefined
+                ? { rawRequest: inspectableRequest }
+                : {}),
+            },
           );
         }
         const retryAfterMs = extractRetryAfterMs(error.headers);
@@ -1509,6 +1570,7 @@ export class OpenAIChatCompletionsProvider implements Provider {
           apiErrorParam?: string;
           requestId?: string;
           rawBody?: string;
+          rawRequest?: unknown;
           reason?: ProviderErrorReason;
         } = { cause: error };
         if (retryAfterMs !== undefined) {
@@ -1535,6 +1597,9 @@ export class OpenAIChatCompletionsProvider implements Provider {
         if (normalized.reason) {
           errorOptions.reason = normalized.reason;
         }
+        if (inspectableRequest !== undefined) {
+          errorOptions.rawRequest = inspectableRequest;
+        }
         throw new ProviderError(
           formattedMessage,
           this.name,
@@ -1548,7 +1613,13 @@ export class OpenAIChatCompletionsProvider implements Provider {
         }`,
         this.name,
         undefined,
-        abortReason ? { cause: error, abortReason } : { cause: error },
+        {
+          cause: error,
+          ...(abortReason ? { abortReason } : {}),
+          ...(inspectableRequest !== undefined
+            ? { rawRequest: inspectableRequest }
+            : {}),
+        },
       );
     }
   }

package/src/providers/openai/google-thought-signature.ts

@@ -266,12 +266,6 @@ export function backfillGoogleThoughtSignatures(
   return added;
 }
 
-export function backfillUnsignedGoogleThoughtSignatures(
-  params: unknown,
-): boolean {
-  return backfillGoogleThoughtSignatures(params);
-}
-
 const UNKNOWN_EXTRA_CONTENT_REJECTION =
   /unknown|unexpected|unrecognized|additional propert|extra (?:field|property)|not (?:a )?valid|invalid (?:argument|parameter|field|property)/i;
 

package/src/providers/openai/responses-provider.ts

@@ -294,6 +294,7 @@ export class OpenAIResponsesProvider implements Provider {
       configObj.promptCacheKey.length > 0
         ? (configObj.promptCacheKey as string)
         : undefined;
+    let inspectableRequest: unknown | undefined;
 
     try {
       const effectiveModel = modelOverride ?? this.model;
@@ -428,6 +429,7 @@ export class OpenAIResponsesProvider implements Provider {
       }
 
       Object.assign(params, this.buildExtraCreateParams(options));
+      inspectableRequest = params;
 
       const { signal: timeoutSignal, cleanup: cleanupTimeout } =
         createStreamTimeout(this.streamTimeoutMs, signal);
@@ -736,6 +738,9 @@ export class OpenAIResponsesProvider implements Provider {
             maxTokens: overflow.maxTokens,
             statusCode: error.status,
             cause: error,
+            ...(inspectableRequest !== undefined
+              ? { rawRequest: inspectableRequest }
+              : {}),
           });
         }
         const retryAfterMs = extractRetryAfterMs(error.headers);
@@ -750,6 +755,7 @@ export class OpenAIResponsesProvider implements Provider {
           apiErrorParam?: string;
           requestId?: string;
           rawBody?: string;
+          rawRequest?: unknown;
           reason?: ProviderErrorReason;
         } = { cause: error };
         if (retryAfterMs !== undefined) {
@@ -776,6 +782,9 @@ export class OpenAIResponsesProvider implements Provider {
         if (normalized.reason) {
           errorOptions.reason = normalized.reason;
         }
+        if (inspectableRequest !== undefined) {
+          errorOptions.rawRequest = inspectableRequest;
+        }
         throw new ProviderError(
           formattedMessage,
           this.name,
@@ -789,7 +798,20 @@ export class OpenAIResponsesProvider implements Provider {
         }`,
         this.name,
         undefined,
-        abortReason ? { cause: error, abortReason } : { cause: error },
+        abortReason
+          ? {
+              cause: error,
+              abortReason,
+              ...(inspectableRequest !== undefined
+                ? { rawRequest: inspectableRequest }
+                : {}),
+            }
+          : {
+              cause: error,
+              ...(inspectableRequest !== undefined
+                ? { rawRequest: inspectableRequest }
+                : {}),
+            },
       );
     }
   }

package/src/providers/ratelimit.ts

@@ -62,9 +62,7 @@ export class RateLimitProvider implements Provider {
     // calls from bypassing the rate limit during the async gap.
     this.recordRequest();
 
-    const response = await this.inner.sendMessage(messages, options);
-
-    return response;
+    return this.inner.sendMessage(messages, options);
   }
 
   private enforceRequestRate(): void {

package/src/providers/search-provider-catalog.ts

@@ -163,6 +163,30 @@ export const SEARCH_PROVIDER_CATALOG: readonly SearchProviderCatalogEntry[] = [
     supportsApiBase: true,
     defaultApiBase: "https://api.fastcrw.com",
   },
+  {
+    id: "searxng",
+    displayName: "SearXNG",
+    kind: "byok",
+    keyless: true,
+    apiKeyPrefix: "token (optional)",
+    envVar: "SEARXNG_API_KEY",
+    secretKey: "searxng",
+    fallbackOrder: 7,
+    privacyPolicyUrl: "https://docs.searxng.org/own-instance.html",
+    supportsApiBase: true,
+  },
+  {
+    id: "tinyfish",
+    displayName: "TinyFish",
+    kind: "byok",
+    apiKeyPrefix: "TinyFish API key...",
+    envVar: "TINYFISH_API_KEY",
+    secretKey: "tinyfish",
+    fallbackOrder: 8,
+    privacyPolicyUrl: "https://www.tinyfish.ai/privacy-policy",
+    supportsApiBase: true,
+    defaultApiBase: "https://api.search.tinyfish.ai",
+  },
 ];
 
 /** Provider ids accepted by the web-search config schema. */