@vellumai/assistant 0.11.2-staging.1 → 0.11.2-staging.3

package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts

@@ -0,0 +1,478 @@
+/**
+ * Serialization guard for orphaned tool results on the OpenAI providers.
+ *
+ * Both transports validate pairing server-side: the Responses API rejects a
+ * `function_call_output` whose `call_id` was not emitted as a `function_call`
+ * earlier in the request, and the Chat Completions API rejects a tool-role
+ * message whose `tool_call_id` is not in a preceding assistant message's
+ * `tool_calls`. A `tool_result` with no backward match in the request (e.g.
+ * its `tool_use` truncated away) is therefore degraded into plain user text
+ * with an `[orphaned tool result]` prefix, preserving the information while
+ * keeping the request valid. Paired results serialize unchanged.
+ */
+import { describe, expect, test } from "bun:test";
+
+import type { Message } from "../../types.js";
+import { OpenAIChatCompletionsProvider } from "../chat-completions-provider.js";
+import { OpenAIResponsesProvider } from "../responses-provider.js";
+
+// ---------------------------------------------------------------------------
+// Responses transport harness
+// ---------------------------------------------------------------------------
+
+type ResponsesStreamEvent = { type: string; [key: string]: unknown };
+
+const RESPONSES_OK_EVENTS: ResponsesStreamEvent[] = [
+  { type: "response.output_text.delta", delta: "ok" },
+  {
+    type: "response.completed",
+    response: {
+      model: "gpt-5.2",
+      status: "completed",
+      output: [],
+      usage: { input_tokens: 10, output_tokens: 2 },
+    },
+  },
+];
+
+function makeEventStream(
+  events: ResponsesStreamEvent[],
+): AsyncIterable<ResponsesStreamEvent> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const event of events) {
+        yield event;
+      }
+    },
+  };
+}
+
+/** Swap `responses.create` for a canned stream that records request params. */
+function stubResponsesCreate(provider: OpenAIResponsesProvider): {
+  input: () => unknown[];
+} {
+  let captured: unknown;
+  const inner = provider as unknown as {
+    client: {
+      responses: {
+        create: (
+          params: unknown,
+        ) => Promise<AsyncIterable<ResponsesStreamEvent>>;
+      };
+    };
+  };
+  inner.client.responses.create = async (params) => {
+    captured = params;
+    return makeEventStream(RESPONSES_OK_EVENTS);
+  };
+  return {
+    input: () => (captured as { input: unknown[] }).input,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Chat Completions transport harness
+// ---------------------------------------------------------------------------
+
+type ChatChunk = {
+  choices: Array<{
+    delta: { content?: string | null };
+    finish_reason?: string | null;
+  }>;
+  usage?: { prompt_tokens: number; completion_tokens: number };
+};
+
+const CHAT_OK_CHUNKS: ChatChunk[] = [
+  {
+    choices: [{ delta: { content: "ok" }, finish_reason: "stop" }],
+    usage: { prompt_tokens: 10, completion_tokens: 2 },
+  },
+];
+
+function makeChunkStream(chunks: ChatChunk[]): AsyncIterable<ChatChunk> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const chunk of chunks) {
+        yield chunk;
+      }
+    },
+  };
+}
+
+type ChatMessageParam = {
+  role: string;
+  content: string | Array<Record<string, unknown>>;
+  tool_call_id?: string;
+};
+
+/** Swap chat `create` for a canned stream that records request params. */
+function stubChatCreate(provider: OpenAIChatCompletionsProvider): {
+  messages: () => ChatMessageParam[];
+} {
+  let captured: unknown;
+  const inner = provider as unknown as {
+    client: {
+      chat: {
+        completions: {
+          create: (params: unknown) => Promise<AsyncIterable<ChatChunk>>;
+        };
+      };
+    };
+  };
+  inner.client.chat.completions.create = async (params) => {
+    captured = params;
+    return makeChunkStream(CHAT_OK_CHUNKS);
+  };
+  return {
+    messages: () => (captured as { messages: ChatMessageParam[] }).messages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+/**
+ * Persisted tool id shapes: `call_` is the Responses-native pass-through
+ * (conversations that always ran on this transport), `toolu_` is an
+ * Anthropic-shaped history routed to an OpenAI call site (cross-provider
+ * routing, e.g. a compaction call over imported history). The guard must be
+ * id-format-agnostic, so every case runs across both.
+ */
+const ID_SHAPES = [
+  { label: "call_ ids", pairedId: "call_paired", orphanId: "call_orphan" },
+  { label: "toolu_ ids", pairedId: "toolu_paired", orphanId: "toolu_orphan" },
+] as const;
+
+/** History whose tool_result is paired with a preceding tool_use. */
+function pairedHistory(id: string): Message[] {
+  return [
+    { role: "user", content: [{ type: "text", text: "Read the file" }] },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id,
+          name: "file_read",
+          input: { path: "/tmp/a" },
+        },
+      ],
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: id,
+          content: "file contents",
+        },
+      ],
+    },
+  ];
+}
+
+/**
+ * History whose leading tool_result has no matching tool_use anywhere in
+ * the request (the shape a pairing-blind front truncation produces).
+ */
+function orphanHistory(id: string): Message[] {
+  return [
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: id,
+          content: "stranded output",
+        },
+        { type: "text", text: "continue from here" },
+      ],
+    },
+    { role: "assistant", content: [{ type: "text", text: "continuing" }] },
+    { role: "user", content: [{ type: "text", text: "thanks" }] },
+  ];
+}
+
+/**
+ * A tool_result that arrives BEFORE its tool_use in the request order. The
+ * API contract only accepts backward matches, so this is orphaned too.
+ */
+function forwardReferenceHistory(): Message[] {
+  return [
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "call_later",
+          content: "premature output",
+        },
+        { type: "text", text: "odd ordering" },
+      ],
+    },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id: "call_later",
+          name: "file_read",
+          input: { path: "/tmp/b" },
+        },
+      ],
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "call_later",
+          content: "real output",
+        },
+      ],
+    },
+  ];
+}
+
+// ---------------------------------------------------------------------------
+// Responses transport
+// ---------------------------------------------------------------------------
+
+describe("OpenAIResponsesProvider orphan tool_result guard", () => {
+  for (const shape of ID_SHAPES) {
+    test(`emits function_call_output for a paired tool_result (${shape.label})`, async () => {
+      const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+      const stub = stubResponsesCreate(provider);
+
+      await provider.sendMessage(pairedHistory(shape.pairedId));
+
+      const input = stub.input() as Array<Record<string, unknown>>;
+      expect(input).toHaveLength(3);
+      expect(input[1]).toEqual({
+        type: "function_call",
+        call_id: shape.pairedId,
+        name: "file_read",
+        arguments: '{"path":"/tmp/a"}',
+      });
+      expect(input[2]).toEqual({
+        type: "function_call_output",
+        call_id: shape.pairedId,
+        output: "file contents",
+      });
+    });
+
+    test(`degrades an orphaned tool_result to user text instead of function_call_output (${shape.label})`, async () => {
+      const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+      const stub = stubResponsesCreate(provider);
+
+      await provider.sendMessage(orphanHistory(shape.orphanId));
+
+      const input = stub.input() as Array<Record<string, unknown>>;
+      // No function_call_output items at all: the only tool_result was
+      // orphaned.
+      expect(input.some((item) => item.type === "function_call_output")).toBe(
+        false,
+      );
+
+      // The orphan's content is preserved as prefixed text inside the user
+      // message, alongside the message's own text.
+      const firstUser = input[0] as {
+        type: string;
+        role: string;
+        content: Array<{ type: string; text?: string }>;
+      };
+      expect(firstUser.type).toBe("message");
+      expect(firstUser.role).toBe("user");
+      const texts = firstUser.content.map((part) => part.text ?? "");
+      expect(texts).toContain("continue from here");
+      expect(texts).toContain("[orphaned tool result] stranded output");
+    });
+  }
+
+  test("treats a forward-referencing tool_result as orphaned, keeps the backward match", async () => {
+    const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+    const stub = stubResponsesCreate(provider);
+
+    await provider.sendMessage(forwardReferenceHistory());
+
+    const input = stub.input() as Array<Record<string, unknown>>;
+    const outputs = input.filter(
+      (item) => item.type === "function_call_output",
+    );
+    // Only the tool_result AFTER the function_call serializes as an output.
+    expect(outputs).toEqual([
+      {
+        type: "function_call_output",
+        call_id: "call_later",
+        output: "real output",
+      },
+    ]);
+    // The premature result is degraded into the first user message.
+    const firstUser = input[0] as {
+      content: Array<{ type: string; text?: string }>;
+    };
+    const texts = firstUser.content.map((part) => part.text ?? "");
+    expect(texts).toContain("[orphaned tool result] premature output");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Chat Completions transport
+// ---------------------------------------------------------------------------
+
+describe("OpenAIChatCompletionsProvider orphan tool_result guard", () => {
+  function makeProvider(): OpenAIChatCompletionsProvider {
+    return new OpenAIChatCompletionsProvider("sk-test", "test-model", {
+      providerName: "openai",
+      providerLabel: "OpenAI",
+    });
+  }
+
+  for (const shape of ID_SHAPES) {
+    test(`emits a tool message for a paired tool_result (${shape.label})`, async () => {
+      const provider = makeProvider();
+      const stub = stubChatCreate(provider);
+
+      await provider.sendMessage(pairedHistory(shape.pairedId));
+
+      const sent = stub.messages();
+      const toolMessages = sent.filter((message) => message.role === "tool");
+      expect(toolMessages).toEqual([
+        {
+          role: "tool",
+          tool_call_id: shape.pairedId,
+          content: "file contents",
+        },
+      ]);
+    });
+
+    test(`degrades an orphaned tool_result to user text instead of a tool message (${shape.label})`, async () => {
+      const provider = makeProvider();
+      const stub = stubChatCreate(provider);
+
+      await provider.sendMessage(orphanHistory(shape.orphanId));
+
+      const sent = stub.messages();
+      expect(sent.some((message) => message.role === "tool")).toBe(false);
+
+      const firstUser = sent.find((message) => message.role === "user");
+      expect(firstUser).toBeDefined();
+      const content = firstUser!.content;
+      const texts = Array.isArray(content)
+        ? content.map((part) => (part.text as string) ?? "")
+        : [content];
+      expect(texts).toContain("continue from here");
+      expect(texts).toContain("[orphaned tool result] stranded output");
+    });
+  }
+
+  test("treats a forward-referencing tool_result as orphaned, keeps the backward match", async () => {
+    const provider = makeProvider();
+    const stub = stubChatCreate(provider);
+
+    await provider.sendMessage(forwardReferenceHistory());
+
+    const sent = stub.messages();
+    const toolMessages = sent.filter((message) => message.role === "tool");
+    expect(toolMessages).toEqual([
+      {
+        role: "tool",
+        tool_call_id: "call_later",
+        content: "real output",
+      },
+    ]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Cross-transport agreement
+// ---------------------------------------------------------------------------
+
+describe("orphan degradation agrees across both OpenAI transports", () => {
+  /** Every degraded orphan text a request carried, in request order. */
+  async function responsesOrphanTexts(history: Message[]): Promise<string[]> {
+    const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+    const stub = stubResponsesCreate(provider);
+    await provider.sendMessage(history);
+    const input = stub.input() as Array<Record<string, unknown>>;
+    return input
+      .flatMap((item) =>
+        Array.isArray(item.content) ? (item.content as unknown[]) : [],
+      )
+      .map((part) => (part as { text?: string }).text ?? "")
+      .filter((text) => text.startsWith("[orphaned"));
+  }
+
+  async function chatOrphanTexts(history: Message[]): Promise<string[]> {
+    const provider = new OpenAIChatCompletionsProvider(
+      "sk-test",
+      "test-model",
+      {
+        providerName: "openai",
+        providerLabel: "OpenAI",
+      },
+    );
+    const stub = stubChatCreate(provider);
+    await provider.sendMessage(history);
+    return (
+      stub
+        .messages()
+        // A user message carrying a single text part serializes as a plain
+        // string on this transport rather than a one-element array.
+        .flatMap((message) =>
+          Array.isArray(message.content)
+            ? (message.content as unknown[]).map(
+                (part) => (part as { text?: string }).text ?? "",
+              )
+            : [String(message.content ?? "")],
+        )
+        .filter((text) => text.startsWith("[orphaned"))
+    );
+  }
+
+  // The detection rule and the degraded wording live in one shared helper
+  // (`serializeToolResult`), so the two transports cannot drift into
+  // different markers for the same input. Byte equality is the assertion:
+  // re-inlining the rule in one converter and editing it there fails here,
+  // which is the regression this pins.
+  test("both transports degrade the same orphan to byte-identical text", async () => {
+    const history = orphanHistory("call_orphan");
+
+    const [fromResponses, fromChat] = await Promise.all([
+      responsesOrphanTexts(history),
+      chatOrphanTexts(history),
+    ]);
+
+    expect(fromResponses).toEqual(["[orphaned tool result] stranded output"]);
+    expect(fromChat).toEqual(fromResponses);
+  });
+
+  test("both transports carry an executor failure into the same degraded text", async () => {
+    // `is_error` prefixing is part of the shared payload rule, so it must
+    // survive degradation identically on both transports.
+    const history: Message[] = [
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "call_orphan",
+            content: "boom",
+            is_error: true,
+          },
+        ],
+      },
+      { role: "assistant", content: [{ type: "text", text: "continuing" }] },
+      { role: "user", content: [{ type: "text", text: "thanks" }] },
+    ];
+
+    const [fromResponses, fromChat] = await Promise.all([
+      responsesOrphanTexts(history),
+      chatOrphanTexts(history),
+    ]);
+
+    expect(fromResponses).toEqual(["[orphaned tool result] [ERROR] boom"]);
+    expect(fromChat).toEqual(fromResponses);
+  });
+});

package/src/providers/openai/chat-completions-provider.ts

@@ -46,6 +46,7 @@ import {
   OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES,
   openAIInputAudioFormat,
 } from "./input-audio.js";
+import { serializeToolResult } from "./orphaned-tool-result.js";
 
 /**
  * Detect OpenAI-compatible context-overflow signals on an `OpenAI.APIError`.
@@ -956,9 +957,19 @@ export class OpenAIChatCompletionsProvider implements Provider {
       });
     }
 
+    // Tool-call ids emitted in assistant messages earlier in this request.
+    // The API rejects a tool-role message whose `tool_call_id` has no
+    // preceding assistant `tool_calls` entry, so tool results are only
+    // serialized as tool messages when their call was emitted first
+    // (backward matches only).
+    const emittedToolCallIds = new Set<string>();
     for (const msg of messages) {
       if (msg.role === "assistant") {
-        result.push(this.toOpenAIAssistantMessage(msg));
+        const assistantMessage = this.toOpenAIAssistantMessage(msg);
+        for (const toolCall of assistantMessage.tool_calls ?? []) {
+          emittedToolCallIds.add(toolCall.id);
+        }
+        result.push(assistantMessage);
       } else {
         // User messages may contain tool_result blocks mixed with text/image
         const toolResults = msg.content.filter(
@@ -975,46 +986,51 @@ export class OpenAIChatCompletionsProvider implements Provider {
         // Emit tool results as separate tool-role messages
         // OpenAI's API only supports string content in tool messages, so media
         // from contentBlocks is collected and injected as a user message below.
+        // A tool_result whose id has no preceding assistant tool call in this
+        // request is orphaned; its content is degraded into the user message
+        // instead of being sent as a rejectable tool message.
         const toolResultMedia: ContentBlock[] = [];
+        const orphanedResultBlocks: ContentBlock[] = [];
         for (const tr of toolResults) {
-          let textContent = tr.content;
-          if (tr.contentBlocks && tr.contentBlocks.length > 0) {
-            const extraText = tr.contentBlocks
-              .filter(
-                (cb): cb is Extract<ContentBlock, { type: "text" }> =>
-                  cb.type === "text",
+          // Media this transport can carry: images always, plus inline audio
+          // when the model accepts it. The text payload and the orphan
+          // decision are the shared cross-transport rule.
+          for (const cb of tr.contentBlocks ?? []) {
+            if (cb.type === "image") {
+              toolResultMedia.push(cb);
+            } else if (
+              audioInputEnabled &&
+              cb.type === "file" &&
+              isOpenAICompatInlineAudio(
+                cb.source.media_type,
+                mediaSourceByteLength(cb.source),
               )
-              .map((cb) => cb.text);
-            if (extraText.length > 0) {
-              textContent = textContent + "\n" + extraText.join("\n");
-            }
-            for (const cb of tr.contentBlocks) {
-              if (cb.type === "image") {
-                toolResultMedia.push(cb);
-              } else if (
-                audioInputEnabled &&
-                cb.type === "file" &&
-                isOpenAICompatInlineAudio(
-                  cb.source.media_type,
-                  mediaSourceByteLength(cb.source),
-                )
-              ) {
-                toolResultMedia.push(cb);
-              }
+            ) {
+              toolResultMedia.push(cb);
             }
           }
+          const serialized = serializeToolResult(tr, emittedToolCallIds);
+          if (serialized.kind === "orphaned") {
+            orphanedResultBlocks.push(serialized.block);
+            continue;
+          }
           result.push({
             role: "tool",
             tool_call_id: tr.tool_use_id,
-            content: tr.is_error ? `[ERROR] ${textContent}` : textContent,
+            content: serialized.payload,
           });
         }
 
-        // Emit remaining content + any tool result media as a user message.
-        // Media from tool results (e.g. browser_screenshot, audio a tool read)
-        // must go in a user message because OpenAI-compatible APIs don't
-        // support media parts in tool messages.
-        const userContent = [...otherBlocks, ...toolResultMedia];
+        // Emit remaining content, degraded orphaned results, and any tool
+        // result media as a user message. Media from tool results (e.g.
+        // browser_screenshot, audio a tool read) must go in a user message
+        // because OpenAI-compatible APIs don't support media parts in tool
+        // messages.
+        const userContent = [
+          ...otherBlocks,
+          ...orphanedResultBlocks,
+          ...toolResultMedia,
+        ];
         if (userContent.length > 0) {
           result.push(this.toOpenAIUserMessage(userContent, audioInputEnabled));
         }

package/src/providers/openai/orphaned-tool-result.ts

@@ -0,0 +1,79 @@
+/**
+ * Shared wire-serialization rule for OpenAI tool results.
+ *
+ * Both OpenAI transports reject a tool result whose call was never emitted
+ * earlier in the same request: chat-completions rejects a `tool`-role message
+ * whose `tool_call_id` has no preceding assistant `tool_calls` entry, and the
+ * Responses API rejects a `function_call_output` whose `call_id` has no
+ * preceding `function_call`. Both therefore serialize a result as its native
+ * paired item only on a backward match, and degrade an unmatched one into
+ * plain text carried by the accompanying user message.
+ *
+ * The text payload and the orphan decision are identical across the two
+ * transports, so they live here; only the paired wire shape (and which media
+ * kinds each transport accepts alongside it) differs, and that stays at the
+ * call sites.
+ *
+ * Distinct from `agent/history-repair`'s `[orphaned <type> for <id>]:` marker,
+ * which operates one layer up on persisted history: it repairs the stored
+ * conversation shape (naming the id, and covering `web_search_tool_result`
+ * too) before any transport is chosen. This marker means "the call never
+ * reached this request's wire payload", so the id it would name is absent by
+ * construction. Keeping the two readable apart tells an operator which layer
+ * degraded the block.
+ */
+
+import type { ContentBlock, ToolResultContent } from "../types.js";
+
+/** Text prefix marking a tool result whose call is absent from the request. */
+const ORPHANED_MARKER = "[orphaned tool result]";
+
+/** Text prefix marking a tool result the executor reported as failed. */
+const ERROR_MARKER = "[ERROR]";
+
+/**
+ * How a tool result should be serialized for an OpenAI transport:
+ *
+ * - `paired`: its call was emitted earlier in this request, so the transport
+ *   emits its native tool-output item carrying `payload`.
+ * - `orphaned`: no matching call, so `block` is folded into the user message
+ *   instead of being sent as a rejectable tool-output item.
+ */
+export type ToolResultSerialization =
+  | { kind: "paired"; payload: string }
+  | { kind: "orphaned"; block: ContentBlock };
+
+/**
+ * Flatten a tool result's text (its own `content` plus any text in
+ * `contentBlocks`), mark executor failures, and decide whether the request
+ * can carry it as a paired tool-output item.
+ *
+ * `emittedCallIds` holds the ids already emitted as calls earlier in the same
+ * request, so membership is a backward match by construction. Media in
+ * `contentBlocks` is not consulted: each transport accepts a different set and
+ * collects it separately.
+ */
+export function serializeToolResult(
+  toolResult: ToolResultContent,
+  emittedCallIds: ReadonlySet<string>,
+): ToolResultSerialization {
+  let text = toolResult.content;
+  const extraText = (toolResult.contentBlocks ?? [])
+    .filter(
+      (cb): cb is Extract<ContentBlock, { type: "text" }> => cb.type === "text",
+    )
+    .map((cb) => cb.text);
+  if (extraText.length > 0) {
+    text = text + "\n" + extraText.join("\n");
+  }
+
+  const payload = toolResult.is_error ? `${ERROR_MARKER} ${text}` : text;
+
+  if (!emittedCallIds.has(toolResult.tool_use_id)) {
+    return {
+      kind: "orphaned",
+      block: { type: "text", text: `${ORPHANED_MARKER} ${payload}` },
+    };
+  }
+  return { kind: "paired", payload };
+}