@vellumai/assistant 0.11.2-staging.2 → 0.11.2

package/src/config/bundled-skills/media-processing/services/gemini-video.ts

@@ -15,6 +15,7 @@ import { ApiError, GoogleGenAI } from "@google/genai";
 import { computeRetryDelay, sleep } from "../../../../util/retry.js";
 import { CostTracker } from "./cost-tracker.js";
 import type { MapOutput, SegmentMapResult } from "./gemini-map.js";
+import { resolveMediaAnalysisModel } from "./media-analysis-model.js";
 
 // ---------------------------------------------------------------------------
 // Types
@@ -76,7 +77,7 @@ export async function analyzeVideoDirectly(
   mimeType: string,
   onProgress?: (msg: string) => void,
 ): Promise<MapOutput> {
-  const model = options.model ?? "gemini-2.5-flash";
+  const model = resolveMediaAnalysisModel(options.model);
   const maxRetries = options.maxRetries ?? 3;
 
   // Check file size before uploading

package/src/config/bundled-skills/media-processing/services/media-analysis-model.ts

@@ -0,0 +1,12 @@
+import { resolveModelIntent } from "../../../../providers/model-intents.js";
+
+/**
+ * Resolves the Gemini model used for media analysis. When no override is
+ * supplied (the automatic background-processing path), the default comes
+ * from the catalog-backed vision intent, which is validated against the
+ * provider catalog at module load. Execution and cache identity must both
+ * resolve through this function so they agree on the model.
+ */
+export function resolveMediaAnalysisModel(model?: string): string {
+  return model ?? resolveModelIntent("gemini", "vision-optimized");
+}

package/src/context/compactor.ts

@@ -18,6 +18,7 @@
  * On any parse or resolution failure we abort the compaction and return
  * `compacted: false` — never silently lose messages.
  */
+import { repairHistory } from "../agent/history-repair/history-repair.js";
 import { optimizeImageForTransport } from "../agent/image-optimize.js";
 import type { CompactionConfig } from "../config/schemas/compaction.js";
 import type { LLMCallSite } from "../config/schemas/llm.js";
@@ -1056,9 +1057,14 @@ function extractTextFromResponse(content: ContentBlock[]): string {
 
 // Build the outbound message list for a compaction provider call: apply the
 // same pre-send sanitization bundle as the agent loop's model calls
-// (`preModelCallSanitize` — old tool-result media stripped, AX trees
-// collapsed, historical web-search results converted to text), then append
-// the summarization instruction at the tail.
+// (`preModelCallSanitize`: old tool-result media stripped, AX trees
+// collapsed, historical web-search results converted to text), run the
+// deterministic history repair over the sanitized projection, then append
+// the summarization instruction at the tail. The repair pass downgrades any
+// orphaned `tool_result` (its `tool_use` outside the request, e.g. cut off
+// by front truncation) to plain text and merges consecutive same-role runs,
+// so the request always satisfies the provider's pairing validation. A
+// well-formed history passes through repair structurally unchanged.
 //
 // Matching the loop's projection matters for two reasons. First, the summary
 // call's prefix stays byte-aligned with the agent's warm prompt cache — an
@@ -1076,7 +1082,10 @@ function buildCompactionRequest(
   history: Message[],
   instruction: Message,
 ): Message[] {
-  return [...preModelCallSanitize(history), instruction];
+  return [
+    ...repairHistory(preModelCallSanitize(history)).messages,
+    instruction,
+  ];
 }
 
 // Token headroom a compaction summary call reserves on top of its history: room
@@ -1129,6 +1138,34 @@ function truncateHistoryToBudget(args: {
   if (dropCount === 0) {
     return messages;
   }
+  // Advance the cut to a pair-safe boundary. The budget loop stops wherever
+  // the estimate first fits, which can land between an assistant `tool_use`
+  // and its user `tool_result` and leave an orphaned `tool_result` opening
+  // the retained portion (rejected by providers that validate pairing).
+  // Walk forward to the next clean user boundary, never dropping the final
+  // message (mirroring the budget loop's own bound). When no boundary exists
+  // the requested cut stands; the request-build repair pass downgrades any
+  // orphaned results so the outbound call remains valid.
+  const requestedDropCount = dropCount;
+  if (!isForwardCutBoundary(messages, dropCount)) {
+    for (let i = dropCount + 1; i < messages.length; i++) {
+      if (isForwardCutBoundary(messages, i)) {
+        dropCount = i;
+        break;
+      }
+    }
+  }
+  if (dropCount !== requestedDropCount) {
+    log.info(
+      {
+        requestedDropCount,
+        pairSafeDropCount: dropCount,
+        budgetTokens,
+        totalMessages: messages.length,
+      },
+      "Advanced compaction summary-call front truncation to a pair-safe boundary",
+    );
+  }
   log.info(
     { dropCount, budgetTokens, totalMessages: messages.length },
     "Compaction summary input exceeds context window — truncating from front",

package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts

@@ -0,0 +1,478 @@
+/**
+ * Serialization guard for orphaned tool results on the OpenAI providers.
+ *
+ * Both transports validate pairing server-side: the Responses API rejects a
+ * `function_call_output` whose `call_id` was not emitted as a `function_call`
+ * earlier in the request, and the Chat Completions API rejects a tool-role
+ * message whose `tool_call_id` is not in a preceding assistant message's
+ * `tool_calls`. A `tool_result` with no backward match in the request (e.g.
+ * its `tool_use` truncated away) is therefore degraded into plain user text
+ * with an `[orphaned tool result]` prefix, preserving the information while
+ * keeping the request valid. Paired results serialize unchanged.
+ */
+import { describe, expect, test } from "bun:test";
+
+import type { Message } from "../../types.js";
+import { OpenAIChatCompletionsProvider } from "../chat-completions-provider.js";
+import { OpenAIResponsesProvider } from "../responses-provider.js";
+
+// ---------------------------------------------------------------------------
+// Responses transport harness
+// ---------------------------------------------------------------------------
+
+type ResponsesStreamEvent = { type: string; [key: string]: unknown };
+
+const RESPONSES_OK_EVENTS: ResponsesStreamEvent[] = [
+  { type: "response.output_text.delta", delta: "ok" },
+  {
+    type: "response.completed",
+    response: {
+      model: "gpt-5.2",
+      status: "completed",
+      output: [],
+      usage: { input_tokens: 10, output_tokens: 2 },
+    },
+  },
+];
+
+function makeEventStream(
+  events: ResponsesStreamEvent[],
+): AsyncIterable<ResponsesStreamEvent> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const event of events) {
+        yield event;
+      }
+    },
+  };
+}
+
+/** Swap `responses.create` for a canned stream that records request params. */
+function stubResponsesCreate(provider: OpenAIResponsesProvider): {
+  input: () => unknown[];
+} {
+  let captured: unknown;
+  const inner = provider as unknown as {
+    client: {
+      responses: {
+        create: (
+          params: unknown,
+        ) => Promise<AsyncIterable<ResponsesStreamEvent>>;
+      };
+    };
+  };
+  inner.client.responses.create = async (params) => {
+    captured = params;
+    return makeEventStream(RESPONSES_OK_EVENTS);
+  };
+  return {
+    input: () => (captured as { input: unknown[] }).input,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Chat Completions transport harness
+// ---------------------------------------------------------------------------
+
+type ChatChunk = {
+  choices: Array<{
+    delta: { content?: string | null };
+    finish_reason?: string | null;
+  }>;
+  usage?: { prompt_tokens: number; completion_tokens: number };
+};
+
+const CHAT_OK_CHUNKS: ChatChunk[] = [
+  {
+    choices: [{ delta: { content: "ok" }, finish_reason: "stop" }],
+    usage: { prompt_tokens: 10, completion_tokens: 2 },
+  },
+];
+
+function makeChunkStream(chunks: ChatChunk[]): AsyncIterable<ChatChunk> {
+  return {
+    async *[Symbol.asyncIterator]() {
+      for (const chunk of chunks) {
+        yield chunk;
+      }
+    },
+  };
+}
+
+type ChatMessageParam = {
+  role: string;
+  content: string | Array<Record<string, unknown>>;
+  tool_call_id?: string;
+};
+
+/** Swap chat `create` for a canned stream that records request params. */
+function stubChatCreate(provider: OpenAIChatCompletionsProvider): {
+  messages: () => ChatMessageParam[];
+} {
+  let captured: unknown;
+  const inner = provider as unknown as {
+    client: {
+      chat: {
+        completions: {
+          create: (params: unknown) => Promise<AsyncIterable<ChatChunk>>;
+        };
+      };
+    };
+  };
+  inner.client.chat.completions.create = async (params) => {
+    captured = params;
+    return makeChunkStream(CHAT_OK_CHUNKS);
+  };
+  return {
+    messages: () => (captured as { messages: ChatMessageParam[] }).messages,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Fixtures
+// ---------------------------------------------------------------------------
+
+/**
+ * Persisted tool id shapes: `call_` is the Responses-native pass-through
+ * (conversations that always ran on this transport), `toolu_` is an
+ * Anthropic-shaped history routed to an OpenAI call site (cross-provider
+ * routing, e.g. a compaction call over imported history). The guard must be
+ * id-format-agnostic, so every case runs across both.
+ */
+const ID_SHAPES = [
+  { label: "call_ ids", pairedId: "call_paired", orphanId: "call_orphan" },
+  { label: "toolu_ ids", pairedId: "toolu_paired", orphanId: "toolu_orphan" },
+] as const;
+
+/** History whose tool_result is paired with a preceding tool_use. */
+function pairedHistory(id: string): Message[] {
+  return [
+    { role: "user", content: [{ type: "text", text: "Read the file" }] },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id,
+          name: "file_read",
+          input: { path: "/tmp/a" },
+        },
+      ],
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: id,
+          content: "file contents",
+        },
+      ],
+    },
+  ];
+}
+
+/**
+ * History whose leading tool_result has no matching tool_use anywhere in
+ * the request (the shape a pairing-blind front truncation produces).
+ */
+function orphanHistory(id: string): Message[] {
+  return [
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: id,
+          content: "stranded output",
+        },
+        { type: "text", text: "continue from here" },
+      ],
+    },
+    { role: "assistant", content: [{ type: "text", text: "continuing" }] },
+    { role: "user", content: [{ type: "text", text: "thanks" }] },
+  ];
+}
+
+/**
+ * A tool_result that arrives BEFORE its tool_use in the request order. The
+ * API contract only accepts backward matches, so this is orphaned too.
+ */
+function forwardReferenceHistory(): Message[] {
+  return [
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "call_later",
+          content: "premature output",
+        },
+        { type: "text", text: "odd ordering" },
+      ],
+    },
+    {
+      role: "assistant",
+      content: [
+        {
+          type: "tool_use",
+          id: "call_later",
+          name: "file_read",
+          input: { path: "/tmp/b" },
+        },
+      ],
+    },
+    {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "call_later",
+          content: "real output",
+        },
+      ],
+    },
+  ];
+}
+
+// ---------------------------------------------------------------------------
+// Responses transport
+// ---------------------------------------------------------------------------
+
+describe("OpenAIResponsesProvider orphan tool_result guard", () => {
+  for (const shape of ID_SHAPES) {
+    test(`emits function_call_output for a paired tool_result (${shape.label})`, async () => {
+      const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+      const stub = stubResponsesCreate(provider);
+
+      await provider.sendMessage(pairedHistory(shape.pairedId));
+
+      const input = stub.input() as Array<Record<string, unknown>>;
+      expect(input).toHaveLength(3);
+      expect(input[1]).toEqual({
+        type: "function_call",
+        call_id: shape.pairedId,
+        name: "file_read",
+        arguments: '{"path":"/tmp/a"}',
+      });
+      expect(input[2]).toEqual({
+        type: "function_call_output",
+        call_id: shape.pairedId,
+        output: "file contents",
+      });
+    });
+
+    test(`degrades an orphaned tool_result to user text instead of function_call_output (${shape.label})`, async () => {
+      const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+      const stub = stubResponsesCreate(provider);
+
+      await provider.sendMessage(orphanHistory(shape.orphanId));
+
+      const input = stub.input() as Array<Record<string, unknown>>;
+      // No function_call_output items at all: the only tool_result was
+      // orphaned.
+      expect(input.some((item) => item.type === "function_call_output")).toBe(
+        false,
+      );
+
+      // The orphan's content is preserved as prefixed text inside the user
+      // message, alongside the message's own text.
+      const firstUser = input[0] as {
+        type: string;
+        role: string;
+        content: Array<{ type: string; text?: string }>;
+      };
+      expect(firstUser.type).toBe("message");
+      expect(firstUser.role).toBe("user");
+      const texts = firstUser.content.map((part) => part.text ?? "");
+      expect(texts).toContain("continue from here");
+      expect(texts).toContain("[orphaned tool result] stranded output");
+    });
+  }
+
+  test("treats a forward-referencing tool_result as orphaned, keeps the backward match", async () => {
+    const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+    const stub = stubResponsesCreate(provider);
+
+    await provider.sendMessage(forwardReferenceHistory());
+
+    const input = stub.input() as Array<Record<string, unknown>>;
+    const outputs = input.filter(
+      (item) => item.type === "function_call_output",
+    );
+    // Only the tool_result AFTER the function_call serializes as an output.
+    expect(outputs).toEqual([
+      {
+        type: "function_call_output",
+        call_id: "call_later",
+        output: "real output",
+      },
+    ]);
+    // The premature result is degraded into the first user message.
+    const firstUser = input[0] as {
+      content: Array<{ type: string; text?: string }>;
+    };
+    const texts = firstUser.content.map((part) => part.text ?? "");
+    expect(texts).toContain("[orphaned tool result] premature output");
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Chat Completions transport
+// ---------------------------------------------------------------------------
+
+describe("OpenAIChatCompletionsProvider orphan tool_result guard", () => {
+  function makeProvider(): OpenAIChatCompletionsProvider {
+    return new OpenAIChatCompletionsProvider("sk-test", "test-model", {
+      providerName: "openai",
+      providerLabel: "OpenAI",
+    });
+  }
+
+  for (const shape of ID_SHAPES) {
+    test(`emits a tool message for a paired tool_result (${shape.label})`, async () => {
+      const provider = makeProvider();
+      const stub = stubChatCreate(provider);
+
+      await provider.sendMessage(pairedHistory(shape.pairedId));
+
+      const sent = stub.messages();
+      const toolMessages = sent.filter((message) => message.role === "tool");
+      expect(toolMessages).toEqual([
+        {
+          role: "tool",
+          tool_call_id: shape.pairedId,
+          content: "file contents",
+        },
+      ]);
+    });
+
+    test(`degrades an orphaned tool_result to user text instead of a tool message (${shape.label})`, async () => {
+      const provider = makeProvider();
+      const stub = stubChatCreate(provider);
+
+      await provider.sendMessage(orphanHistory(shape.orphanId));
+
+      const sent = stub.messages();
+      expect(sent.some((message) => message.role === "tool")).toBe(false);
+
+      const firstUser = sent.find((message) => message.role === "user");
+      expect(firstUser).toBeDefined();
+      const content = firstUser!.content;
+      const texts = Array.isArray(content)
+        ? content.map((part) => (part.text as string) ?? "")
+        : [content];
+      expect(texts).toContain("continue from here");
+      expect(texts).toContain("[orphaned tool result] stranded output");
+    });
+  }
+
+  test("treats a forward-referencing tool_result as orphaned, keeps the backward match", async () => {
+    const provider = makeProvider();
+    const stub = stubChatCreate(provider);
+
+    await provider.sendMessage(forwardReferenceHistory());
+
+    const sent = stub.messages();
+    const toolMessages = sent.filter((message) => message.role === "tool");
+    expect(toolMessages).toEqual([
+      {
+        role: "tool",
+        tool_call_id: "call_later",
+        content: "real output",
+      },
+    ]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Cross-transport agreement
+// ---------------------------------------------------------------------------
+
+describe("orphan degradation agrees across both OpenAI transports", () => {
+  /** Every degraded orphan text a request carried, in request order. */
+  async function responsesOrphanTexts(history: Message[]): Promise<string[]> {
+    const provider = new OpenAIResponsesProvider("sk-test", "gpt-5.2");
+    const stub = stubResponsesCreate(provider);
+    await provider.sendMessage(history);
+    const input = stub.input() as Array<Record<string, unknown>>;
+    return input
+      .flatMap((item) =>
+        Array.isArray(item.content) ? (item.content as unknown[]) : [],
+      )
+      .map((part) => (part as { text?: string }).text ?? "")
+      .filter((text) => text.startsWith("[orphaned"));
+  }
+
+  async function chatOrphanTexts(history: Message[]): Promise<string[]> {
+    const provider = new OpenAIChatCompletionsProvider(
+      "sk-test",
+      "test-model",
+      {
+        providerName: "openai",
+        providerLabel: "OpenAI",
+      },
+    );
+    const stub = stubChatCreate(provider);
+    await provider.sendMessage(history);
+    return (
+      stub
+        .messages()
+        // A user message carrying a single text part serializes as a plain
+        // string on this transport rather than a one-element array.
+        .flatMap((message) =>
+          Array.isArray(message.content)
+            ? (message.content as unknown[]).map(
+                (part) => (part as { text?: string }).text ?? "",
+              )
+            : [String(message.content ?? "")],
+        )
+        .filter((text) => text.startsWith("[orphaned"))
+    );
+  }
+
+  // The detection rule and the degraded wording live in one shared helper
+  // (`serializeToolResult`), so the two transports cannot drift into
+  // different markers for the same input. Byte equality is the assertion:
+  // re-inlining the rule in one converter and editing it there fails here,
+  // which is the regression this pins.
+  test("both transports degrade the same orphan to byte-identical text", async () => {
+    const history = orphanHistory("call_orphan");
+
+    const [fromResponses, fromChat] = await Promise.all([
+      responsesOrphanTexts(history),
+      chatOrphanTexts(history),
+    ]);
+
+    expect(fromResponses).toEqual(["[orphaned tool result] stranded output"]);
+    expect(fromChat).toEqual(fromResponses);
+  });
+
+  test("both transports carry an executor failure into the same degraded text", async () => {
+    // `is_error` prefixing is part of the shared payload rule, so it must
+    // survive degradation identically on both transports.
+    const history: Message[] = [
+      {
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: "call_orphan",
+            content: "boom",
+            is_error: true,
+          },
+        ],
+      },
+      { role: "assistant", content: [{ type: "text", text: "continuing" }] },
+      { role: "user", content: [{ type: "text", text: "thanks" }] },
+    ];
+
+    const [fromResponses, fromChat] = await Promise.all([
+      responsesOrphanTexts(history),
+      chatOrphanTexts(history),
+    ]);
+
+    expect(fromResponses).toEqual(["[orphaned tool result] [ERROR] boom"]);
+    expect(fromChat).toEqual(fromResponses);
+  });
+});

package/src/providers/openai/chat-completions-provider.ts

@@ -46,6 +46,7 @@ import {
   OPENAI_COMPAT_MAX_INLINE_AUDIO_BYTES,
   openAIInputAudioFormat,
 } from "./input-audio.js";
+import { serializeToolResult } from "./orphaned-tool-result.js";
 
 /**
  * Detect OpenAI-compatible context-overflow signals on an `OpenAI.APIError`.
@@ -956,9 +957,19 @@ export class OpenAIChatCompletionsProvider implements Provider {
       });
     }
 
+    // Tool-call ids emitted in assistant messages earlier in this request.
+    // The API rejects a tool-role message whose `tool_call_id` has no
+    // preceding assistant `tool_calls` entry, so tool results are only
+    // serialized as tool messages when their call was emitted first
+    // (backward matches only).
+    const emittedToolCallIds = new Set<string>();
     for (const msg of messages) {
       if (msg.role === "assistant") {
-        result.push(this.toOpenAIAssistantMessage(msg));
+        const assistantMessage = this.toOpenAIAssistantMessage(msg);
+        for (const toolCall of assistantMessage.tool_calls ?? []) {
+          emittedToolCallIds.add(toolCall.id);
+        }
+        result.push(assistantMessage);
       } else {
         // User messages may contain tool_result blocks mixed with text/image
         const toolResults = msg.content.filter(
@@ -975,46 +986,51 @@ export class OpenAIChatCompletionsProvider implements Provider {
         // Emit tool results as separate tool-role messages
         // OpenAI's API only supports string content in tool messages, so media
         // from contentBlocks is collected and injected as a user message below.
+        // A tool_result whose id has no preceding assistant tool call in this
+        // request is orphaned; its content is degraded into the user message
+        // instead of being sent as a rejectable tool message.
         const toolResultMedia: ContentBlock[] = [];
+        const orphanedResultBlocks: ContentBlock[] = [];
         for (const tr of toolResults) {
-          let textContent = tr.content;
-          if (tr.contentBlocks && tr.contentBlocks.length > 0) {
-            const extraText = tr.contentBlocks
-              .filter(
-                (cb): cb is Extract<ContentBlock, { type: "text" }> =>
-                  cb.type === "text",
+          // Media this transport can carry: images always, plus inline audio
+          // when the model accepts it. The text payload and the orphan
+          // decision are the shared cross-transport rule.
+          for (const cb of tr.contentBlocks ?? []) {
+            if (cb.type === "image") {
+              toolResultMedia.push(cb);
+            } else if (
+              audioInputEnabled &&
+              cb.type === "file" &&
+              isOpenAICompatInlineAudio(
+                cb.source.media_type,
+                mediaSourceByteLength(cb.source),
               )
-              .map((cb) => cb.text);
-            if (extraText.length > 0) {
-              textContent = textContent + "\n" + extraText.join("\n");
-            }
-            for (const cb of tr.contentBlocks) {
-              if (cb.type === "image") {
-                toolResultMedia.push(cb);
-              } else if (
-                audioInputEnabled &&
-                cb.type === "file" &&
-                isOpenAICompatInlineAudio(
-                  cb.source.media_type,
-                  mediaSourceByteLength(cb.source),
-                )
-              ) {
-                toolResultMedia.push(cb);
-              }
+            ) {
+              toolResultMedia.push(cb);
             }
           }
+          const serialized = serializeToolResult(tr, emittedToolCallIds);
+          if (serialized.kind === "orphaned") {
+            orphanedResultBlocks.push(serialized.block);
+            continue;
+          }
           result.push({
             role: "tool",
             tool_call_id: tr.tool_use_id,
-            content: tr.is_error ? `[ERROR] ${textContent}` : textContent,
+            content: serialized.payload,
           });
         }
 
-        // Emit remaining content + any tool result media as a user message.
-        // Media from tool results (e.g. browser_screenshot, audio a tool read)
-        // must go in a user message because OpenAI-compatible APIs don't
-        // support media parts in tool messages.
-        const userContent = [...otherBlocks, ...toolResultMedia];
+        // Emit remaining content, degraded orphaned results, and any tool
+        // result media as a user message. Media from tool results (e.g.
+        // browser_screenshot, audio a tool read) must go in a user message
+        // because OpenAI-compatible APIs don't support media parts in tool
+        // messages.
+        const userContent = [
+          ...otherBlocks,
+          ...orphanedResultBlocks,
+          ...toolResultMedia,
+        ];
         if (userContent.length > 0) {
           result.push(this.toOpenAIUserMessage(userContent, audioInputEnabled));
         }