@vellumai/assistant 0.12.6-staging.3 → 0.12.7-staging.1

package/src/agent/loop-unparseable-tool-retry.test.ts

@@ -0,0 +1,333 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { createMockProvider } from "../__tests__/helpers/mock-provider.js";
+import { MESSAGE_CATALOGS, MESSAGE_KEYS } from "../i18n/index.js";
+import { resetPluginRegistryAndRegisterDefaults } from "../plugins/defaults/index.js";
+import type {
+  ContentBlock,
+  Message,
+  ProviderResponse,
+} from "../providers/types.js";
+import { wrapUnparseableToolArgs } from "../providers/unparseable-tool-args.js";
+import { type AgentEvent, AgentLoop } from "./loop.js";
+
+const userMessage: Message = {
+  role: "user",
+  content: [{ type: "text", text: "Write the file" }],
+};
+
+function malformedToolResponse(
+  id: string,
+  raw: string,
+  options: { name?: string; text?: string } = {},
+): ProviderResponse {
+  return {
+    content: [
+      ...(options.text ? [{ type: "text" as const, text: options.text }] : []),
+      {
+        type: "tool_use",
+        id,
+        name: options.name ?? "file_write",
+        input: wrapUnparseableToolArgs(raw),
+      },
+    ],
+    model: "mock-model",
+    usage: { inputTokens: 1, outputTokens: 1 },
+    stopReason: "tool_use",
+  };
+}
+
+function toolBlocks(
+  history: Message[],
+  type: "tool_use" | "tool_result",
+): ContentBlock[] {
+  return history
+    .flatMap((message) => message.content)
+    .filter((block) => {
+      return block.type === type;
+    });
+}
+
+describe("AgentLoop: unparseable tool retries", () => {
+  beforeEach(() => {
+    resetPluginRegistryAndRegisterDefaults();
+  });
+
+  test("bounds persisted input and stops after three consecutive malformed turns", async () => {
+    const raw =
+      '{"path":"/workspace/file.md","content":"' + "x".repeat(2_000_000);
+    const { provider, calls } = createMockProvider([
+      malformedToolResponse("bad-1", raw),
+      malformedToolResponse("bad-2", raw),
+      malformedToolResponse("bad-3", raw, { text: "Still trying." }),
+      malformedToolResponse("bad-4", raw),
+    ]);
+    const executionInputs: Record<string, unknown>[] = [];
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "conversation-123",
+      tools: [
+        {
+          name: "file_write",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async (_name, input) => {
+        executionInputs.push(input);
+        return { content: "invalid JSON", isError: true };
+      },
+    });
+
+    const result = await loop.run({
+      requestId: "request-123",
+      messages: [userMessage],
+      onEvent: (event) => {
+        events.push(event);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(3);
+    expect(executionInputs).toHaveLength(2);
+    expect(executionInputs[0]).toEqual(wrapUnparseableToolArgs(raw));
+    expect(toolBlocks(result.history, "tool_use")).toHaveLength(2);
+    expect(toolBlocks(result.history, "tool_result")).toHaveLength(2);
+
+    const serializedHistory = JSON.stringify(result.history);
+    expect(serializedHistory.length).toBeLessThan(5_000);
+    expect(serializedHistory).not.toContain("x".repeat(1_000));
+    expect(result.history.at(-1)?.content).toEqual([
+      { type: "text", text: "Still trying." },
+      {
+        type: "text",
+        text: `\n\n${MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP}`,
+      },
+    ]);
+    expect(events.filter((event) => event.type === "tool_use")).toHaveLength(2);
+    expect(
+      events
+        .filter((event) => event.type === "text_delta")
+        .map((event) => event.text)
+        .join(""),
+    ).toEndWith(
+      `Still trying.\n\n${
+        MESSAGE_CATALOGS.en[MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP]
+      }`,
+    );
+  });
+
+  test("surfaces only the stop notice when assistant text is tool-gated", async () => {
+    const raw = "{" + "x".repeat(1_000);
+    const { provider } = createMockProvider([
+      malformedToolResponse("bad-1", raw, { text: "Private notes one." }),
+      malformedToolResponse("bad-2", raw, { text: "Private notes two." }),
+      malformedToolResponse("bad-3", raw, { text: "Private notes three." }),
+    ]);
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "conversation-gated",
+      tools: [
+        {
+          name: "file_write",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async () => ({ content: "invalid JSON", isError: true }),
+    });
+
+    const result = await loop.run({
+      requestId: "request-gated",
+      messages: [userMessage],
+      onEvent: (event) => {
+        events.push(event);
+      },
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+      suppressAssistantText: true,
+    });
+
+    expect(result.history.at(-1)?.content).toEqual([
+      {
+        type: "text",
+        text: MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP,
+      },
+    ]);
+    expect(
+      events
+        .filter((event) => event.type === "text_delta")
+        .map((event) => event.text)
+        .join(""),
+    ).toBe(
+      MESSAGE_CATALOGS.en[MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP],
+    );
+    const finalMessage = events
+      .filter((event) => event.type === "message_complete")
+      .at(-1);
+    expect(finalMessage?.assistantTextVisibility).toBe("visible");
+  });
+
+  test("streams the localized stop notice while persisting its key", async () => {
+    const raw = "{" + "x".repeat(1_000);
+    const { provider } = createMockProvider([
+      malformedToolResponse("bad-1", raw),
+      malformedToolResponse("bad-2", raw),
+      malformedToolResponse("bad-3", raw),
+    ]);
+    const events: AgentEvent[] = [];
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "conversation-spanish",
+      tools: [
+        {
+          name: "file_write",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async () => ({ content: "invalid JSON", isError: true }),
+    });
+
+    const result = await loop.run({
+      requestId: "request-spanish",
+      messages: [userMessage],
+      onEvent: (event) => {
+        events.push(event);
+      },
+      locale: "es",
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(result.history.at(-1)?.content).toEqual([
+      {
+        type: "text",
+        text: MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP,
+      },
+    ]);
+    expect(
+      events
+        .filter((event) => event.type === "text_delta")
+        .map((event) => event.text)
+        .join(""),
+    ).toBe(
+      MESSAGE_CATALOGS.es[MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP],
+    );
+  });
+
+  test("does not count skill_execute envelopes recovered from valid outer JSON", async () => {
+    const recoverableRaw = JSON.stringify({
+      tool: "document_update",
+      input: "body",
+      activity: "Updating the document",
+    });
+    const malformedRaw = "{" + "x".repeat(1_000);
+    const done: ProviderResponse = {
+      content: [{ type: "text", text: "Done" }],
+      model: "mock-model",
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: "end_turn",
+    };
+    const { provider, calls } = createMockProvider([
+      malformedToolResponse("recover-1", recoverableRaw, {
+        name: "skill_execute",
+      }),
+      malformedToolResponse("bad-1", malformedRaw),
+      malformedToolResponse("recover-2", recoverableRaw, {
+        name: "skill_execute",
+      }),
+      malformedToolResponse("bad-2", malformedRaw),
+      done,
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "conversation-789",
+      tools: [
+        {
+          name: "skill_execute",
+          description: "",
+          input_schema: { type: "object" },
+        },
+        {
+          name: "file_write",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async () => ({ content: "result", isError: false }),
+    });
+
+    const result = await loop.run({
+      requestId: "request-789",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(5);
+    expect(result.history.at(-1)?.content).toEqual([
+      { type: "text", text: "Done" },
+    ]);
+  });
+
+  test("resets the streak after a well-formed tool turn", async () => {
+    const raw = "{" + "x".repeat(1_000);
+    const valid: ProviderResponse = {
+      content: [
+        {
+          type: "tool_use",
+          id: "valid",
+          name: "file_write",
+          input: { path: "/workspace/file.md", content: "ok" },
+        },
+      ],
+      model: "mock-model",
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: "tool_use",
+    };
+    const done: ProviderResponse = {
+      content: [{ type: "text", text: "Done" }],
+      model: "mock-model",
+      usage: { inputTokens: 1, outputTokens: 1 },
+      stopReason: "end_turn",
+    };
+    const { provider, calls } = createMockProvider([
+      malformedToolResponse("bad-1", raw),
+      malformedToolResponse("bad-2", raw),
+      valid,
+      malformedToolResponse("bad-3", raw),
+      malformedToolResponse("bad-4", raw),
+      done,
+    ]);
+    const loop = new AgentLoop({
+      provider,
+      systemPrompt: "system",
+      conversationId: "conversation-456",
+      tools: [
+        {
+          name: "file_write",
+          description: "",
+          input_schema: { type: "object" },
+        },
+      ],
+      toolExecutor: async () => ({ content: "result", isError: false }),
+    });
+
+    const result = await loop.run({
+      requestId: "request-456",
+      messages: [userMessage],
+      onEvent: () => {},
+      trust: { sourceChannel: "vellum", trustClass: "unknown" },
+    });
+
+    expect(calls).toHaveLength(6);
+    expect(result.history.at(-1)?.content).toEqual([
+      { type: "text", text: "Done" },
+    ]);
+  });
+});

package/src/agent/loop.ts

@@ -28,6 +28,12 @@ import type {
   PreModelCallInputContext,
   StopInputContext,
 } from "../hooks/types.js";
+import {
+  MESSAGE_KEYS,
+  resolveStoredMessageText,
+  type SupportedLocale,
+  t,
+} from "../i18n/index.js";
 import {
   timeSyncSection,
   traceAsyncSection,
@@ -52,6 +58,10 @@ import type {
   ToolResultContent,
 } from "../providers/types.js";
 import { NATIVE_WEB_SEARCH_TOOL_NAME } from "../providers/types.js";
+import {
+  boundUnparseableToolArgs,
+  isUnparseableToolArgs,
+} from "../providers/unparseable-tool-args.js";
 import { recordWatchdogEvent } from "../telemetry/watchdog-events-store.js";
 import {
   ABORT_SETTLE_GRACE_MS,
@@ -64,6 +74,7 @@ import {
   applyStreamingSubstitution,
   applySubstitutions,
 } from "../tools/sensitive-output-placeholders.js";
+import { recoverSkillExecuteEnvelope } from "../tools/skills/execute.js";
 import {
   abortedToolResultText,
   isPreemptedByNewMessage,
@@ -83,6 +94,8 @@ const log = getLogger("agent-loop");
 
 /** Watchdog check name for both tool-gated reply outcomes. */
 const SEND_USER_MESSAGE_CHECK = "send_user_message_delivery";
+const MAX_CONSECUTIVE_UNPARSEABLE_TOOL_TURNS = 3;
+const SKILL_EXECUTE_TOOL_NAME = "skill_execute";
 
 /**
  * The user-facing message a `send_user_message` block carries, or null when
@@ -696,6 +709,8 @@ interface AgentLoopRunOptionsBase {
   injectionLedgerResets?: Set<string>;
   signal?: AbortSignal;
   requestId: string;
+  /** Locale for daemon-authored text emitted during this run. */
+  locale?: SupportedLocale;
   /**
    * Explicit model override (provider/model string) for every LLM call in
    * this run. When omitted, the model is resolved through the normal
@@ -1011,12 +1026,31 @@ function toolResultEventFields(
 interface NormalizedToolUse {
   /** Assistant content with at most one `tool_use` block per call id. */
   content: ContentBlock[];
-  /** The `tool_use` blocks in `content`, in order. */
+  /** The bounded `tool_use` blocks in `content`, in order. */
   toolUseBlocks: ToolUseBlock[];
+  /** Original inputs retained only until this iteration's execution attempt. */
+  executionInputsById: Map<string, Record<string, unknown>>;
+  /** Number of executable calls whose provider arguments did not parse. */
+  unparseableCount: number;
   /** Coalesced copies: a call id and name for each block dropped. */
   duplicates: Array<{ id: string; name: string }>;
 }
 
+function isRetryableUnparseableToolCall(block: ToolUseBlock): boolean {
+  if (!isUnparseableToolArgs(block.input)) {
+    return false;
+  }
+  if (block.name !== SKILL_EXECUTE_TOOL_NAME) {
+    return true;
+  }
+  const recovered = recoverSkillExecuteEnvelope(block.input);
+  return (
+    recovered === block.input ||
+    typeof recovered.tool !== "string" ||
+    recovered.tool.length === 0
+  );
+}
+
 /**
  * Resolves an assistant reply's `tool_use` blocks into an executable set keyed
  * by call id: a block with no id gets one, and a repeat of an id already in the
@@ -1030,8 +1064,10 @@ function normalizeToolUseBlocks(
 ): NormalizedToolUse {
   const nextContent: ContentBlock[] = [];
   const toolUseBlocks: ToolUseBlock[] = [];
+  const executionInputsById = new Map<string, Record<string, unknown>>();
   const duplicates: Array<{ id: string; name: string }> = [];
   const seenIds = new Set<string>();
+  let unparseableCount = 0;
 
   for (const block of content) {
     if (block.type !== "tool_use") {
@@ -1042,14 +1078,28 @@ function normalizeToolUseBlocks(
       duplicates.push({ id: block.id, name: block.name });
       continue;
     }
-    const normalized: ToolUseBlock =
-      block.id.length === 0 ? { ...block, id: crypto.randomUUID() } : block;
-    seenIds.add(normalized.id);
-    nextContent.push(normalized);
-    toolUseBlocks.push(normalized);
+    const normalizedId = block.id.length === 0 ? crypto.randomUUID() : block.id;
+    const bounded: ToolUseBlock = {
+      ...block,
+      id: normalizedId,
+      input: boundUnparseableToolArgs(block.input),
+    };
+    seenIds.add(normalizedId);
+    executionInputsById.set(normalizedId, block.input);
+    if (isRetryableUnparseableToolCall(block)) {
+      unparseableCount++;
+    }
+    nextContent.push(bounded);
+    toolUseBlocks.push(bounded);
   }
 
-  return { content: nextContent, toolUseBlocks, duplicates };
+  return {
+    content: nextContent,
+    toolUseBlocks,
+    executionInputsById,
+    unparseableCount,
+    duplicates,
+  };
 }
 
 /**
@@ -1486,6 +1536,7 @@ export class AgentLoop {
     let newMessagesStart = history.length;
     let toolUseTurns = 0;
     let postModelCallContinues = 0;
+    let consecutiveUnparseableToolTurns = 0;
     // Whether the user has been told the OUTCOME of the work so far, not
     // merely that work started. True only while the most recent tool-bearing
     // response was `send_user_message` and nothing else: a response that sends
@@ -1741,6 +1792,7 @@ export class AgentLoop {
       // request from a generation that died mid-flight. Declared here so that
       // catch can reach it.
       let streamedTokens = false;
+      let unparseableToolRetryLimitReached = false;
 
       try {
         // ── Pre-call budget gate ─────────────────────────────────────
@@ -2728,6 +2780,44 @@ export class AgentLoop {
           content: normalizedToolUse.content,
         };
         toolUseBlocks = normalizedToolUse.toolUseBlocks;
+        if (normalizedToolUse.unparseableCount > 0) {
+          consecutiveUnparseableToolTurns++;
+        } else {
+          consecutiveUnparseableToolTurns = 0;
+        }
+        if (
+          consecutiveUnparseableToolTurns >=
+          MAX_CONSECUTIVE_UNPARSEABLE_TOOL_TURNS
+        ) {
+          rlog.warn(
+            {
+              turn: toolUseTurns,
+              consecutiveUnparseableToolTurns,
+              unparseableToolCount: normalizedToolUse.unparseableCount,
+            },
+            "Repeated unparseable tool calls reached the per-run limit",
+          );
+          unparseableToolRetryLimitReached = true;
+          const visiblePrefix = suppressAssistantText
+            ? []
+            : assistantMessage.content.filter(
+                (block) => block.type !== "tool_use",
+              );
+          assistantMessage = {
+            role: "assistant",
+            content: [
+              ...visiblePrefix,
+              {
+                type: "text",
+                text: `${
+                  hasVisibleText(visiblePrefix) ? "\n\n" : ""
+                }${MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP}`,
+              },
+            ],
+          };
+          toolUseBlocks = [];
+          userToldOutcome = false;
+        }
 
         // At the no-tool stop boundary the retry decision is actionable: a
         // recovery hook may repair history and ask to re-query (a tool-bearing
@@ -2736,6 +2826,7 @@ export class AgentLoop {
         // persisting it; the per-run backstop keeps a misbehaving hook from
         // spinning forever.
         if (
+          !unparseableToolRetryLimitReached &&
           toolUseBlocks.length === 0 &&
           postModelCallDecision === "continue"
         ) {
@@ -2797,11 +2888,40 @@ export class AgentLoop {
         // Surface the finalized text if the client saw nothing live (a
         // deferred stream, a hook-rewritten empty turn, or the tool-gated
         // fallback where no `send_user_message` call ever reached the user).
-        const textVisibility = textVisibilityOf(
-          emitFinalAssistantText(assistantMessage.content, {
+        let fallbackSurfaced = false;
+        if (unparseableToolRetryLimitReached && streamedVisibleText) {
+          onEvent({
+            type: "text_delta",
+            text: resolveStoredMessageText(
+              assistantTextOf(assistantMessage.content.slice(-1)),
+              options.locale,
+            ),
+          });
+          fallbackSurfaced = suppressAssistantText;
+        } else {
+          const displayContent = unparseableToolRetryLimitReached
+            ? assistantMessage.content.map((block) =>
+                block.type === "text" &&
+                block.text.trim() ===
+                  MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP
+                  ? {
+                      ...block,
+                      text: block.text.replace(
+                        MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP,
+                        t(
+                          MESSAGE_KEYS.AGENT_LOOP_UNPARSEABLE_TOOL_RETRY_STOP,
+                          options.locale,
+                        ),
+                      ),
+                    }
+                  : block,
+              )
+            : assistantMessage.content;
+          fallbackSurfaced = emitFinalAssistantText(displayContent, {
             turnEnding: toolUseBlocks.length === 0,
-          }),
-        );
+          });
+        }
+        const textVisibility = textVisibilityOf(fallbackSurfaced);
 
         history.push(assistantMessage);
 
@@ -2914,7 +3034,8 @@ export class AgentLoop {
               }
               const result = await this.toolExecutor!(
                 toolUse.name,
-                toolUse.input,
+                normalizedToolUse.executionInputsById.get(toolUse.id) ??
+                  toolUse.input,
                 (chunk) => {
                   onEvent({
                     type: "tool_output_chunk",