@convex-dev/agent 0.3.2 → 0.6.0-alpha.0

package/src/client/mockModel.ts

@@ -1,7 +1,7 @@
 import type {
-  LanguageModelV2,
-  LanguageModelV2Content,
-  LanguageModelV2StreamPart,
+  LanguageModelV3,
+  LanguageModelV3Content,
+  LanguageModelV3StreamPart,
 } from "@ai-sdk/provider";
 import { simulateReadableStream, type ProviderMetadata } from "ai";
 import { assert, pick } from "convex-helpers";
@@ -12,14 +12,20 @@ B B B B B B B B B B B B B B B
 C C C C C C C C C C C C C C C
 D D D D D D D D D D D D D D D
 `;
-const DEFAULT_USAGE = { outputTokens: 10, inputTokens: 3, totalTokens: 13 };
+const DEFAULT_USAGE = {
+  outputTokens: 10,
+  inputTokens: 3,
+  totalTokens: 13,
+  inputTokenDetails: undefined,
+  outputTokenDetails: undefined,
+};
 
 export type MockModelArgs = {
-  provider?: LanguageModelV2["provider"];
-  modelId?: LanguageModelV2["modelId"];
+  provider?: LanguageModelV3["provider"];
+  modelId?: LanguageModelV3["modelId"];
   supportedUrls?:
-    | LanguageModelV2["supportedUrls"]
-    | (() => LanguageModelV2["supportedUrls"]);
+    | LanguageModelV3["supportedUrls"]
+    | (() => LanguageModelV3["supportedUrls"]);
   chunkDelayInMs?: number;
   initialDelayInMs?: number;
   /** A list of the responses for multiple steps.
@@ -27,15 +33,15 @@ export type MockModelArgs = {
    * then the next list would be after the tool response or another tool call.
    * Tool responses come from actual tool calls!
    */
-  contentSteps?: LanguageModelV2Content[][];
+  contentSteps?: LanguageModelV3Content[][];
   /** A single list of content responded from each step.
    * Provide contentSteps instead if you want to do multi-step responses with
    * tool calls.
    */
-  content?: LanguageModelV2Content[];
+  content?: LanguageModelV3Content[];
   // provide either content, contentResponses or doGenerate & doStream
-  doGenerate?: LanguageModelV2["doGenerate"];
-  doStream?: LanguageModelV2["doStream"];
+  doGenerate?: LanguageModelV3["doGenerate"];
+  doStream?: LanguageModelV3["doStream"];
   providerMetadata?: ProviderMetadata;
   fail?:
     | boolean
@@ -49,23 +55,23 @@ function atMostOneOf(...args: unknown[]) {
   return args.filter(Boolean).length <= 1;
 }
 
-export function mockModel(args?: MockModelArgs): LanguageModelV2 {
+export function mockModel(args?: MockModelArgs): LanguageModelV3 {
   return new MockLanguageModel(args ?? {});
 }
 
-export class MockLanguageModel implements LanguageModelV2 {
-  readonly specificationVersion = "v2";
+export class MockLanguageModel implements LanguageModelV3 {
+  readonly specificationVersion = "v3";
 
-  private _supportedUrls: () => LanguageModelV2["supportedUrls"];
+  private _supportedUrls: () => LanguageModelV3["supportedUrls"];
 
-  readonly provider: LanguageModelV2["provider"];
-  readonly modelId: LanguageModelV2["modelId"];
+  readonly provider: LanguageModelV3["provider"];
+  readonly modelId: LanguageModelV3["modelId"];
 
-  doGenerate: LanguageModelV2["doGenerate"];
-  doStream: LanguageModelV2["doStream"];
+  doGenerate: LanguageModelV3["doGenerate"];
+  doStream: LanguageModelV3["doStream"];
 
-  doGenerateCalls: Parameters<LanguageModelV2["doGenerate"]>[0][] = [];
-  doStreamCalls: Parameters<LanguageModelV2["doStream"]>[0][] = [];
+  doGenerateCalls: Parameters<LanguageModelV3["doGenerate"]>[0][] = [];
+  doStreamCalls: Parameters<LanguageModelV3["doStream"]>[0][] = [];
 
   constructor(args: MockModelArgs) {
     assert(
@@ -95,19 +101,19 @@ export class MockLanguageModel implements LanguageModelV2 {
       "Mock error message";
     const metadata = pick(args, ["providerMetadata"]);
 
-    const chunkResponses: LanguageModelV2StreamPart[][] = contentSteps.map(
+    const chunkResponses: LanguageModelV3StreamPart[][] = contentSteps.map(
       (content) => {
-        const chunks: LanguageModelV2StreamPart[] = [
+        const chunks: LanguageModelV3StreamPart[] = [
           { type: "stream-start", warnings: [] },
         ];
         chunks.push(
-          ...content.flatMap((c, ci): LanguageModelV2StreamPart[] => {
+          ...content.flatMap((c, ci): LanguageModelV3StreamPart[] => {
             if (c.type !== "text" && c.type !== "reasoning") {
               return [c];
             }
             const metadata = pick(c, ["providerMetadata"]);
             const deltas = c.text.split(" ");
-            const parts: LanguageModelV2StreamPart[] = [];
+            const parts: LanguageModelV3StreamPart[] = [];
             if (c.type === "reasoning") {
               parts.push({
                 type: "reasoning-start",
@@ -122,7 +128,7 @@ export class MockLanguageModel implements LanguageModelV2 {
                       delta: (di ? " " : "") + delta,
                       id: `reasoning-${ci}`,
                       ...metadata,
-                    }) satisfies LanguageModelV2StreamPart,
+                    }) satisfies LanguageModelV3StreamPart,
                 ),
               );
               parts.push({
@@ -144,7 +150,7 @@ export class MockLanguageModel implements LanguageModelV2 {
                       delta: (di ? " " : "") + delta,
                       id: `txt-${ci}`,
                       ...metadata,
-                    }) satisfies LanguageModelV2StreamPart,
+                    }) satisfies LanguageModelV3StreamPart,
                 ),
               );
               parts.push({
@@ -166,7 +172,7 @@ export class MockLanguageModel implements LanguageModelV2 {
           type: "finish",
           finishReason: fail ? "error" : "stop",
           usage: DEFAULT_USAGE,
-          ...metadata,
+          ...(metadata as any),
         });
         return chunks;
       },
@@ -187,7 +193,7 @@ export class MockLanguageModel implements LanguageModelV2 {
           content: contentSteps[callIndex % contentSteps.length],
           finishReason: "stop" as const,
           usage: DEFAULT_USAGE,
-          ...metadata,
+          ...(metadata as any),
           warnings: [],
         };
         callIndex++;
@@ -196,10 +202,6 @@ export class MockLanguageModel implements LanguageModelV2 {
         throw new Error("Unexpected: no content or doGenerate");
       }
     };
-    this._supportedUrls =
-      typeof supportedUrls === "function"
-        ? supportedUrls
-        : async () => supportedUrls;
     this.doStream = async (options) => {
       this.doStreamCalls.push(options);
 

package/src/client/saveInputMessages.ts

@@ -31,7 +31,7 @@ export async function saveInputMessages(
     storageOptions?: {
       saveMessages?: "all" | "promptAndOutput";
     };
-  } & Pick<Config, "usageHandler" | "textEmbeddingModel" | "callSettings">,
+  } & Pick<Config, "usageHandler" | "textEmbeddingModel" | "embeddingModel" | "callSettings">,
 ): Promise<{
   promptMessageId: string | undefined;
   pendingMessage: MessageDoc;
@@ -64,7 +64,7 @@ export async function saveInputMessages(
         model: string;
       }
     | undefined;
-  if (args.textEmbeddingModel && toSave.length) {
+  if ((args.embeddingModel ?? args.textEmbeddingModel) && toSave.length) {
     assert(
       "runAction" in ctx,
       "You must be in an action context to generate embeddings",

package/src/client/search.test.ts

@@ -159,6 +159,7 @@ describe("search.ts", () => {
                 type: "tool-call",
                 toolCallId: "call_123",
                 toolName: "test",
+                input: {},
                 args: {},
               },
             ],
@@ -202,6 +203,7 @@ describe("search.ts", () => {
                 type: "tool-call",
                 toolCallId: "call_orphaned",
                 toolName: "test",
+                input: {},
                 args: {},
               },
             ],
@@ -234,6 +236,170 @@ describe("search.ts", () => {
       expect(result[0]._id).toBe("0");
       expect(result[1]._id).toBe("3");
     });
+
+    it("should keep tool calls with approval responses (but no tool-result yet)", () => {
+      const messages: MessageDoc[] = [
+        {
+          _id: "1",
+          message: {
+            role: "assistant",
+            content: [
+              { type: "text", text: "I'll run the dangerous tool" },
+              {
+                type: "tool-call",
+                toolCallId: "call_123",
+                toolName: "dangerousTool",
+                input: { action: "delete" },
+                args: { action: "delete" },
+              },
+              {
+                type: "tool-approval-request",
+                toolCallId: "call_123",
+                approvalId: "approval_456",
+              },
+            ],
+          },
+          order: 1,
+        } as MessageDoc,
+        {
+          _id: "2",
+          message: {
+            role: "tool",
+            content: [
+              {
+                type: "tool-approval-response",
+                approvalId: "approval_456",
+                approved: true,
+              },
+            ],
+          },
+          order: 2,
+        } as MessageDoc,
+      ];
+
+      const result = filterOutOrphanedToolMessages(messages);
+      expect(result).toHaveLength(2);
+      // The assistant message should still contain the tool-call
+      expect(result[0]._id).toBe("1");
+      const assistantContent = result[0].message?.content;
+      expect(Array.isArray(assistantContent)).toBe(true);
+      if (Array.isArray(assistantContent)) {
+        const toolCall = assistantContent.find((p) => p.type === "tool-call");
+        expect(toolCall).toBeDefined();
+        expect(toolCall?.toolCallId).toBe("call_123");
+      }
+      // The tool message with approval response should be kept
+      expect(result[1]._id).toBe("2");
+    });
+
+    it("should filter out tool calls with approval request but NO approval response", () => {
+      const messages: MessageDoc[] = [
+        {
+          _id: "1",
+          message: {
+            role: "assistant",
+            content: [
+              { type: "text", text: "I'll run the dangerous tool" },
+              {
+                type: "tool-call",
+                toolCallId: "call_123",
+                toolName: "dangerousTool",
+                input: { action: "delete" },
+                args: { action: "delete" },
+              },
+              {
+                type: "tool-approval-request",
+                toolCallId: "call_123",
+                approvalId: "approval_456",
+              },
+            ],
+          },
+          order: 1,
+        } as MessageDoc,
+        // No approval response provided
+      ];
+
+      const result = filterOutOrphanedToolMessages(messages);
+      expect(result).toHaveLength(1);
+      // The assistant message should have the tool-call filtered out
+      const assistantContent = result[0].message?.content;
+      expect(Array.isArray(assistantContent)).toBe(true);
+      if (Array.isArray(assistantContent)) {
+        // Text and approval-request should remain, but tool-call should be filtered
+        expect(assistantContent).toHaveLength(2);
+        expect(assistantContent.find((p) => p.type === "text")).toBeDefined();
+        expect(
+          assistantContent.find((p) => p.type === "tool-approval-request"),
+        ).toBeDefined();
+        expect(
+          assistantContent.find((p) => p.type === "tool-call"),
+        ).toBeUndefined();
+      }
+    });
+
+    it("should handle mix of tool calls with results and with approvals", () => {
+      const messages: MessageDoc[] = [
+        {
+          _id: "1",
+          message: {
+            role: "assistant",
+            content: [
+              {
+                type: "tool-call",
+                toolCallId: "call_with_result",
+                toolName: "safeTool",
+                input: {},
+                args: {},
+              },
+              {
+                type: "tool-call",
+                toolCallId: "call_with_approval",
+                toolName: "dangerousTool",
+                input: {},
+                args: {},
+              },
+              {
+                type: "tool-approval-request",
+                toolCallId: "call_with_approval",
+                approvalId: "approval_789",
+              },
+            ],
+          },
+          order: 1,
+        } as MessageDoc,
+        {
+          _id: "2",
+          message: {
+            role: "tool",
+            content: [
+              {
+                type: "tool-result",
+                toolCallId: "call_with_result",
+                result: "success",
+              },
+              {
+                type: "tool-approval-response",
+                approvalId: "approval_789",
+                approved: true,
+              },
+            ],
+          },
+          order: 2,
+        } as MessageDoc,
+      ];
+
+      const result = filterOutOrphanedToolMessages(messages);
+      expect(result).toHaveLength(2);
+      // Both tool calls should be kept
+      const assistantContent = result[0].message?.content;
+      expect(Array.isArray(assistantContent)).toBe(true);
+      if (Array.isArray(assistantContent)) {
+        const toolCalls = assistantContent.filter(
+          (p) => p.type === "tool-call",
+        );
+        expect(toolCalls).toHaveLength(2);
+      }
+    });
   });
 
   describe("fetchContextMessages", () => {

package/src/client/search.ts

@@ -36,10 +36,20 @@ const DEFAULT_VECTOR_SCORE_THRESHOLD = 0.0;
 // the 8k token limit for some models.
 const MAX_EMBEDDING_TEXT_LENGTH = 10_000;
 
-export type GetEmbedding = (text: string) => Promise<{
-  embedding: number[];
-  textEmbeddingModel: string | EmbeddingModel<string>;
-}>;
+export type GetEmbedding = (text: string) => Promise<
+  | {
+      embedding: number[];
+      /** @deprecated Use embeddingModel instead. */
+      textEmbeddingModel: string | EmbeddingModel;
+      embeddingModel?: string | EmbeddingModel;
+    }
+  | {
+      embedding: number[];
+      /** @deprecated Use embeddingModel instead. */
+      textEmbeddingModel?: string | EmbeddingModel;
+      embeddingModel: string | EmbeddingModel;
+    }
+>;
 
 /**
  * Fetch the context messages for a thread.
@@ -178,8 +188,10 @@ export async function fetchRecentAndSearchMessages(
       if (!embedding && args.getEmbedding) {
         const embeddingFields = await args.getEmbedding(text);
         embedding = embeddingFields.embedding;
-        embeddingModel = embeddingFields.textEmbeddingModel
-          ? getModelName(embeddingFields.textEmbeddingModel)
+        const effectiveModel =
+          embeddingFields.embeddingModel ?? embeddingFields.textEmbeddingModel;
+        embeddingModel = effectiveModel
+          ? getModelName(effectiveModel)
           : undefined;
         // TODO: if the text matches the target message, save the embedding
         // for the target message and return the embeddingId on the message.
@@ -225,12 +237,19 @@ export async function fetchRecentAndSearchMessages(
 
 /**
  * Filter out tool messages that don't have both a tool call and response.
+ * For the approval workflow, tool calls with approval responses (but no tool-results yet)
+ * should also be kept.
  * @param docs The messages to filter.
  * @returns The filtered messages.
  */
 export function filterOutOrphanedToolMessages(docs: MessageDoc[]) {
   const toolCallIds = new Set<string>();
   const toolResultIds = new Set<string>();
+  // Track approval workflow: toolCallId → approvalId
+  const approvalRequestsByToolCallId = new Map<string, string>();
+  // Track which approvalIds have responses
+  const approvalResponseIds = new Set<string>();
+
   const result: MessageDoc[] = [];
   for (const doc of docs) {
     if (doc.message && Array.isArray(doc.message.content)) {
@@ -239,17 +258,43 @@ export function filterOutOrphanedToolMessages(docs: MessageDoc[]) {
           toolCallIds.add(content.toolCallId);
         } else if (content.type === "tool-result") {
           toolResultIds.add(content.toolCallId);
+        } else if (content.type === "tool-approval-request") {
+          const approvalRequest = content as {
+            type: "tool-approval-request";
+            toolCallId: string;
+            approvalId: string;
+          };
+          approvalRequestsByToolCallId.set(
+            approvalRequest.toolCallId,
+            approvalRequest.approvalId,
+          );
+        } else if (content.type === "tool-approval-response") {
+          const approvalResponse = content as {
+            type: "tool-approval-response";
+            approvalId: string;
+          };
+          approvalResponseIds.add(approvalResponse.approvalId);
         }
       }
     }
   }
+
+  // Helper: check if tool call has a corresponding approval response
+  const hasApprovalResponse = (toolCallId: string) => {
+    const approvalId = approvalRequestsByToolCallId.get(toolCallId);
+    return approvalId !== undefined && approvalResponseIds.has(approvalId);
+  };
+
   for (const doc of docs) {
     if (
       doc.message?.role === "assistant" &&
       Array.isArray(doc.message.content)
     ) {
       const content = doc.message.content.filter(
-        (p) => p.type !== "tool-call" || toolResultIds.has(p.toolCallId),
+        (p) =>
+          p.type !== "tool-call" ||
+          toolResultIds.has(p.toolCallId) ||
+          hasApprovalResponse(p.toolCallId),
       );
       if (content.length) {
         result.push({
@@ -261,9 +306,14 @@ export function filterOutOrphanedToolMessages(docs: MessageDoc[]) {
         });
       }
     } else if (doc.message?.role === "tool") {
-      const content = doc.message.content.filter((c) =>
-        toolCallIds.has(c.toolCallId),
-      );
+      const content = doc.message.content.filter((c) => {
+        // tool-result parts have toolCallId
+        if (c.type === "tool-result") {
+          return toolCallIds.has(c.toolCallId);
+        }
+        // tool-approval-response parts don't have toolCallId, so include them
+        return true;
+      });
       if (content.length) {
         result.push({
           ...doc,
@@ -294,7 +344,10 @@ export async function embedMessages(
     userId: string | undefined;
     threadId: string | undefined;
     agentName?: string;
-  } & Pick<Config, "usageHandler" | "textEmbeddingModel" | "callSettings">,
+  } & Pick<
+    Config,
+    "usageHandler" | "textEmbeddingModel" | "embeddingModel" | "callSettings"
+  >,
   messages: (ModelMessage | Message)[],
 ): Promise<
   | {
@@ -304,7 +357,9 @@ export async function embedMessages(
     }
   | undefined
 > {
-  if (!options.textEmbeddingModel) {
+  const textEmbeddingModel =
+    options.embeddingModel ?? options.textEmbeddingModel;
+  if (!textEmbeddingModel) {
     return undefined;
   }
   let embeddings:
@@ -340,7 +395,7 @@ export async function embedMessages(
   if (textEmbeddings.embeddings.length > 0) {
     const dimension = textEmbeddings.embeddings[0].length;
     validateVectorDimension(dimension);
-    const model = getModelName(options.textEmbeddingModel);
+    const model = getModelName(textEmbeddingModel);
     embeddings = { vectors: embeddingsOrNull, dimension, model };
   }
   return embeddings;
@@ -355,7 +410,19 @@ export async function embedMessages(
  */
 export async function embedMany(
   ctx: ActionCtx,
-  {
+  args: {
+    userId: string | undefined;
+    threadId: string | undefined;
+    values: string[];
+    abortSignal?: AbortSignal;
+    headers?: Record<string, string>;
+    agentName?: string;
+  } & Pick<
+    Config,
+    "usageHandler" | "textEmbeddingModel" | "embeddingModel" | "callSettings"
+  >,
+): Promise<{ embeddings: number[][] }> {
+  const {
     userId,
     threadId,
     values,
@@ -364,24 +431,17 @@ export async function embedMany(
     agentName,
     usageHandler,
     textEmbeddingModel,
+    embeddingModel,
     callSettings,
-  }: {
-    userId: string | undefined;
-    threadId: string | undefined;
-    values: string[];
-    abortSignal?: AbortSignal;
-    headers?: Record<string, string>;
-    agentName?: string;
-  } & Pick<Config, "usageHandler" | "textEmbeddingModel" | "callSettings">,
-): Promise<{ embeddings: number[][] }> {
-  const embeddingModel = textEmbeddingModel;
+  } = args;
+  const effectiveEmbeddingModel = embeddingModel ?? textEmbeddingModel;
   assert(
-    embeddingModel,
-    "a textEmbeddingModel is required to be set for vector search",
+    effectiveEmbeddingModel,
+    "an embeddingModel (or textEmbeddingModel) is required to be set for vector search",
   );
   const result = await embedMany_({
     ...callSettings,
-    model: embeddingModel,
+    model: effectiveEmbeddingModel,
     values,
     abortSignal,
     headers,
@@ -391,13 +451,24 @@ export async function embedMany(
       userId,
       threadId,
       agentName,
-      model: getModelName(embeddingModel),
-      provider: getProviderName(embeddingModel),
+      model: getModelName(effectiveEmbeddingModel),
+      provider: getProviderName(effectiveEmbeddingModel),
       providerMetadata: undefined,
       usage: {
         inputTokens: result.usage.tokens,
         outputTokens: 0,
         totalTokens: result.usage.tokens,
+        // These detail fields are required by LanguageModelUsage type but we don't
+        // have the granular data, so we provide objects with undefined values.
+        inputTokenDetails: {
+          cacheReadTokens: undefined,
+          cacheWriteTokens: undefined,
+          noCacheTokens: undefined,
+        },
+        outputTokenDetails: {
+          textTokens: undefined,
+          reasoningTokens: undefined,
+        },
       },
     });
   }
@@ -418,17 +489,28 @@ export async function generateAndSaveEmbeddings(
     threadId: string | undefined;
     userId: string | undefined;
     agentName?: string;
-    textEmbeddingModel: EmbeddingModel<string>;
+    /**
+     * @deprecated Use embeddingModel instead.
+     */
+    textEmbeddingModel?: EmbeddingModel;
+    embeddingModel?: EmbeddingModel;
   } & Pick<Config, "usageHandler" | "callSettings">,
   messages: MessageDoc[],
 ) {
+  const effectiveEmbeddingModel =
+    args.embeddingModel ?? args.textEmbeddingModel;
+  if (!effectiveEmbeddingModel) {
+    throw new Error(
+      "an embeddingModel (or textEmbeddingModel) is required to generate and save embeddings",
+    );
+  }
   const toEmbed = messages.filter((m) => !m.embeddingId && m.message);
   if (toEmbed.length === 0) {
     return;
   }
   const embeddings = await embedMessages(
     ctx,
-    args,
+    { ...args, embeddingModel: effectiveEmbeddingModel },
     toEmbed.map((m) => m.message!),
   );
   if (embeddings && embeddings.vectors.some((v) => v !== null)) {
@@ -473,7 +555,8 @@ export async function fetchContextWithPrompt(
   order: number | undefined;
   stepOrder: number | undefined;
 }> {
-  const { threadId, userId, textEmbeddingModel } = args;
+  const { threadId, userId, textEmbeddingModel, embeddingModel } = args;
+  const effectiveEmbeddingModel = embeddingModel ?? textEmbeddingModel;
 
   const promptArray = getPromptArray(args.prompt);
 
@@ -496,8 +579,8 @@ export async function fetchContextWithPrompt(
       contextOptions: args.contextOptions ?? {},
       getEmbedding: async (text) => {
         assert(
-          textEmbeddingModel,
-          "A textEmbeddingModel is required to be set on the Agent that you're doing vector search with",
+          effectiveEmbeddingModel,
+          "An embeddingModel (or textEmbeddingModel) is required to be set on the Agent that you're doing vector search with",
         );
         return {
           embedding: (
@@ -505,10 +588,10 @@ export async function fetchContextWithPrompt(
               ...args,
               userId,
               values: [text],
-              textEmbeddingModel,
+              embeddingModel: effectiveEmbeddingModel,
             })
           ).embeddings[0],
-          textEmbeddingModel,
+          embeddingModel: effectiveEmbeddingModel,
         };
       },
     },
@@ -531,7 +614,7 @@ export async function fetchContextWithPrompt(
         promptArray.push(promptMessage.message);
       }
     }
-    if (!promptMessage.embeddingId && textEmbeddingModel) {
+    if (!promptMessage.embeddingId && effectiveEmbeddingModel) {
       // Lazily generate embeddings for the prompt message, if it doesn't have
       // embeddings yet. This can happen if the message was saved in a mutation
       // where the LLM is not available.
@@ -541,7 +624,7 @@ export async function fetchContextWithPrompt(
         {
           ...args,
           userId,
-          textEmbeddingModel,
+          embeddingModel: effectiveEmbeddingModel,
         },
         [promptMessage],
       );