@convex-dev/agent 0.1.18-alpha.1 → 0.5.0-alpha.1

package/src/client/index.ts

@@ -1,7 +1,7 @@
-import type { EmbeddingModelV1, LanguageModelV1 } from "@ai-sdk/provider";
+import type { LanguageModelV2, EmbeddingModelV2 } from "@ai-sdk/provider";
 import type {
   AssistantContent,
-  CoreMessage,
+  ModelMessage,
   DeepPartial,
   FilePart,
   GenerateObjectResult,
@@ -12,11 +12,15 @@ import type {
   StreamTextResult,
   ToolSet,
   UserContent,
+  StopCondition,
+  Schema,
+  ToolChoice,
 } from "ai";
 import {
   embedMany,
   generateObject,
   generateText,
+  stepCountIs,
   streamObject,
   streamText,
 } from "ai";
@@ -38,15 +42,14 @@ import {
   type VectorDimension,
 } from "../component/vector/tables.js";
 import {
-  type AIMessageWithoutId,
   deserializeMessage,
-  promptOrMessagesToCoreMessages,
   serializeMessage,
   serializeNewMessagesInStep,
   serializeObjectResult,
 } from "../mapping.js";
 import { extractText, isTool } from "../shared.js";
 import {
+  type Message,
   type MessageEmbeddings,
   type MessageStatus,
   type MessageWithMetadata,
@@ -57,7 +60,7 @@ import {
   vSafeObjectArgs,
   vTextArgs,
 } from "../validators.js";
-import { createTool, wrapTools } from "./createTool.js";
+import { createTool, wrapTools, type ToolCtx } from "./createTool.js";
 import { listMessages } from "./listMessages.js";
 import { fetchContextMessages } from "./search.js";
 import {
@@ -86,9 +89,15 @@ import type {
   UsageHandler,
   UserActionCtx,
 } from "./types.js";
+import type z from "zod";
 
+export { stepCountIs } from "ai";
 export { vMessageDoc, vThreadDoc } from "../component/schema.js";
-export { serializeDataOrUrl } from "../mapping.js";
+export {
+  serializeDataOrUrl,
+  deserializeMessage,
+  serializeMessage,
+} from "../mapping.js";
 // NOTE: these are also exported via @convex-dev/agent/validators
 // a future version may put them all here or move these over there
 export {
@@ -106,16 +115,12 @@ export {
 } from "../validators.js";
 export type { ToolCtx } from "./createTool.js";
 export { getFile, storeFile } from "./files.js";
-export { filterOutOrphanedToolMessages } from "./search.js";
-export { abortStream, listStreams } from "./streaming.js";
 export {
-  createTool,
-  extractText,
+  filterOutOrphanedToolMessages,
   fetchContextMessages,
-  isTool,
-  listMessages,
-  syncStreams,
-};
+} from "./search.js";
+export { abortStream, listStreams, syncStreams } from "./streaming.js";
+export { createTool, extractText, isTool, listMessages };
 export {
   definePlaygroundAPI,
   type PlaygroundAPI,
@@ -175,7 +180,7 @@ export class Agent<
        * const myAgent = new Agent(components.agent, {
        *   chat: openai.chat("gpt-4o-mini"),
        */
-      chat: LanguageModelV1;
+      chat: LanguageModelV2;
       /**
        * The model to use for text embeddings. Optional.
        * If specified, it will use this for generating vector embeddings
@@ -186,7 +191,7 @@ export class Agent<
        * const myAgent = new Agent(components.agent, {
        *   textEmbedding: openai.embedding("text-embedding-3-small")
        */
-      textEmbedding?: EmbeddingModelV1<string>;
+      textEmbedding?: EmbeddingModelV2<string>;
       /**
        * The default system prompt to put in each request.
        * Override per-prompt by passing the "system" parameter.
@@ -212,9 +217,9 @@ export class Agent<
       storageOptions?: StorageOptions;
       /**
        * When generating or streaming text with tools available, this
-       * determines the default max number of iterations.
+       * determines when to stop. Defaults to stepCountIs(1).
        */
-      maxSteps?: number;
+      stopWhen?: StopCondition<AgentTools> | Array<StopCondition<AgentTools>>;
       /**
        * The maximum number of calls to make to an LLM in case it fails.
        * This can be overridden at each generate/stream callsite.
@@ -471,16 +476,17 @@ export class Agent<
       threadId,
       messageId,
       agent: this,
-    };
+    } satisfies ToolCtx;
+    type Tools = TOOLS extends undefined ? AgentTools : TOOLS;
     const tools = wrapTools(
       toolCtx,
       args.tools ?? threadTools ?? this.options.tools,
-    ) as TOOLS extends undefined ? AgentTools : TOOLS;
+    ) as Tools;
     const saveOutput = opts.storageOptions?.saveMessages !== "none";
     try {
-      const result = (await generateText({
+      const result = (await generateText<Tools, OUTPUT, OUTPUT_PARTIAL>({
         // Can be overridden
-        maxSteps: this.options.maxSteps,
+        stopWhen: this.options.stopWhen as StopCondition<Tools> | undefined,
         ...aiArgs,
         tools,
         onStepFinish: async (step) => {
@@ -516,11 +522,7 @@ export class Agent<
           }
           return args.onStepFinish?.(step);
         },
-      })) as GenerateTextResult<
-        TOOLS extends undefined ? AgentTools : TOOLS,
-        OUTPUT
-      > &
-        GenerationOutputMetadata;
+      })) as GenerateTextResult<Tools, OUTPUT> & GenerationOutputMetadata;
       result.messageId = messageId;
       result.order = order;
       return result;
@@ -624,8 +626,8 @@ export class Agent<
         : undefined;
 
     const result = streamText({
-      // Can be overridden
-      maxSteps: this.options.maxSteps,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      stopWhen: this.options.stopWhen as any, // Can be overridden
       ...aiArgs,
       tools,
       abortSignal: streamer?.abortController.signal ?? aiArgs.abortSignal,
@@ -787,7 +789,7 @@ export class Agent<
    * Use {@link continueThread} to get a version of this function already scoped
    * to a thread (and optionally userId).
    */
-  async streamObject<T>(
+  async streamObject<T extends z.Schema | Schema>(
     ctx: ActionCtx,
     {
       userId: argsUserId,
@@ -819,7 +821,8 @@ export class Agent<
     });
     const { args: aiArgs, messageId, order, userId } = context;
     const saveOutput = opts.storageOptions?.saveMessages !== "none";
-    const stream = streamObject<T>({
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const stream = streamObject<any>({
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       ...(aiArgs as any),
       onError: async (error) => {
@@ -840,9 +843,6 @@ export class Agent<
               request: await stream.request,
               response: result.response,
               providerMetadata: result.providerMetadata,
-              experimental_providerMetadata:
-                result.experimental_providerMetadata,
-              logprobs: undefined,
               toJsonResponse: stream.toTextStreamResponse,
             },
             model: aiArgs.model.modelId,
@@ -1037,7 +1037,7 @@ export class Agent<
     args: {
       userId: string | undefined;
       threadId: string | undefined;
-      messages: CoreMessage[];
+      messages: (ModelMessage | Message)[];
       /**
        * If provided, it will search for messages up to and including this message.
        * Note: if this is far in the past, text and vector search results may be more
@@ -1125,7 +1125,7 @@ export class Agent<
       userId: string | undefined;
       threadId: string | undefined;
     },
-    messages: CoreMessage[],
+    messages: (ModelMessage | Message)[],
   ) {
     if (!this.options.textEmbedding) {
       return undefined;
@@ -1151,7 +1151,6 @@ export class Agent<
       threadId,
       values: messageTexts as string[],
     });
-    // TODO: record usage of embeddings
     // Then assemble the embeddings into a single array with nulls for the messages without text.
     const embeddingsOrNull = Array(messages.length).fill(null);
     textIndexes.forEach((i, j) => {
@@ -1218,7 +1217,7 @@ export class Agent<
         userId: messagesMissingEmbeddings[0]!.userId,
         threadId: messagesMissingEmbeddings[0]!.threadId,
       },
-      messagesMissingEmbeddings.map((m) => m!.message!),
+      messagesMissingEmbeddings.map((m) => deserializeMessage(m!.message!)),
     );
     if (!embeddings) {
       if (!this.options.textEmbedding) {
@@ -1291,7 +1290,7 @@ export class Agent<
     const embeddings = await this.generateEmbeddings(
       ctx,
       { userId: args.userId, threadId: args.threadId },
-      messages.map((m) => m.message),
+      messages.map((m) => deserializeMessage(m.message)),
     );
     const saved = await ctx.runMutation(this.component.messages.addMessages, {
       userId: args.userId,
@@ -1336,7 +1335,7 @@ export class Agent<
     const embeddings = await this.generateEmbeddings(
       ctx,
       { userId: args.userId, threadId: args.threadId },
-      messages.map((m) => m.message),
+      messages.map((m) => deserializeMessage(m.message)),
     );
 
     await ctx.runMutation(this.component.messages.addMessages, {
@@ -1392,7 +1391,7 @@ export class Agent<
       messageId: string;
       patch: {
         /** The message to replace the existing message. */
-        message: CoreMessage & { id?: string };
+        message: (ModelMessage & { id?: string }) | Message;
         /** The status to set on the message. */
         status: "success" | "error";
         /** The error message to set on the message. */
@@ -1507,8 +1506,8 @@ export class Agent<
       endOrder: number;
       endStepOrder?: number;
     },
-  ): Promise<void> {
-    await ctx.runMutation(this.component.messages.deleteByOrder, {
+  ): Promise<{ isDone: boolean; lastOrder?: number; lastStepOrder?: number }> {
+    return ctx.runMutation(this.component.messages.deleteByOrder, {
       threadId: args.threadId,
       startOrder: args.startOrder,
       startStepOrder: args.startStepOrder,
@@ -1560,11 +1559,11 @@ export class Agent<
   async _saveMessagesAndFetchContext<
     T extends {
       id?: string;
-      prompt?: string;
-      messages?: CoreMessage[] | AIMessageWithoutId[];
+      prompt?: string | (ModelMessage | Message)[];
+      messages?: (ModelMessage | Message)[];
       system?: string;
       promptMessageId?: string;
-      model?: LanguageModelV1;
+      model?: LanguageModelV2;
       maxRetries?: number;
     },
   >(
@@ -1580,14 +1579,19 @@ export class Agent<
       threadId: string | undefined;
     } & Options,
   ): Promise<{
-    args: T & { model: LanguageModelV1 };
+    args: T & { model: LanguageModelV2 };
     userId: string | undefined;
     messageId: string | undefined;
     order: number | undefined;
     stepOrder: number | undefined;
   }> {
     // If only a promptMessageId is provided, this will be empty.
-    const messages = promptOrMessagesToCoreMessages(args);
+    const messages = args.messages ?? [];
+    const prompt: (ModelMessage | Message)[] = !args.prompt
+      ? []
+      : Array.isArray(args.prompt)
+        ? args.prompt
+        : [{ role: "user", content: args.prompt }];
     const userId =
       argsUserId ??
       (threadId &&
@@ -1603,45 +1607,46 @@ export class Agent<
     });
     // If it was a promptMessageId, pop it off context messages
     // and add to the end of messages.
-    // TODO: slice it from the prompt message, to append all of them
+    const promptMessageIndex = args.promptMessageId
+      ? contextMessages.findIndex((m) => m._id === args.promptMessageId)
+      : -1;
     const promptMessage =
-      !!args.promptMessageId &&
-      contextMessages.at(-1)?._id === args.promptMessageId
-        ? contextMessages.pop()
+      promptMessageIndex !== -1
+        ? contextMessages.splice(promptMessageIndex, 1)[0]
         : undefined;
-    if (promptMessage && args.prompt) {
-      // If they specify both a promptMessageId and a prompt, we prefer
-      // the prompt to stand in for the promptMessageId message.
-      promptMessage.message = { role: "user", content: args.prompt };
-    }
+
     let messageId = promptMessage?._id;
     let order = promptMessage?.order;
     let stepOrder = promptMessage?.stepOrder;
     if (
       threadId &&
-      messages.length &&
+      messages.length + prompt.length &&
       storageOptions?.saveMessages !== "none" &&
       // If it was a promptMessageId, we don't want to save it again.
       (!args.promptMessageId || storageOptions?.saveMessages === "all")
     ) {
       const saveAll = storageOptions?.saveMessages === "all";
-      const coreMessages = saveAll ? messages : messages.slice(-1);
+      const coreMessages = [...messages, ...prompt];
+      const toSave = saveAll ? coreMessages : coreMessages.slice(-1);
+      const metadata = Array.from({ length: toSave.length }, () => ({}));
+      metadata[metadata.length - 1] = { id: args.id };
       const saved = await this.saveMessages(ctx, {
         threadId,
         userId,
-        messages: coreMessages,
-        metadata: coreMessages.map((_, i) =>
-          i === coreMessages.length - 1 ? { id: args.id } : {},
-        ),
+        messages: toSave,
+        metadata,
         failPendingSteps: true,
       });
       messageId = saved.lastMessageId;
       order = saved.messages.at(-1)?.order;
       stepOrder = saved.messages.at(-1)?.stepOrder;
     }
+
     if (promptMessage?.message) {
-      // Add the message after saving the messages, so it's not saved again.
-      messages.push(deserializeMessage(promptMessage.message));
+      if (!args.prompt) {
+        // If they override the prompt, we skip the existing prompt message.
+        messages.push(deserializeMessage(promptMessage.message));
+      }
       // Lazily generate embeddings for the prompt message, if it doesn't have
       // embeddings yet. This can happen if the message was saved in a mutation
       // where the LLM is not available.
@@ -1650,10 +1655,28 @@ export class Agent<
       }
     }
 
+    const prePrompt = contextMessages.map((m) =>
+      deserializeMessage(m.message!),
+    );
+    let existingResponses: ModelMessage[] = [];
+    if (promptMessageIndex !== -1) {
+      // pull any messages that already responded to the prompt off
+      // and add them after the prompt
+      existingResponses = prePrompt.splice(promptMessageIndex);
+    }
+
     let processedMessages = [
-      ...contextMessages.map((m) => deserializeMessage(m.message!)),
+      ...prePrompt,
       ...messages,
+      ...prompt,
+      ...existingResponses,
     ];
+    if (promptMessageIndex === -1) {
+      processedMessages.push(...prompt);
+    } else {
+      // We add the prompt where the prompt message was
+      processedMessages.splice(promptMessageIndex, 0, ...prompt);
+    }
 
     // Process messages to inline localhost files (if not, file urls pointing to localhost will be sent to LLM providers)
     if (process.env.CONVEX_CLOUD_URL?.startsWith("http://127.0.0.1")) {
@@ -1668,7 +1691,7 @@ export class Agent<
         model: model ?? this.options.chat,
         system: args.system ?? this.options.instructions,
         messages: processedMessages,
-      } as T & { model: LanguageModelV1 },
+      } as T & { model: LanguageModelV2 },
       userId,
       messageId,
       order,
@@ -1707,8 +1730,8 @@ export class Agent<
         provider: embeddingModel.provider,
         providerMetadata: undefined,
         usage: {
-          promptTokens: result.usage.tokens,
-          completionTokens: 0,
+          inputTokens: result.usage.tokens,
+          outputTokens: 0,
           totalTokens: result.usage.tokens,
         },
       });
@@ -1722,11 +1745,11 @@ export class Agent<
    * able to access localhost URLs.
    */
   private async _inlineMessagesFiles(
-    messages: CoreMessage[],
-  ): Promise<CoreMessage[]> {
+    messages: (ModelMessage | Message)[],
+  ): Promise<(ModelMessage | Message)[]> {
     // Process each message to convert localhost URLs to base64
     return Promise.all(
-      messages.map(async (message): Promise<CoreMessage> => {
+      messages.map(async (message): Promise<ModelMessage | Message> => {
         if (
           (message.role !== "user" && message.role !== "assistant") ||
           typeof message.content === "string" ||
@@ -1845,15 +1868,15 @@ export class Agent<
    * Create an action out of this agent so you can call it from workflows or other actions
    * without a wrapping function.
    * @param spec Configuration for the agent acting as an action, including
-   *   {@link ContextOptions}, {@link StorageOptions}, and maxSteps.
+   *   {@link ContextOptions}, {@link StorageOptions}, and {@link stopWhen}.
    */
   asTextAction<DataModel extends GenericDataModel>(
     spec?: {
       /**
-       * The maximum number of steps to take in this action.
-       * Defaults to the {@link Agent.maxSteps} option.
+       * When to stop generating text.
+       * Defaults to the {@link Agent["options"].stopWhen} option.
        */
-      maxSteps?: number;
+      stopWhen?: StopCondition<AgentTools> | Array<StopCondition<AgentTools>>;
       /**
        * The {@link ContextOptions} to use for fetching contextual messages and
        * saving input/output messages.
@@ -1902,7 +1925,7 @@ export class Agent<
         }
       : { customCtx?: never }),
   ) {
-    const maxSteps = spec?.maxSteps ?? this.options.maxSteps;
+    const stopWhen = spec?.stopWhen ?? this.options.stopWhen;
     return internalActionGeneric({
       args: vTextArgs,
       handler: async (ctx_, args) => {
@@ -1910,9 +1933,17 @@ export class Agent<
           args.stream === true ? spec?.stream || true : spec?.stream ?? false;
         const targetArgs = { userId: args.userId, threadId: args.threadId };
         const llmArgs = {
-          maxSteps,
+          stopWhen,
           ...omit(args, ["storageOptions", "contextOptions"]),
-        };
+          messages: args.messages?.map(deserializeMessage),
+          prompt: Array.isArray(args.prompt)
+            ? args.prompt.map(deserializeMessage)
+            : args.prompt,
+          toolChoice: args.toolChoice as ToolChoice<AgentTools>,
+        } satisfies StreamingTextArgs<AgentTools>;
+        if (args.maxSteps) {
+          llmArgs.stopWhen = stepCountIs(args.maxSteps);
+        }
         const opts = {
           ...this.options,
           ...(spec && pick(spec, ["contextOptions", "storageOptions"])),
@@ -1925,7 +1956,13 @@ export class Agent<
             : ctx_
         ) as UserActionCtx & CustomCtx;
         if (stream) {
-          const result = await this.streamText(ctx, targetArgs, llmArgs, opts);
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const result = await this.streamText<any>(
+            ctx,
+            targetArgs,
+            llmArgs,
+            opts,
+          );
           await result.consumeStream();
           return {
             text: await result.text,
@@ -1935,7 +1972,13 @@ export class Agent<
             warnings: result.warnings,
           };
         } else {
-          const res = await this.generateText(ctx, targetArgs, llmArgs, opts);
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          const res = await this.generateText<any>(
+            ctx,
+            targetArgs,
+            llmArgs,
+            opts,
+          );
           return {
             text: res.text,
             messageId: res.messageId,
@@ -1952,28 +1995,27 @@ export class Agent<
    * it from workflows or other actions without a wrapping function.
    * @param spec Configuration for the agent acting as an action, including
    * the normal parameters to {@link generateObject}, plus {@link ContextOptions}
-   * and maxSteps.
+   * and stopWhen.
    */
   asObjectAction<T>(
-    spec: OurObjectArgs<T> & { maxSteps?: number },
+    spec: OurObjectArgs<T>,
     options?: {
       contextOptions?: ContextOptions;
       storageOptions?: StorageOptions;
     },
   ) {
-    const maxSteps = spec?.maxSteps ?? this.options.maxSteps;
     return internalActionGeneric({
       args: vSafeObjectArgs,
       handler: async (ctx, args) => {
-        const overrides = pick(args, ["userId", "threadId"]);
+        const { userId, threadId, ...rest } = args;
+        const overrides = pick(rest, ["contextOptions", "storageOptions"]);
         const value = await this.generateObject(
           ctx,
-          { userId: args.userId, threadId: args.threadId },
+          { userId, threadId },
           {
             ...spec,
-            maxSteps,
-            ...omit(args, ["userId", "threadId"]),
-          } as unknown as OurObjectArgs<unknown>,
+            ...omit(rest, ["contextOptions", "storageOptions"]),
+          } as OurObjectArgs<unknown>,
           { ...this.options, ...options, ...overrides },
         );
         return {
@@ -2020,7 +2062,7 @@ export class Agent<
       handler: async (ctx, args) => {
         const { lastMessageId, messages } = await this.saveMessages(ctx, {
           ...args,
-          messages: args.messages.map((m) => m.message),
+          messages: args.messages.map((m) => deserializeMessage(m.message)),
           metadata: args.messages.map(({ message: _, ...m }) => m),
         });
         return {
@@ -2032,8 +2074,6 @@ export class Agent<
   }
 }
 
-type CoreMessageMaybeWithId = CoreMessage & { id?: string | undefined };
-
 /**
  * Create a thread to store messages with an Agent.
  * @param ctx The context from a mutation or action.
@@ -2092,7 +2132,7 @@ type SaveMessagesArgs = {
   /**
    * The messages to save.
    */
-  messages: CoreMessageMaybeWithId[];
+  messages: ((ModelMessage & { id?: string | undefined }) | Message)[];
   /**
    * Metadata to save with the messages. Each element corresponds to the
    * message at the same index.
@@ -2186,7 +2226,7 @@ type SaveMessageArgs = {
       /**
        * The message to save.
        */
-      message: CoreMessage;
+      message: ModelMessage | Message;
     }
   | {
       /*

package/src/client/search.ts

@@ -5,13 +5,14 @@ import type {
   RunQueryCtx,
 } from "./types.js";
 import type { MessageDoc } from "../component/schema.js";
-import type { CoreMessage } from "ai";
+import type { ModelMessage } from "ai";
 import { assert } from "convex-helpers";
 import {
   DEFAULT_MESSAGE_RANGE,
   DEFAULT_RECENT_MESSAGES,
   extractText,
 } from "../shared.js";
+import type { Message } from "../validators.js";
 
 const DEFAULT_VECTOR_SCORE_THRESHOLD = 0.0;
 
@@ -34,7 +35,7 @@ export async function fetchContextMessages(
   args: {
     userId: string | undefined;
     threadId: string | undefined;
-    messages: CoreMessage[];
+    messages: (ModelMessage | Message)[];
     /**
      * If provided, it will search for messages up to and including this message.
      * Note: if this is far in the past, text and vector search results may be more

package/src/client/streaming.ts

@@ -9,7 +9,7 @@ import type {
   StreamArgs,
   StreamDelta,
   StreamMessage,
-  TextStreamPart,
+  vTextStreamPartV5,
 } from "../validators.js";
 import type { MessageDoc } from "../component/schema.js";
 import type {
@@ -20,6 +20,7 @@ import type {
   SyncStreamsReturnValue,
 } from "./types.js";
 import { omit } from "convex-helpers";
+import type { Infer } from "convex/values";
 
 /**
  * A function that handles fetching stream deltas, used with the React hooks
@@ -158,7 +159,7 @@ export function mergeTransforms<TOOLS extends ToolSet>(
 export class DeltaStreamer {
   public streamId: string | undefined;
   public readonly options: Required<StreamingOptions>;
-  #nextParts: TextStreamPart[] = [];
+  #nextParts: Infer<typeof vTextStreamPartV5>[] = [];
   #nextOrder: number;
   #nextStepOrder: number;
   #latestWrite: number = 0;
@@ -205,7 +206,7 @@ export class DeltaStreamer {
       });
     }
   }
-  public async addParts(parts: TextStreamPart[]) {
+  public async addParts(parts: Infer<typeof vTextStreamPartV5>[]) {
     if (this.abortController.signal.aborted) {
       return;
     }