@convex-dev/agent 0.2.6-alpha.1 → 0.2.7

package/src/client/start.ts

@@ -0,0 +1,315 @@
+import {
+  stepCountIs,
+  type CallSettings,
+  type GenerateObjectResult,
+  type IdGenerator,
+  type LanguageModel,
+  type ModelMessage,
+  type StepResult,
+  type StopCondition,
+  type ToolSet,
+} from "ai";
+import {
+  serializeNewMessagesInStep,
+  serializeObjectResult,
+} from "../mapping.js";
+import { embedMessages, fetchContextWithPrompt } from "./search.js";
+import type {
+  ActionCtx,
+  AgentComponent,
+  Config,
+  Options,
+  UserActionCtx,
+} from "./types.js";
+import { saveMessages } from "./messages.js";
+import type { Message, MessageDoc } from "../validators.js";
+import {
+  getModelName,
+  getProviderName,
+  type ModelOrMetadata,
+} from "../shared.js";
+import { wrapTools, type ToolCtx } from "./createTool.js";
+import type { Agent } from "./index.js";
+import { omit } from "convex-helpers";
+import { saveInputMessages } from "./saveInputMessages.js";
+
+export async function start<
+  T,
+  Tools extends ToolSet = ToolSet,
+  CustomCtx extends object = object,
+>(
+  ctx: ActionCtx & CustomCtx,
+  component: AgentComponent,
+  /**
+   * These are the arguments you'll pass to the LLM call such as
+   * `generateText` or `streamText`. This function will look up the context
+   * and provide functions to save the steps, abort the generation, and more.
+   * The type of the arguments returned infers from the type of the arguments
+   * you pass here.
+   */
+  args: T & {
+    /**
+     * If provided, this message will be used as the "prompt" for the LLM call,
+     * instead of the prompt or messages.
+     * This is useful if you want to first save a user message, then use it as
+     * the prompt for the LLM call in another call.
+     */
+    promptMessageId?: string;
+    /**
+     * The model to use for the LLM calls. This will override the model specified
+     * in the Agent constructor.
+     */
+    model?: LanguageModel;
+    /**
+     * The tools to use for the tool calls. This will override tools specified
+     * in the Agent constructor or createThread / continueThread.
+     */
+    tools?: Tools;
+    /**
+     * The single prompt message to use for the LLM call. This will be the
+     * last message in the context. If it's a string, it will be a user role.
+     */
+    prompt?: string | (ModelMessage | Message)[];
+    /**
+     * If provided alongside prompt, the ordering will be:
+     * 1. system prompt
+     * 2. search context
+     * 3. recent messages
+     * 4. these messages
+     * 5. prompt messages, including those already on the same `order` as
+     *   the promptMessageId message, if provided.
+     */
+    messages?: (ModelMessage | Message)[];
+    /**
+     * The abort signal to be passed to the LLM call. If triggered, it will
+     * mark the pending message as failed. If the generation is asynchronously
+     * aborted, it will trigger this signal when detected.
+     */
+    abortSignal?: AbortSignal;
+    stopWhen?: StopCondition<Tools> | Array<StopCondition<Tools>>;
+    _internal?: { generateId?: IdGenerator };
+  },
+  {
+    threadId,
+    ...opts
+  }: Options &
+    Config & {
+      userId?: string | null;
+      threadId?: string;
+      languageModel: LanguageModel;
+      agentName: string;
+      agentForToolCtx?: Agent;
+    },
+): Promise<{
+  args: T & {
+    system?: string;
+    model: LanguageModel;
+    messages: ModelMessage[];
+    prompt?: never;
+    tools?: Tools;
+  } & CallSettings;
+  order: number;
+  stepOrder: number;
+  userId: string | undefined;
+  promptMessageId: string | undefined;
+  updateModel: (model: ModelOrMetadata | undefined) => void;
+  save: <TOOLS extends ToolSet>(
+    toSave:
+      | { step: StepResult<TOOLS> }
+      | { object: GenerateObjectResult<unknown> },
+    createPendingMessage?: boolean,
+  ) => Promise<void>;
+  fail: (reason: string) => Promise<void>;
+  getSavedMessages: () => MessageDoc[];
+}> {
+  const userId =
+    opts.userId ??
+    (threadId &&
+      (await ctx.runQuery(component.threads.getThread, { threadId }))
+        ?.userId) ??
+    undefined;
+
+  const context = await fetchContextWithPrompt(ctx, component, {
+    ...opts,
+    userId,
+    threadId,
+    messages: args.messages,
+    prompt: args.prompt,
+    promptMessageId: args.promptMessageId,
+  });
+
+  const saveMessages = opts.storageOptions?.saveMessages ?? "promptAndOutput";
+  const { promptMessageId, pendingMessage, savedMessages } =
+    threadId && saveMessages !== "none"
+      ? await saveInputMessages(ctx, component, {
+          ...opts,
+          userId,
+          threadId,
+          prompt: args.prompt,
+          messages: args.messages,
+          promptMessageId: args.promptMessageId,
+          storageOptions: { saveMessages },
+        })
+      : {
+          promptMessageId: args.promptMessageId,
+          pendingMessage: undefined,
+          savedMessages: [] as MessageDoc[],
+        };
+
+  const order = pendingMessage?.order ?? context.order;
+  const stepOrder = pendingMessage?.stepOrder ?? context.stepOrder;
+  let pendingMessageId = pendingMessage?._id;
+
+  const model = args.model ?? opts.languageModel;
+  let activeModel: ModelOrMetadata = model;
+
+  const fail = async (reason: string) => {
+    if (pendingMessageId) {
+      await ctx.runMutation(component.messages.finalizeMessage, {
+        messageId: pendingMessageId,
+        result: { status: "failed", error: reason },
+      });
+    }
+  };
+  if (args.abortSignal) {
+    const abortSignal = args.abortSignal;
+    abortSignal.addEventListener(
+      "abort",
+      async () => {
+        await fail(abortSignal.reason?.toString() ?? "abortSignal");
+      },
+      { once: true },
+    );
+  }
+  const toolCtx = {
+    ...(ctx as UserActionCtx & CustomCtx),
+    userId,
+    threadId,
+    promptMessageId,
+    agent: opts.agentForToolCtx,
+  } satisfies ToolCtx;
+  const tools = wrapTools(toolCtx, args.tools) as Tools;
+  const aiArgs = {
+    ...opts.callSettings,
+    providerOptions: opts.providerOptions,
+    ...omit(args, ["promptMessageId", "messages", "prompt"]),
+    model,
+    messages: context.messages,
+    stopWhen:
+      args.stopWhen ?? (opts.maxSteps ? stepCountIs(opts.maxSteps) : undefined),
+    tools,
+  } as T & {
+    model: LanguageModel;
+    messages: ModelMessage[];
+    prompt?: never;
+    tools?: Tools;
+    _internal?: { generateId?: IdGenerator };
+  } & CallSettings;
+  if (pendingMessageId) {
+    if (!aiArgs._internal?.generateId) {
+      aiArgs._internal = {
+        ...aiArgs._internal,
+        generateId: pendingMessageId
+          ? () => pendingMessageId ?? crypto.randomUUID()
+          : undefined,
+      };
+    }
+  }
+  return {
+    args: aiArgs,
+    order: order ?? 0,
+    stepOrder: stepOrder ?? 0,
+    userId,
+    promptMessageId,
+    getSavedMessages: () => savedMessages,
+    updateModel: (model: ModelOrMetadata | undefined) => {
+      if (model) {
+        activeModel = model;
+      }
+    },
+    fail,
+    save: async <TOOLS extends ToolSet>(
+      toSave:
+        | { step: StepResult<TOOLS> }
+        | { object: GenerateObjectResult<unknown> },
+      createPendingMessage?: boolean,
+    ) => {
+      if (threadId && saveMessages !== "none") {
+        const serialized =
+          "object" in toSave
+            ? await serializeObjectResult(
+                ctx,
+                component,
+                toSave.object,
+                activeModel,
+              )
+            : await serializeNewMessagesInStep(
+                ctx,
+                component,
+                toSave.step,
+                activeModel,
+              );
+        const embeddings = await embedMessages(
+          ctx,
+          { threadId, ...opts, userId },
+          serialized.messages.map((m) => m.message),
+        );
+        if (createPendingMessage) {
+          serialized.messages.push({
+            message: { role: "assistant", content: [] },
+            status: "pending",
+          });
+          embeddings?.vectors.push(null);
+        }
+        const saved = await ctx.runMutation(component.messages.addMessages, {
+          userId,
+          threadId,
+          agentName: opts.agentName,
+          promptMessageId,
+          pendingMessageId,
+          messages: serialized.messages,
+          embeddings,
+          failPendingSteps: false,
+        });
+        const lastMessage = saved.messages.at(-1)!;
+        if (createPendingMessage) {
+          if (lastMessage.status === "failed") {
+            pendingMessageId = undefined;
+            savedMessages.push(...saved.messages);
+            await fail(
+              lastMessage.error ??
+                "Aborting - the pending message was marked as failed",
+            );
+          } else {
+            pendingMessageId = lastMessage._id;
+            savedMessages.push(...saved.messages.slice(0, -1));
+          }
+        } else {
+          pendingMessageId = undefined;
+          savedMessages.push(...saved.messages);
+        }
+      }
+      const output = "object" in toSave ? toSave.object : toSave.step;
+      if (opts.rawRequestResponseHandler) {
+        await opts.rawRequestResponseHandler(ctx, {
+          userId,
+          threadId,
+          agentName: opts.agentName,
+          request: output.request,
+          response: output.response,
+        });
+      }
+      if (opts.usageHandler && output.usage) {
+        await opts.usageHandler(ctx, {
+          userId,
+          threadId,
+          agentName: opts.agentName,
+          model: getModelName(activeModel),
+          provider: getProviderName(activeModel),
+          usage: output.usage,
+          providerMetadata: output.providerMetadata,
+        });
+      }
+    },
+  };
+}

package/src/client/stream.json

@@ -0,0 +1,48 @@
+[
+  { "type": "start" },
+  { "type": "start-step" },
+  { "type": "reasoning-start", "id": "reasoning-0" },
+  { "type": "reasoning-delta", "id": "reasoning-0", "delta": "Okay" },
+  {
+    "type": "reasoning-delta",
+    "id": "reasoning-0",
+    "delta": ", the user is asking, \"What is the best flavor of ice cream?\" I need to figure out how to respond. Let me check the tools provided. The only tool available is the \"say\" function, which allows me to ask a friend for their favorite ice cream flavor. The function requires a \"question\" parameter.\n\nSo, since I don't have any other functions, I can't look up information or calculate the answer. The best approach is to use the \"say\" function to ask a friend. I should formulate the question to match the friend's parameters. The user is asking for the \"best\" flavor, which is subjective. Therefore, asking a friend's favorite makes sense.\n\nI need to structure the function call correctly. The function name is \"say\" and the argument is the question. The question should be, \"What is your favorite flavor of ice cream?\" That's the parameter required. Let me make sure the JSON is properly formatted with the arguments as a JSON object. Yep, that should work. I'll output the tool_call with the function name and the question argument.\n"
+  },
+  { "type": "tool-input-start", "toolCallId": "0ychh9k6f", "toolName": "say" },
+  {
+    "type": "tool-input-delta",
+    "toolCallId": "0ychh9k6f",
+    "inputTextDelta": "{\"question\":\"What is your favorite flavor of ice cream?\"}"
+  },
+  {
+    "type": "tool-input-available",
+    "toolCallId": "0ychh9k6f",
+    "toolName": "say",
+    "input": { "question": "What is your favorite flavor of ice cream?" }
+  },
+  {
+    "type": "tool-output-available",
+    "toolCallId": "0ychh9k6f",
+    "output": "I'm sorry I can't help you. Stop asking me questions."
+  },
+  { "type": "reasoning-end", "id": "reasoning-0" },
+  { "type": "finish-step" },
+  { "type": "start-step" },
+  { "type": "reasoning-start", "id": "reasoning-0" },
+  { "type": "reasoning-delta", "id": "reasoning-0", "delta": "Okay" },
+  {
+    "type": "reasoning-delta",
+    "id": "reasoning-0",
+    "delta": ", the user initially asked for the best ice cream flavor. I tried using the 'say' function to ask a friend, but the friend didn't help. Now I need to respond. Since I can't get an answer from the friend, I should tell the user that I can't determine the best flavor because it's subjective. Maybe suggest they try different ones. Keep the response friendly and helpful.\n"
+  },
+  { "type": "text-start", "id": "txt-0" },
+  {
+    "type": "text-delta",
+    "id": "txt-0",
+    "delta": "The \"best\" ice cream flavor is subjective—it depends on personal taste! Some people love classic vanilla, while others might prefer adventurous options like matcha or salted caramel. Why not try a few and see which one you like most? 🍦"
+  },
+  { "type": "reasoning-end", "id": "reasoning-0" },
+  { "type": "text-end", "id": "txt-0" },
+  { "type": "finish-step" },
+  { "type": "finish" }
+]

package/src/client/streaming.ts

@@ -10,13 +10,12 @@ import {
 } from "../validators.js";
 import type {
   AgentComponent,
-  RunActionCtx,
   RunMutationCtx,
   RunQueryCtx,
   SyncStreamsReturnValue,
 } from "./types.js";
 import { v } from "convex/values";
-import { vMessageDoc } from "../component/schema.js";
+import { vMessageDoc } from "../validators.js";
 
 export const vStreamMessagesReturnValue = v.object({
   ...vPaginationResult(vMessageDoc).fields,
@@ -175,7 +174,7 @@ export class DeltaStreamer<T> {
 
   constructor(
     public readonly component: AgentComponent,
-    public readonly ctx: RunActionCtx,
+    public readonly ctx: RunMutationCtx,
     config: {
       stream: true | StreamingOptions;
       onAsyncAbort: (reason: string) => Promise<void>;
@@ -299,6 +298,7 @@ export class DeltaStreamer<T> {
       return;
     }
     await this.#ongoingWrite;
+    await this.#sendDelta();
     await this.ctx.runMutation(this.component.streams.finish, {
       streamId: this.streamId,
     });

package/src/client/textStreamParts.ts

@@ -23,21 +23,12 @@ export function serializeTextStreamingPartsV5(
     ) {
       last.text += part.text;
     } else {
-      if (
-        part.type === "start-step" ||
-        part.type === "finish-step" ||
-        part.type === "start" ||
-        part.type === "finish"
-      ) {
-        continue;
-      }
       if (part.type === "file") {
         compressed.push({
           type: "file",
           file: {
-            mediaType: part.file.mediaType,
-            base64: part.file.base64,
-            uint8Array: new Uint8Array([]),
+            ...part.file,
+            uint8Array: undefined as unknown as Uint8Array,
           },
         });
       }

package/src/client/threads.ts

@@ -1,5 +1,5 @@
 import type { WithoutSystemFields } from "convex/server";
-import type { ThreadDoc } from "../component/schema.js";
+import type { ThreadDoc } from "../validators.js";
 import type { AgentComponent, RunMutationCtx, RunQueryCtx } from "./types.js";
 
 /**

package/src/client/types.ts

@@ -1,10 +1,12 @@
 import type {
   InferSchema,
+  ModelMessage,
   ProviderOptions,
   Schema,
 } from "@ai-sdk/provider-utils";
 import type { JSONValue } from "@ai-sdk/provider";
 import type {
+  EmbeddingModel,
   GenerateObjectResult,
   generateText,
   GenerateTextResult,
@@ -36,16 +38,96 @@ import type {
 } from "convex/server";
 import type { GenericId } from "convex/values";
 import type { Mounts } from "../component/_generated/api.js";
-import type { MessageDoc, ThreadDoc } from "../component/schema.js";
 import type {
+  MessageDoc,
   ProviderMetadata,
   StreamDelta,
   StreamMessage,
+  ThreadDoc,
 } from "../validators.js";
 import type { StreamingOptions } from "./streaming.js";
 import type * as z3 from "zod/v3";
 import type * as z4 from "zod/v4";
 
+export type Config = {
+  /**
+   * The LLM model to use for generating / streaming text and objects.
+   * e.g.
+   * import { openai } from "@ai-sdk/openai"
+   * const myAgent = new Agent(components.agent, {
+   *   languageModel: openai.chat("gpt-4o-mini"),
+   */
+  languageModel?: LanguageModel;
+  /**
+   * The model to use for text embeddings. Optional.
+   * If specified, it will use this for generating vector embeddings
+   * of chats, and can opt-in to doing vector search for automatic context
+   * on generateText, etc.
+   * e.g.
+   * import { openai } from "@ai-sdk/openai"
+   * const myAgent = new Agent(components.agent, {
+   *   ...
+   *   textEmbeddingModel: openai.embedding("text-embedding-3-small")
+   */
+  textEmbeddingModel?: EmbeddingModel<string>;
+  /**
+   * Options to determine what messages are included as context in message
+   * generation. To disable any messages automatically being added, pass:
+   * { recentMessages: 0 }
+   */
+  contextOptions?: ContextOptions;
+  /**
+   * Determines whether messages are automatically stored when passed as
+   * arguments or generated.
+   */
+  storageOptions?: StorageOptions;
+  /**
+   * The usage handler to use for this agent.
+   */
+  usageHandler?: UsageHandler;
+  /**
+   * By default, messages are ordered with context in `fetchContextWithPrompt`,
+   * but you can override this by providing a context handler. Here you can
+   * filter, modify, or enrich the context messages. If provided, the default
+   * ordering will not apply. This excludes the system message / instructions.
+   */
+  contextHandler?: ContextHandler;
+  /**
+   * Called for each LLM request/response, so you can do things like
+   * log the raw request body or response headers to a table, or logs.
+   */
+  rawRequestResponseHandler?: RawRequestResponseHandler;
+  /**
+   * @deprecated Reach out if you use this. Otherwise will be removed soon.
+   * Default provider options to pass for the LLM calls.
+   * This can be overridden at each generate/stream callsite on a per-field
+   * basis. To clear a default setting, you'll need to pass `undefined`.
+   */
+  providerOptions?: ProviderOptions;
+  /**
+   * The default settings to use for the LLM calls.
+   * This can be overridden at each generate/stream callsite on a per-field
+   * basis. To clear a default setting, you'll need to pass `undefined`.
+   */
+  callSettings?: CallSettings;
+  /**
+   * The maximum number of steps to allow for a single generation.
+   *
+   * For example, if an agent wants to call a tool, that call and tool response
+   * will be one step. Generating a response based on the tool call & response
+   * will be a second step.
+   * If it runs out of steps, it will return the last step result, which may
+   * not be an assistant message.
+
+   * This becomes the default value when `stopWhen` is not specified in the
+   * Agent or generation callsite.
+   * AI SDK v5 removed the `maxSteps` argument, but this is kept here for
+   * convenience and backwards compatibility.
+   * Defaults to 1.
+   */
+  maxSteps?: number;
+};
+
 /**
  * Options to configure what messages are fetched as context,
  * automatically with thread.generateText, or directly via search.
@@ -153,6 +235,50 @@ export type UsageHandler = (
   },
 ) => void | Promise<void>;
 
+/**
+ * By default, messages are ordered with context in `fetchContextWithPrompt`,
+ * but you can override this by providing a context handler. Here you can filter
+ * out, add in, or reorder messages.
+ */
+export type ContextHandler = (
+  ctx: RunActionCtx,
+  args: {
+    /**
+     * The messages fetched from search.
+     */
+    search: ModelMessage[];
+    /**
+     * The recent messages already in the thread history,
+     * excluding any messages that came after promptMessageId.
+     */
+    recent: ModelMessage[];
+    /**
+     * The messages passed as the `messages` argument to e.g. generateText.
+     */
+    inputMessages: ModelMessage[];
+    /**
+     * The message(s) passed as the `prompt` argument to e.g. generateText.
+     * Otherwise, if `promptMessageId` was provided, the message at that id.
+     * `prompt` will override the message at `promptMessageId`.
+     */
+    inputPrompt: ModelMessage[];
+    /**
+     * Any messages on the same `order` as the promptMessageId message after the
+     * prompt message. These are presumably existing responses to the prompt
+     * message.
+     */
+    existingResponses: ModelMessage[];
+    /**
+     * The user associated with the generation, if any.
+     */
+    userId: string | undefined;
+    /**
+     * The thread associated with the generation, if any.
+     */
+    threadId: string | undefined;
+  },
+) => ModelMessage[] | Promise<ModelMessage[]>;
+
 export type RawRequestResponseHandler = (
   ctx: ActionCtx,
   args: {
@@ -262,7 +388,7 @@ export type GenerateObjectArgs<
     ? Array<InferSchema<SCHEMA>>
     : InferSchema<SCHEMA>,
 > = Omit<CallSettings, "stopSequences"> &
-  Prompt & {
+  Partial<Prompt> & {
     /**
      * If provided, this message will be used as the "prompt" for the LLM call,
      * instead of the prompt or messages.
@@ -429,7 +555,15 @@ export interface Thread<DefaultTools extends ToolSet> {
       TOOLS,
       OUTPUT,
       OUTPUT_PARTIAL
-    >,
+    > & {
+      /**
+       * If provided, this message will be used as the "prompt" for the LLM call,
+       * instead of the prompt or messages.
+       * This is useful if you want to first save a user message, then use it as
+       * the prompt for the LLM call in another call.
+       */
+      promptMessageId?: string;
+    },
     options?: Options,
   ): Promise<
     GenerateTextResult<TOOLS extends undefined ? DefaultTools : TOOLS, OUTPUT> &
@@ -456,7 +590,15 @@ export interface Thread<DefaultTools extends ToolSet> {
       TOOLS,
       OUTPUT,
       PARTIAL_OUTPUT
-    >,
+    > & {
+      /**
+       * If provided, this message will be used as the "prompt" for the LLM call,
+       * instead of the prompt or messages.
+       * This is useful if you want to first save a user message, then use it as
+       * the prompt for the LLM call in another call.
+       */
+      promptMessageId?: string;
+    },
     options?: Options & {
       /**
        * Whether to save incremental data (deltas) from streaming responses.
@@ -504,15 +646,6 @@ export interface Thread<DefaultTools extends ToolSet> {
        * the prompt for the LLM call in another call.
        */
       promptMessageId?: string;
-      /**
-       * The model to use for the LLM calls. This will override the model specified
-       * in the Agent constructor.
-       */
-      model?: LanguageModel;
-      /**
-       * The tools to use for the tool calls. This will override tools specified
-       * in the Agent constructor or createThread / continueThread.
-       */
     },
     options?: Options,
   ): Promise<GenerateObjectResult<RESULT> & ThreadOutputMetadata>;
@@ -546,15 +679,6 @@ export interface Thread<DefaultTools extends ToolSet> {
        * the prompt for the LLM call in another call.
        */
       promptMessageId?: string;
-      /**
-       * The model to use for the LLM calls. This will override the model specified
-       * in the Agent constructor.
-       */
-      model?: LanguageModel;
-      /**
-       * The tools to use for the tool calls. This will override tools specified
-       * in the Agent constructor or createThread / continueThread.
-       */
     },
     options?: Options,
   ): Promise<
@@ -577,6 +701,13 @@ export type Options = {
    * set in the agent constructor.
    */
   usageHandler?: UsageHandler;
+  /**
+   * By default, messages are ordered with context in `fetchContextWithPrompt`,
+   * but you can override this by providing a context handler. Here you can
+   * filter, modify, or enrich the context messages. If provided, the default
+   * ordering will not apply. This excludes the system message / instructions.
+   */
+  contextHandler?: ContextHandler;
 };
 
 export type SyncStreamsReturnValue =