@convex-dev/agent 0.3.2 → 0.6.0-alpha.0

package/src/client/createTool.ts

@@ -1,9 +1,24 @@
-import type { FlexibleSchema } from "@ai-sdk/provider-utils";
-import type { Tool, ToolCallOptions, ToolSet } from "ai";
+import type { ToolResultOutput } from "@ai-sdk/provider-utils";
+import type {
+  FlexibleSchema,
+  ModelMessage,
+  Tool,
+  ToolExecutionOptions,
+  ToolSet,
+} from "ai";
 import { tool } from "ai";
-import type { Agent } from "./index.js";
 import type { GenericActionCtx, GenericDataModel } from "convex/server";
 import type { ProviderOptions } from "../validators.js";
+import type { Agent } from "./index.js";
+
+const MIGRATION_URL = "https://github.com/get-convex/agent/blob/main/MIGRATION.md";
+const warnedDeprecations = new Set<string>();
+function warnDeprecation(key: string, message: string) {
+  if (!warnedDeprecations.has(key)) {
+    warnedDeprecations.add(key);
+    console.warn(`[@convex-dev/agent] ${message}\n  See: ${MIGRATION_URL}`);
+  }
+}
 
 export type ToolCtx<DataModel extends GenericDataModel = GenericDataModel> =
   GenericActionCtx<DataModel> & {
@@ -13,79 +28,254 @@ export type ToolCtx<DataModel extends GenericDataModel = GenericDataModel> =
     messageId?: string;
   };
 
+/**
+ * Function that is called to determine if the tool needs approval before it can be executed.
+ */
+export type ToolNeedsApprovalFunctionCtx<
+  INPUT,
+  Ctx extends ToolCtx = ToolCtx,
+> = (
+  ctx: Ctx,
+  input: INPUT,
+  options: {
+    /**
+     * The ID of the tool call. You can use it e.g. when sending tool-call related information with stream data.
+     */
+    toolCallId: string;
+    /**
+     * Messages that were sent to the language model to initiate the response that contained the tool call.
+     * The messages **do not** include the system prompt nor the assistant response that contained the tool call.
+     */
+    messages: ModelMessage[];
+    /**
+     * Additional context.
+     *
+     * Experimental (can break in patch releases).
+     */
+    experimental_context?: unknown;
+  },
+) => boolean | PromiseLike<boolean>;
+
+export type ToolExecuteFunctionCtx<
+  INPUT,
+  OUTPUT,
+  Ctx extends ToolCtx = ToolCtx,
+> = (
+  ctx: Ctx,
+  input: INPUT,
+  options: ToolExecutionOptions,
+) => AsyncIterable<OUTPUT> | PromiseLike<OUTPUT>;
+
+type NeverOptional<N, T> = 0 extends 1 & N
+  ? Partial<T>
+  : [N] extends [never]
+    ? Partial<Record<keyof T, undefined>>
+    : T;
+
+export type ToolOutputPropertiesCtx<
+  INPUT,
+  OUTPUT,
+  Ctx extends ToolCtx = ToolCtx,
+> = NeverOptional<
+  OUTPUT,
+  | {
+      /**
+       * An async function that is called with the arguments from the tool call and produces a result.
+       * If `execute` (or `handler`) is not provided, the tool will not be executed automatically.
+       *
+       * @param input - The input of the tool call.
+       * @param options.abortSignal - A signal that can be used to abort the tool call.
+       */
+      execute: ToolExecuteFunctionCtx<INPUT, OUTPUT, Ctx>;
+      outputSchema?: FlexibleSchema<OUTPUT>;
+      handler?: never;
+    }
+  | {
+      /** @deprecated Use execute instead. */
+      handler: ToolExecuteFunctionCtx<INPUT, OUTPUT, Ctx>;
+      outputSchema?: FlexibleSchema<OUTPUT>;
+      execute?: never;
+    }
+  | {
+      outputSchema: FlexibleSchema<OUTPUT>;
+      execute?: never;
+      handler?: never;
+    }
+>;
+
+export type ToolInputProperties<INPUT> =
+  | {
+      /**
+       * The schema of the input that the tool expects.
+       * The language model will use this to generate the input.
+       * It is also used to validate the output of the language model.
+       *
+       * You can use descriptions on the schema properties to make the input understandable for the language model.
+       */
+      inputSchema: FlexibleSchema<INPUT>;
+      args?: never;
+    }
+  | {
+      /**
+       * The schema of the input that the tool expects. The language model will use this to generate the input.
+       * It is also used to validate the output of the language model.
+       * Use descriptions to make the input understandable for the language model.
+       *
+       * @deprecated Use inputSchema instead.
+       */
+      args: FlexibleSchema<INPUT>;
+      inputSchema?: never;
+    };
+
 /**
  * This is a wrapper around the ai.tool function that adds extra context to the
  * tool call, including the action context, userId, threadId, and messageId.
  * @param tool The tool. See https://sdk.vercel.ai/docs/ai-sdk-core/tools-and-tool-calling
- * but swap parameters for args and handler for execute.
+ * Currently contains deprecated parameters `args` and `handler` to maintain backwards compatibility
+ * but these will be removed in the future. Use `inputSchema` and `execute` instead, respectively.
+ * 
  * @returns A tool to be used with the AI SDK.
  */
-export function createTool<INPUT, OUTPUT, Ctx extends ToolCtx = ToolCtx>(def: {
-  /**
-  An optional description of what the tool does.
-  Will be used by the language model to decide whether to use the tool.
-  Not used for provider-defined tools.
+export function createTool<INPUT, OUTPUT, Ctx extends ToolCtx = ToolCtx>(
+  def: {
+    /**
+     * An optional description of what the tool does.
+     * Will be used by the language model to decide whether to use the tool.
+     * Not used for provider-defined tools.
      */
-  description?: string;
-  /**
-  The schema of the input that the tool expects. The language model will use this to generate the input.
-  It is also used to validate the output of the language model.
-  Use descriptions to make the input understandable for the language model.
+    description?: string;
+    /**
+     * An optional title of the tool.
      */
-  args: FlexibleSchema<INPUT>;
-  /**
-  An async function that is called with the arguments from the tool call and produces a result.
-  If not provided, the tool will not be executed automatically.
-
-  @args is the input of the tool call.
-  @options.abortSignal is a signal that can be used to abort the tool call.
+    title?: string;
+    /**
+     * Additional provider-specific metadata. They are passed through
+     * to the provider from the AI SDK and enable provider-specific
+     * functionality that can be fully encapsulated in the provider.
      */
-  handler: (
-    ctx: Ctx,
-    args: INPUT,
-    options: ToolCallOptions,
-  ) => PromiseLike<OUTPUT> | AsyncIterable<OUTPUT>;
-  /**
-   * Provide the context to use, e.g. when defining the tool at runtime.
-   */
-  ctx?: Ctx;
-  /**
-   * Optional function that is called when the argument streaming starts.
-   * Only called when the tool is used in a streaming context.
-   */
-  onInputStart?: (
-    ctx: Ctx,
-    options: ToolCallOptions,
-  ) => void | PromiseLike<void>;
-  /**
-   * Optional function that is called when an argument streaming delta is available.
-   * Only called when the tool is used in a streaming context.
-   */
-  onInputDelta?: (
-    ctx: Ctx,
-    options: { inputTextDelta: string } & ToolCallOptions,
-  ) => void | PromiseLike<void>;
-  /**
-   * Optional function that is called when a tool call can be started,
-   * even if the execute function is not provided.
-   */
-  onInputAvailable?: (
-    ctx: Ctx,
-    options: {
-      input: [INPUT] extends [never] ? undefined : INPUT;
-    } & ToolCallOptions,
-  ) => void | PromiseLike<void>;
+    providerOptions?: ProviderOptions;
+  } & ToolInputProperties<INPUT> & {
+      /**
+       * An optional list of input examples that show the language
+       * model what the input should look like.
+       */
+      inputExamples?: Array<{
+        input: NoInfer<INPUT>;
+      }>;
+      /**
+       * Whether the tool needs approval before it can be executed.
+       */
+      needsApproval?:
+        | boolean
+        | ToolNeedsApprovalFunctionCtx<
+            [INPUT] extends [never] ? unknown : INPUT,
+            Ctx
+          >;
+      /**
+       * Strict mode setting for the tool.
+       *
+       * Providers that support strict mode will use this setting to determine
+       * how the input should be generated. Strict mode will always produce
+       * valid inputs, but it might limit what input schemas are supported.
+       */
+      strict?: boolean;
+      /**
+       * Provide the context to use, e.g. when defining the tool at runtime.
+       */
+      ctx?: Ctx;
+      /**
+       * Optional function that is called when the argument streaming starts.
+       * Only called when the tool is used in a streaming context.
+       */
+      onInputStart?: (
+        ctx: Ctx,
+        options: ToolExecutionOptions,
+      ) => void | PromiseLike<void>;
+      /**
+       * Optional function that is called when an argument streaming delta is available.
+       * Only called when the tool is used in a streaming context.
+       */
+      onInputDelta?: (
+        ctx: Ctx,
+        options: { inputTextDelta: string } & ToolExecutionOptions,
+      ) => void | PromiseLike<void>;
+      /**
+       * Optional function that is called when a tool call can be started,
+       * even if the execute function is not provided.
+       */
+      onInputAvailable?: (
+        ctx: Ctx,
+        options: {
+          input: [INPUT] extends [never] ? unknown : INPUT;
+        } & ToolExecutionOptions,
+      ) => void | PromiseLike<void>;
+    } & ToolOutputPropertiesCtx<INPUT, OUTPUT, Ctx> & {
+      /**
+       * Optional conversion function that maps the tool result to an output that can be used by the language model.
+       *
+       * If not provided, the tool result will be sent as a JSON object.
+       */
+      toModelOutput?: (
+        ctx: Ctx,
+        options: {
+          /**
+           * The ID of the tool call. You can use it e.g. when sending tool-call related information with stream data.
+           */
+          toolCallId: string;
+          /**
+           * The input of the tool call.
+           */
+          input: [INPUT] extends [never] ? unknown : INPUT;
+          /**
+           * The output of the tool call.
+           */
+          output: 0 extends 1 & OUTPUT
+            ? any
+            : [OUTPUT] extends [never]
+              ? any
+              : NoInfer<OUTPUT>;
+        },
+      ) => ToolResultOutput | PromiseLike<ToolResultOutput>;
+    },
+): Tool<INPUT, OUTPUT> {
+  const inputSchema = def.inputSchema ?? def.args;
+  if (!inputSchema)
+    throw new Error("To use a Convex tool, you must provide an `inputSchema` (or `args`)");
+
+  if (def.args && !def.inputSchema) {
+    warnDeprecation(
+      "createTool.args",
+      "createTool: 'args' is deprecated. Use 'inputSchema' instead.",
+    );
+  }
+  if (def.handler && !def.execute) {
+    warnDeprecation(
+      "createTool.handler",
+      "createTool: 'handler' is deprecated. Use 'execute' instead.",
+    );
+  }
 
-  // Extra AI SDK pass-through options.
-  providerOptions?: ProviderOptions;
-}): Tool<INPUT, OUTPUT> {
-  const t = tool({
+  const executeHandler = def.execute ?? def.handler;
+  if (!executeHandler && !def.outputSchema)
+    throw new Error(
+      "To use a Convex tool, you must either provide an execute" +
+        " handler function, define an outputSchema, or both",
+    );
+
+  const t = tool<INPUT, OUTPUT>({
     type: "function",
     __acceptsCtx: true,
     ctx: def.ctx,
     description: def.description,
-    inputSchema: def.args,
-    execute(args: INPUT, options: ToolCallOptions) {
+    title: def.title,
+    providerOptions: def.providerOptions,
+    inputSchema,
+    inputExamples: def.inputExamples,
+    needsApproval(this: Tool<INPUT, OUTPUT>, input, options) {
+      const needsApproval = def.needsApproval;
+      if (!needsApproval || typeof needsApproval === "boolean")
+        return Boolean(needsApproval);
+
       if (!getCtx(this)) {
         throw new Error(
           "To use a Convex tool, you must either provide the ctx" +
@@ -93,18 +283,52 @@ export function createTool<INPUT, OUTPUT, Ctx extends ToolCtx = ToolCtx>(def: {
             " call it (which injects the ctx, userId and threadId)",
         );
       }
-      return def.handler(getCtx(this), args, options);
+      return needsApproval(getCtx(this), input, options);
     },
-    providerOptions: def.providerOptions,
+    strict: def.strict,
+    ...(executeHandler
+      ? {
+          execute(
+            this: Tool<INPUT, OUTPUT>,
+            input: INPUT,
+            options: ToolExecutionOptions,
+          ) {
+            if (!getCtx(this)) {
+              throw new Error(
+                "To use a Convex tool, you must either provide the ctx" +
+                  " at definition time (dynamically in an action), or use the Agent to" +
+                  " call it (which injects the ctx, userId and threadId)",
+              );
+            }
+            return executeHandler(getCtx(this), input, options);
+          },
+        }
+      : {}),
+    outputSchema: def.outputSchema,
   });
   if (def.onInputStart) {
-    t.onInputStart = def.onInputStart.bind(t, getCtx(t));
+    const origOnInputStart = def.onInputStart;
+    t.onInputStart = function (this: Tool<INPUT, OUTPUT>, options) {
+      return origOnInputStart.call(this, getCtx(this), options);
+    };
   }
   if (def.onInputDelta) {
-    t.onInputDelta = def.onInputDelta.bind(t, getCtx(t));
+    const origOnInputDelta = def.onInputDelta;
+    t.onInputDelta = function (this: Tool<INPUT, OUTPUT>, options) {
+      return origOnInputDelta.call(this, getCtx(this), options);
+    };
   }
   if (def.onInputAvailable) {
-    t.onInputAvailable = def.onInputAvailable.bind(t, getCtx(t));
+    const origOnInputAvailable = def.onInputAvailable;
+    t.onInputAvailable = function (this: Tool<INPUT, OUTPUT>, options) {
+      return origOnInputAvailable.call(this, getCtx(this), options);
+    };
+  }
+  if (def.toModelOutput) {
+    const origToModelOutput = def.toModelOutput;
+    t.toModelOutput = function (this: Tool<INPUT, OUTPUT>, options) {
+      return origToModelOutput.call(this, getCtx(this), options);
+    };
   }
   return t;
 }

package/src/client/files.ts

@@ -92,7 +92,7 @@ export async function storeFile(
     storageId: newStorageId,
     hash,
     filename,
-    mimeType: blob.type,
+    mediaType: blob.type,
   });
   const url = (await ctx.storage.getUrl(storageId as Id<"_storage">))!;
   if (storageId !== newStorageId) {
@@ -142,8 +142,10 @@ export async function getFile(
   if (!url) {
     throw new Error(`File not found in storage: ${file.storageId}`);
   }
+  // Support both mediaType (preferred) and mimeType (deprecated)
+  const mediaType = file.mediaType ?? file.mimeType ?? "";
   return {
-    ...getParts(url, file.mimeType, file.filename),
+    ...getParts(url, mediaType, file.filename),
     file: {
       fileId,
       url,

package/src/client/index.test.ts

@@ -193,6 +193,7 @@ describe("filterOutOrphanedToolMessages", () => {
           type: "tool-call",
           toolCallId: "1",
           toolName: "tool1",
+          input: { test: "test" },
           args: { test: "test" },
         },
       ],

package/src/client/index.ts

@@ -6,6 +6,7 @@ import type {
 } from "@ai-sdk/provider-utils";
 import type {
   CallSettings,
+  EmbeddingModel,
   GenerateObjectResult,
   GenerateTextResult,
   LanguageModel,
@@ -17,6 +18,15 @@ import type {
   ToolSet,
 } from "ai";
 import { generateObject, generateText, stepCountIs, streamObject } from "ai";
+
+const MIGRATION_URL = "https://github.com/get-convex/agent/blob/main/MIGRATION.md";
+const warnedDeprecations = new Set<string>();
+function warnDeprecation(key: string, message: string) {
+  if (!warnedDeprecations.has(key)) {
+    warnedDeprecations.add(key);
+    console.warn(`[@convex-dev/agent] ${message}\n  See: ${MIGRATION_URL}`);
+  }
+}
 import { assert, omit, pick } from "convex-helpers";
 import {
   internalActionGeneric,
@@ -86,6 +96,7 @@ import type {
   UsageHandler,
   QueryCtx,
   AgentPrompt,
+  Output,
 } from "./types.js";
 import { streamText } from "./streamText.js";
 import { errorToString, willContinue } from "./utils.js";
@@ -241,7 +252,22 @@ export class Agent<
         | StopCondition<NoInfer<AgentTools>>
         | Array<StopCondition<NoInfer<AgentTools>>>;
     },
-  ) {}
+  ) {
+    if (this.options.textEmbeddingModel && !this.options.embeddingModel) {
+      warnDeprecation(
+        "textEmbeddingModel",
+        "textEmbeddingModel is deprecated. Use embeddingModel instead.",
+      );
+    }
+  }
+
+  /**
+   * Get the embedding model, prioritizing embeddingModel over textEmbeddingModel.
+   * @private
+   */
+  private getEmbeddingModel(): EmbeddingModel | undefined {
+    return this.options.embeddingModel ?? this.options.textEmbeddingModel;
+  }
 
   /**
    * Start a new thread with the agent. This will have a fresh history, though if
@@ -416,9 +442,7 @@ export class Agent<
         ...args,
         tools: (args.tools ?? this.options.tools) as Tools,
         system: args.system ?? this.options.instructions,
-        stopWhen: (args.stopWhen ?? this.options.stopWhen) as
-          | StopCondition<Tools>
-          | Array<StopCondition<Tools>>,
+        stopWhen: (args.stopWhen ?? this.options.stopWhen) as any,
       },
       {
         ...this.options,
@@ -444,8 +468,7 @@ export class Agent<
    */
   async generateText<
     TOOLS extends ToolSet | undefined = undefined,
-    OUTPUT = never,
-    OUTPUT_PARTIAL = never,
+    OUTPUT extends Output<any, any, any> = never,
   >(
     ctx: ActionCtx & CustomCtx,
     threadOpts: { userId?: string | null; threadId?: string },
@@ -454,7 +477,7 @@ export class Agent<
      * {@link generateText} function, along with Agent prompt options.
      */
     generateTextArgs: AgentPrompt &
-      TextArgs<AgentTools, TOOLS, OUTPUT, OUTPUT_PARTIAL>,
+      TextArgs<AgentTools, TOOLS, OUTPUT>,
     options?: Options,
   ): Promise<
     GenerateTextResult<TOOLS extends undefined ? AgentTools : TOOLS, OUTPUT> &
@@ -469,7 +492,7 @@ export class Agent<
     type Tools = TOOLS extends undefined ? AgentTools : TOOLS;
     const steps: StepResult<Tools>[] = [];
     try {
-      const result = (await generateText<Tools, OUTPUT, OUTPUT_PARTIAL>({
+      const result = (await generateText<Tools, OUTPUT>({
         ...args,
         prepareStep: async (options) => {
           const result = await generateTextArgs.prepareStep?.(options);
@@ -504,8 +527,7 @@ export class Agent<
    */
   async streamText<
     TOOLS extends ToolSet | undefined = undefined,
-    OUTPUT = never,
-    PARTIAL_OUTPUT = never,
+    OUTPUT extends Output<any, any, any> = never,
   >(
     ctx: ActionCtx & CustomCtx,
     threadOpts: { userId?: string | null; threadId?: string },
@@ -514,7 +536,7 @@ export class Agent<
      * {@link streamText} function, along with Agent prompt options.
      */
     streamTextArgs: AgentPrompt &
-      StreamingTextArgs<AgentTools, TOOLS, OUTPUT, PARTIAL_OUTPUT>,
+      StreamingTextArgs<AgentTools, TOOLS, OUTPUT>,
     /**
      * The {@link ContextOptions} and {@link StorageOptions}
      * options to use for fetching contextual messages and saving input/output messages.
@@ -535,12 +557,12 @@ export class Agent<
   ): Promise<
     StreamTextResult<
       TOOLS extends undefined ? AgentTools : TOOLS,
-      PARTIAL_OUTPUT
+      OUTPUT
     > &
       GenerationOutputMetadata
   > {
     type Tools = TOOLS extends undefined ? AgentTools : TOOLS;
-    return streamText<Tools, OUTPUT, PARTIAL_OUTPUT>(
+    return streamText<Tools, OUTPUT>(
       ctx,
       this.component,
       {
@@ -548,9 +570,7 @@ export class Agent<
         model: streamTextArgs.model ?? this.options.languageModel,
         tools: (streamTextArgs.tools ?? this.options.tools) as Tools,
         system: streamTextArgs.system ?? this.options.instructions,
-        stopWhen: (streamTextArgs.stopWhen ?? this.options.stopWhen) as
-          | StopCondition<Tools>
-          | Array<StopCondition<Tools>>,
+        stopWhen: (streamTextArgs.stopWhen ?? this.options.stopWhen) as any,
       },
       {
         ...threadOpts,
@@ -746,7 +766,7 @@ export class Agent<
     const { skipEmbeddings, ...rest } = args;
     if (args.embeddings) {
       embeddings = args.embeddings;
-    } else if (!skipEmbeddings && this.options.textEmbeddingModel) {
+    } else if (!skipEmbeddings && this.getEmbeddingModel()) {
       if (!("runAction" in ctx)) {
         console.warn(
           "You're trying to save messages and generate embeddings, but you're in a mutation. " +
@@ -862,9 +882,10 @@ export class Agent<
       contextOptions,
       getEmbedding: async (text) => {
         assert("runAction" in ctx);
+        const embeddingModel = this.getEmbeddingModel();
         assert(
-          this.options.textEmbeddingModel,
-          "A textEmbeddingModel is required to be set on the Agent that you're doing vector search with",
+          embeddingModel,
+          "An embeddingModel (or textEmbeddingModel) is required to be set on the Agent that you're doing vector search with",
         );
         return {
           embedding: (
@@ -876,7 +897,7 @@ export class Agent<
               values: [text],
             })
           ).embeddings[0],
-          textEmbeddingModel: this.options.textEmbeddingModel,
+          embeddingModel: embeddingModel,
         };
       },
     });
@@ -975,10 +996,10 @@ export class Agent<
             .join(", "),
       );
     }
-    const { textEmbeddingModel } = this.options;
-    if (!textEmbeddingModel) {
+    const embeddingModel = this.getEmbeddingModel();
+    if (!embeddingModel) {
       throw new Error(
-        "No embeddings were generated for the messages. You must pass a textEmbeddingModel to the agent constructor.",
+        "No embeddings were generated for the messages. You must pass an embeddingModel (or textEmbeddingModel) to the agent constructor.",
       );
     }
     await generateAndSaveEmbeddings(
@@ -989,7 +1010,7 @@ export class Agent<
         agentName: this.options.name,
         threadId: messages[0].threadId,
         userId: messages[0].userId,
-        textEmbeddingModel,
+        embeddingModel,
       },
       messages,
     );
@@ -1440,7 +1461,7 @@ export class Agent<
         } as GenerateObjectArgs<FlexibleSchema<T>>;
         const ctx = (
           options?.customCtx
-            ? { ...ctx_, ...options.customCtx(ctx_, targetArgs, llmArgs) }
+            ? { ...ctx_, ...options.customCtx(ctx_, targetArgs, llmArgs as any) }
             : ctx_
         ) as GenericActionCtx<GenericDataModel> & CustomCtx;
         const value = await this.generateObject(ctx, targetArgs, llmArgs, {