@convex-dev/agent 0.1.15-alpha.1 → 0.1.15

package/src/client/index.ts

@@ -1,5 +1,6 @@
 import type { EmbeddingModelV1, LanguageModelV1 } from "@ai-sdk/provider";
 import type {
+  AssistantContent,
   CoreMessage,
   DeepPartial,
   FilePart,
@@ -10,6 +11,7 @@ import type {
   StreamObjectResult,
   StreamTextResult,
   ToolSet,
+  UserContent,
 } from "ai";
 import { generateObject, generateText, streamObject, streamText } from "ai";
 import { assert } from "convex-helpers";
@@ -22,6 +24,7 @@ import {
 } from "convex/server";
 import { v } from "convex/values";
 import type { MessageDoc, ThreadDoc } from "../component/schema.js";
+import type { threadFieldsSupportingPatch } from "../component/threads.js";
 import {
   validateVectorDimension,
   type VectorDimension,
@@ -36,23 +39,27 @@ import {
 } from "../mapping.js";
 import { extractText, isTool } from "../shared.js";
 import {
-  type MessageWithMetadata,
+  type MessageEmbeddings,
   type MessageStatus,
+  type MessageWithMetadata,
   type ProviderMetadata,
   type StreamArgs,
   type Usage,
   vMessageWithMetadata,
   vSafeObjectArgs,
   vTextArgs,
-  type MessageEmbeddings,
 } from "../validators.js";
 import { createTool, wrapTools } from "./createTool.js";
+import { listMessages } from "./listMessages.js";
+import { fetchContextMessages } from "./search.js";
 import {
   DeltaStreamer,
   mergeTransforms,
   type StreamingOptions,
+  syncStreams,
 } from "./streaming.js";
 import type {
+  ActionCtx,
   AgentComponent,
   ContextOptions,
   GenerationOutputMetadata,
@@ -63,7 +70,6 @@ import type {
   RunActionCtx,
   RunMutationCtx,
   RunQueryCtx,
-  ActionCtx,
   StorageOptions,
   StreamingTextArgs,
   SyncStreamsReturnValue,
@@ -71,14 +77,9 @@ import type {
   Thread,
   UsageHandler,
 } from "./types.js";
-import type { threadFieldsSupportingPatch } from "../component/threads.js";
-import { listMessages } from "./listMessages.js";
-import { syncStreams } from "./streaming.js";
-import { fetchContextMessages } from "./search.js";
 
-export { storeFile, getFile } from "./files.js";
-export { serializeDataOrUrl } from "../mapping.js";
 export { vMessageDoc, vThreadDoc } from "../component/schema.js";
+export { serializeDataOrUrl } from "../mapping.js";
 export {
   vAssistantMessage,
   vContextOptions,
@@ -93,7 +94,9 @@ export {
   vUserMessage,
 } from "../validators.js";
 export type { ToolCtx } from "./createTool.js";
+export { getFile, storeFile } from "./files.js";
 export { filterOutOrphanedToolMessages } from "./search.js";
+export { abortStream, listStreams } from "./streaming.js";
 export {
   createTool,
   extractText,
@@ -421,7 +424,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
       threadId,
       ...options,
     });
-    const { args: aiArgs, messageId, userId } = context;
+    const { args: aiArgs, messageId, order, userId } = context;
     const toolCtx = { ...ctx, userId, threadId, messageId, agent: this };
     const tools = wrapTools(
       toolCtx,
@@ -476,6 +479,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
       > &
         GenerationOutputMetadata;
       result.messageId = messageId;
+      result.order = order;
       return result;
     } catch (error) {
       if (threadId && messageId) {
@@ -642,6 +646,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
     > &
       GenerationOutputMetadata;
     result.messageId = messageId;
+    result.order = order;
     return result;
   }
 
@@ -674,7 +679,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
       threadId,
       ...options,
     });
-    const { args: aiArgs, messageId, userId } = context;
+    const { args: aiArgs, messageId, order, userId } = context;
     const trackUsage = usageHandler ?? this.options.usageHandler;
     const saveOutputMessages = this._shouldSaveOutputMessages(
       options?.storageOptions
@@ -696,6 +701,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
         });
       }
       result.messageId = messageId;
+      result.order = order;
       if (this.options.rawRequestResponseHandler) {
         await this.options.rawRequestResponseHandler(ctx, {
           userId,
@@ -760,7 +766,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
       threadId,
       ...options,
     });
-    const { args: aiArgs, messageId, userId } = context;
+    const { args: aiArgs, messageId, order, userId } = context;
     const trackUsage = usageHandler ?? this.options.usageHandler;
     const saveOutputMessages = this._shouldSaveOutputMessages(
       options?.storageOptions
@@ -821,6 +827,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
     }) as StreamObjectResult<DeepPartial<T>, T, never> &
       GenerationOutputMetadata;
     stream.messageId = messageId;
+    stream.order = order;
     return stream;
   }
 
@@ -846,6 +853,12 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
     const { lastMessageId, messages } = await this.saveMessages(ctx, {
       threadId: args.threadId,
       userId: args.userId,
+      embeddings: args.embedding
+        ? {
+            model: args.embedding.model,
+            vectors: [args.embedding.vector],
+          }
+        : undefined,
       messages:
         args.prompt !== undefined
           ? [{ role: "user", content: args.prompt }]
@@ -882,7 +895,6 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
     let embeddings:
       | {
           vectors: (number[] | null)[];
-          dimension: VectorDimension;
           model: string;
         }
       | undefined;
@@ -1678,7 +1690,7 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
     return Promise.all(
       messages.map(async (message): Promise<CoreMessage> => {
         if (
-          message.role !== "user" ||
+          (message.role !== "user" && message.role !== "assistant") ||
           typeof message.content === "string" ||
           !Array.isArray(message.content)
         ) {
@@ -1688,6 +1700,10 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
         const processedContent = await Promise.all(
           message.content.map(async (part) => {
             if (part.type === "image" && part.image instanceof URL) {
+              assert(
+                message.role === "user",
+                "Images can only be in user messages"
+              );
               if (this._isLocalhostUrl(part.image)) {
                 const imageData = await this._downloadFile(part.image);
                 return {
@@ -1711,11 +1727,17 @@ export class Agent<AgentTools extends ToolSet = ToolSet> {
             return part;
           })
         );
-
-        return {
-          ...message,
-          content: processedContent,
-        };
+        if (message.role === "user") {
+          return {
+            ...message,
+            content: processedContent as UserContent,
+          };
+        } else {
+          return {
+            ...message,
+            content: processedContent as AssistantContent,
+          };
+        }
       })
     );
   }
@@ -2017,7 +2039,7 @@ type SaveMessagesArgs = {
   /**
    * The embeddings to save with the messages.
    */
-  embeddings?: MessageEmbeddings;
+  embeddings?: Omit<MessageEmbeddings, "dimension">;
 };
 
 /**
@@ -2033,12 +2055,24 @@ export async function saveMessages(
     agentName?: string;
   }
 ) {
+  let embeddings: MessageEmbeddings | undefined;
+  if (args.embeddings) {
+    const dimension = args.embeddings.vectors.find((v) => v !== null)?.length;
+    if (dimension) {
+      validateVectorDimension(dimension);
+      embeddings = {
+        model: args.embeddings.model,
+        dimension,
+        vectors: args.embeddings.vectors,
+      };
+    }
+  }
   const result = await ctx.runMutation(component.messages.addMessages, {
     threadId: args.threadId,
     userId: args.userId,
     agentName: args.agentName,
     promptMessageId: args.promptMessageId,
-    embeddings: args.embeddings,
+    embeddings,
     messages: await Promise.all(
       args.messages.map(async (m, i) => {
         const { message, fileIds } = await serializeMessage(ctx, component, m);
@@ -2107,19 +2141,22 @@ export async function saveMessage(
     agentName?: string;
   }
 ) {
-  let embeddings: MessageEmbeddings | undefined;
-  if (args.embedding) {
-    const dimension = args.embedding.vector.length;
-    validateVectorDimension(dimension);
+  let embeddings:
+    | {
+        vectors: number[][];
+        model: string;
+      }
+    | undefined;
+  if (args.embedding && args.embedding.vector) {
     embeddings = {
       model: args.embedding.model,
-      dimension,
       vectors: [args.embedding.vector],
     };
   }
   const { lastMessageId, messages } = await saveMessages(ctx, component, {
     threadId: args.threadId,
     userId: args.userId,
+    agentName: args.agentName,
     messages:
       args.prompt !== undefined
         ? [{ role: "user", content: args.prompt }]

package/src/client/streaming.ts

@@ -8,15 +8,18 @@ import type {
   ProviderOptions,
   StreamArgs,
   StreamDelta,
+  StreamMessage,
   TextStreamPart,
 } from "../validators.js";
 import type { MessageDoc } from "../component/schema.js";
 import type {
   AgentComponent,
   RunActionCtx,
+  RunMutationCtx,
   RunQueryCtx,
   SyncStreamsReturnValue,
 } from "./types.js";
+import { omit } from "convex-helpers";
 
 /**
  * A function that handles fetching stream deltas, used with the React hooks
@@ -41,10 +44,10 @@ export async function syncStreams(
   if (args.streamArgs.kind === "list") {
     return {
       kind: "list",
-      messages: await ctx.runQuery(component.streams.list, {
+      messages: await listStreams(ctx, component, {
         threadId: args.threadId,
         startOrder: args.streamArgs.startOrder,
-        statuses: args.includeStatuses,
+        includeStatuses: args.includeStatuses,
       }),
     };
   } else {
@@ -58,6 +61,55 @@ export async function syncStreams(
   }
 }
 
+export async function abortStream(
+  ctx: RunMutationCtx,
+  component: AgentComponent,
+  args: {
+    reason: string;
+  } & ({ streamId: string } | { threadId: string; order: number })
+): Promise<boolean> {
+  if ("streamId" in args) {
+    return await ctx.runMutation(component.streams.abort, {
+      reason: args.reason,
+      streamId: args.streamId,
+    });
+  } else {
+    return await ctx.runMutation(component.streams.abortByOrder, {
+      reason: args.reason,
+      threadId: args.threadId,
+      order: args.order,
+    });
+  }
+}
+
+/**
+ * List the streaming messages for a thread.
+ * @param ctx A ctx object from a query, mutation, or action.
+ * @param component The agent component, usually `components.agent`.
+ * @param args.threadId The thread to list streams for.
+ * @param args.startOrder The order of the messages in the thread to start listing from.
+ * @param args.includeStatuses The statuses to include in the list.
+ * @returns The streams for the thread.
+ */
+export async function listStreams(
+  ctx: RunQueryCtx,
+  component: AgentComponent,
+  {
+    threadId,
+    startOrder,
+    includeStatuses,
+  }: {
+    threadId: string;
+    startOrder?: number;
+    includeStatuses?: ("streaming" | "finished" | "aborted")[];
+  }
+): Promise<StreamMessage[]> {
+  return ctx.runQuery(component.streams.list, {
+    threadId,
+    startOrder,
+    statuses: includeStatuses,
+  });
+}
 
 export type StreamingOptions = {
   /**
@@ -137,7 +189,6 @@ export class DeltaStreamer {
             ...DEFAULT_STREAMING_OPTIONS,
             ...options,
           };
-    this.metadata = metadata;
     this.#nextParts = [];
     this.#nextOrder = metadata.order ?? 0;
     this.#nextStepOrder = (metadata.stepOrder ?? 0) + 1;
@@ -162,7 +213,7 @@ export class DeltaStreamer {
       this.streamId = await this.ctx.runMutation(
         this.component.streams.create,
         {
-          ...this.metadata,
+          ...omit(this.metadata, ["abortSignal"]),
           order: this.#nextOrder,
           stepOrder: this.#nextStepOrder,
         }

package/src/client/types.ts

@@ -131,7 +131,7 @@ export type StorageOptions = {
   saveOutputMessages?: boolean;
 };
 
-export type GenerationOutputMetadata = { messageId?: string };
+export type GenerationOutputMetadata = { messageId?: string; order?: number };
 
 export type UsageHandler = (
   ctx: RunActionCtx,
@@ -333,9 +333,7 @@ export type OurStreamObjectArgs<T> = StreamObjectArgs<T> &
     "onError" | "onFinish" | "abortSignal"
   >;
 
-type ThreadOutputMetadata = GenerationOutputMetadata & {
-  messageId: string;
-};
+type ThreadOutputMetadata = Required<GenerationOutputMetadata>;
 
 /**
  * The interface for a thread returned from {@link createThread} or {@link continueThread}.

package/src/component/_generated/api.d.ts

@@ -1437,7 +1437,13 @@ export type Mounts = {
       "mutation",
       "public",
       { reason: string; streamId: string },
-      null
+      boolean
+    >;
+    abortByOrder: FunctionReference<
+      "mutation",
+      "public",
+      { order: number; reason: string; threadId: string },
+      boolean
     >;
     addDelta: FunctionReference<
       "mutation",

package/src/component/streams.ts

@@ -151,30 +151,63 @@ export const list = query({
   },
 });
 
-export const abort = mutation({
+export const abortByOrder = mutation({
   args: {
-    streamId: v.id("streamingMessages"),
+    threadId: v.id("threads"),
+    order: v.number(),
     reason: v.string(),
   },
-  returns: v.null(),
+  returns: v.boolean(),
   handler: async (ctx, args) => {
-    const stream = await ctx.db.get(args.streamId);
-    if (!stream) {
-      throw new Error(`Stream not found: ${args.streamId}`);
-    }
-    if (stream.state.kind !== "streaming") {
-      console.warn(
-        `Stream trying to abort but not currently streaming (${stream.state.kind}): ${args.streamId}`
-      );
-      return;
+    const streams = await ctx.db
+      .query("streamingMessages")
+      .withIndex("threadId_state_order_stepOrder", (q) =>
+        q
+          .eq("threadId", args.threadId)
+          .eq("state.kind", "streaming")
+          .eq("order", args.order)
+      )
+      .take(100);
+    for (const stream of streams) {
+      await abortById(ctx, {
+        streamId: stream._id,
+        reason: args.reason,
+      });
     }
-    await cleanupTimeoutFn(ctx, stream);
-    await ctx.db.patch(args.streamId, {
-      state: { kind: "aborted", reason: args.reason },
-    });
+    return streams.length > 0;
+  },
+});
+
+export const abort = mutation({
+  args: {
+    streamId: v.id("streamingMessages"),
+    reason: v.string(),
   },
+  returns: v.boolean(),
+  handler: abortById,
 });
 
+async function abortById(
+  ctx: MutationCtx,
+  args: { streamId: Id<"streamingMessages">; reason: string }
+) {
+  const stream = await ctx.db.get(args.streamId);
+  if (!stream) {
+    throw new Error(`Stream not found: ${args.streamId}`);
+  }
+  if (stream.state.kind !== "streaming") {
+    console.warn(
+      `Stream trying to abort but not currently streaming (${stream.state.kind}): ${args.streamId}`
+    );
+    return false;
+  }
+  await cleanupTimeoutFn(ctx, stream);
+  await ctx.db.patch(args.streamId, {
+    state: { kind: "aborted", reason: args.reason },
+  });
+  return true;
+}
+
 async function cleanupTimeoutFn(
   ctx: MutationCtx,
   stream: Doc<"streamingMessages">

package/src/mapping.ts

@@ -125,7 +125,6 @@ export async function serializeObjectResult(
   const { message, fileIds } = await serializeMessage(ctx, component, {
     role: "assistant" as const,
     content: text,
-    id: result.response.id,
   });
   return {
     messages: [
@@ -187,6 +186,9 @@ export async function serializeContent(
           }
           return { ...part, data };
         }
+        case "tool-result": {
+          return { ...part, result: part.result ?? null };
+        }
         default:
           return part;
       }
@@ -281,6 +283,11 @@ export function guessMimeType(buf: ArrayBuffer | string): string {
   return "application/octet-stream";
 }
 
+/**
+ * Serialize an AI SDK `DataContent` or `URL` to a Convex-serializable format.
+ * @param dataOrUrl - The data or URL to serialize.
+ * @returns The serialized data as an ArrayBuffer or the URL as a string.
+ */
 export function serializeDataOrUrl(
   dataOrUrl: DataContent | URL
 ): ArrayBuffer | string {

package/src/react/deltas.ts

@@ -2,6 +2,7 @@ import type { TextPart, ToolCallPart, ToolResultPart } from "ai";
 import type { MessageDoc } from "../client/index.js";
 import type {
   Message,
+  MessageStatus,
   StreamDelta,
   StreamMessage,
   TextStreamPart,
@@ -99,6 +100,12 @@ export function applyDeltasToStreamMessage(
     cursor = delta.end;
     parts.push(...delta.parts);
   }
+  if (existing && existing.messages.length > 0 && !changed) {
+    const lastMessage = existing.messages.at(-1)!;
+    if (statusFromStreamStatus(streamMessage.status) !== lastMessage.status) {
+      changed = true;
+    }
+  }
   if (!changed) {
     return [
       existing ?? { streamId: streamMessage.streamId, cursor, messages: [] },
@@ -115,6 +122,7 @@ export function applyDeltasToStreamMessage(
     currentMessage = {
       ...lastMessage,
       message: cloneMessageAndContent(lastMessage.message),
+      status: statusFromStreamStatus(streamMessage.status),
     };
   } else {
     const newMessage = createStreamingMessage(
@@ -248,6 +256,21 @@ function getLastContent(message: MessageDoc) {
   return undefined;
 }
 
+function statusFromStreamStatus(
+  status: StreamMessage["status"]
+): MessageStatus {
+  switch (status) {
+    case "streaming":
+      return "pending";
+    case "finished":
+      return "success";
+    case "aborted":
+      return "failed";
+    default:
+      return "pending";
+  }
+}
+
 export function createStreamingMessage(
   threadId: string,
   message: StreamMessage,
@@ -259,9 +282,7 @@ export function createStreamingMessage(
     ...rest,
     _id: `${streamId}-${index}`,
     _creationTime: Date.now(),
-    status: (
-      { streaming: "pending", finished: "success", aborted: "failed" } as const
-    )[message.status],
+    status: statusFromStreamStatus(message.status),
     threadId,
     tool: false,
   };

package/src/react/index.ts

@@ -114,7 +114,7 @@ export function useThreadMessages<
     const streamListMessages =
       streamMessages?.map((m) => ({
         ...m,
-        streaming: !m.status || m.status === "streaming",
+        streaming: !m.status || m.status === "pending",
       })) ?? [];
     return {
       ...paginated,

package/src/react/optimisticallySendMessage.ts

@@ -28,7 +28,7 @@ export function optimisticallySendMessage(
       argsToMatch: { threadId: args.threadId, streamArgs: undefined },
       item: {
         _creationTime: Date.now(),
-        _id: crypto.randomUUID(),
+        _id: randomUUID(),
         order,
         stepOrder,
         status: "pending",
@@ -44,3 +44,13 @@ export function optimisticallySendMessage(
     });
   };
 }
+
+export function randomUUID() {
+  if (typeof crypto !== "undefined") {
+    return crypto.randomUUID();
+  }
+  return (
+    Math.random().toString(36).substring(2, 15) +
+    Math.random().toString(36).substring(2, 15)
+  );
+}

package/src/react/toUIMessages.ts

@@ -9,6 +9,7 @@ export type UIMessage = AIUIMessage & {
   order: number;
   stepOrder: number;
   status: "streaming" | MessageStatus;
+  agentName?: string;
 };
 
 export function toUIMessages(
@@ -36,6 +37,7 @@ export function toUIMessages(
         ...common,
         role: "system",
         content: text,
+        agentName: message.agentName,
         parts: [{ type: "text", text }],
       });
     } else if (coreMessage.role === "user") {
@@ -69,6 +71,7 @@ export function toUIMessages(
         assistantMessage = {
           ...common,
           role: "assistant",
+          agentName: message.agentName,
           content: "",
           parts: [],
         };