@convex-dev/agent 0.0.16 → 0.0.17-alpha.2

package/src/component/schema.ts

@@ -9,6 +9,10 @@ import {
   vSource,
   vLanguageModelV1CallWarning,
   vFinishReason,
+  vProviderOptions,
+  vProviderMetadata,
+  vReasoningDetails,
+  vFile,
 } from "../validators.js";
 import { typedV } from "convex-helpers/validators";
 import vectorTables, { vVectorId } from "./vector/tables.js";
@@ -29,35 +33,44 @@ export const schema = defineSchema({
     id: v.optional(v.string()), // external id, e.g. from Vercel AI SDK
     userId: v.optional(v.string()), // useful for future indexes (text search)
     threadId: v.id("threads"),
+    // TODO: is this redunant with message at last step @ order - 1?
     parentMessageId: v.optional(v.id("messages")),
     stepId: v.optional(v.id("steps")),
-    agentName: v.optional(v.string()),
-    message: v.optional(vMessage),
+    // Repeats until a non-tool message.
+    order: v.number(),
+    stepOrder: v.number(),
+    embeddingId: v.optional(vVectorId),
     error: v.optional(v.string()),
+    status: vMessageStatus,
+
+    // Context on how it was generated
+    agentName: v.optional(v.string()),
     model: v.optional(v.string()),
     provider: v.optional(v.string()),
+    providerOptions: v.optional(vProviderOptions), // Sent to model
+
+    // The result
+    message: v.optional(vMessage),
+    // Convenience fields extracted from the message
+    tool: v.boolean(), // either tool call (assistant) or tool result (tool)
     text: v.optional(v.string()),
-    embeddingId: v.optional(vVectorId),
-    // TODO: add sub-messages back in? or be able to skip them?
-    tool: v.boolean(),
-    // Repeats until a non-tool message.
-    // Unset if it's not in a thread.
-    order: v.number(),
-    stepOrder: v.number(),
+    files: v.optional(v.array(vFile)),
+
+    // Result metadata
     usage: v.optional(vUsage),
-    finishReason: v.optional(vFinishReason),
-    providerMetadata: v.optional(v.record(v.string(), v.any())),
+    providerMetadata: v.optional(vProviderMetadata), // Received from model
     sources: v.optional(v.array(vSource)),
     reasoning: v.optional(v.string()),
+    reasoningDetails: v.optional(vReasoningDetails),
     warnings: v.optional(v.array(vLanguageModelV1CallWarning)),
-    fileId: v.optional(v.id("files")),
-    status: vMessageStatus,
+    finishReason: v.optional(vFinishReason),
   })
     // Allows finding successful visible messages in order
     // Also surface pending messages separately to e.g. stream
     .index("threadId_status_tool_order_stepOrder", [
       "threadId",
       "status",
+      // TODO: we might not need this to be in the index..
       "tool",
       "order",
       "stepOrder",
@@ -124,6 +137,12 @@ export const schema = defineSchema({
     .index("hash", ["hash"])
     .index("refcount", ["refcount"]),
   ...vectorTables,
+  // To authenticate playground usage
+  // Delete a key to invalidate it
+  // Provide a name to easily identify it / invalidate by name
+  apiKeys: defineTable({
+    name: v.optional(v.string()),
+  }).index("name", ["name"]),
 });
 
 export const vv = typedV(schema);

package/src/component/threads.ts

@@ -0,0 +1,184 @@
+import { assert, omit, pick } from "convex-helpers";
+import { paginator } from "convex-helpers/server/pagination";
+import { partial } from "convex-helpers/validators";
+import { ObjectType } from "convex/values";
+import { paginationResultValidator } from "../validators.js";
+import { api, internal } from "./_generated/api.js";
+import { Doc } from "./_generated/dataModel.js";
+import {
+  action,
+  internalMutation,
+  mutation,
+  MutationCtx,
+  query,
+} from "./_generated/server.js";
+import { schema, v } from "./schema.js";
+import { paginationOptsValidator } from "convex/server";
+import { deleteMessage } from "./messages.js";
+import { ThreadDoc, vThreadDoc } from "../client/index.js";
+
+function publicThreadOrNull(thread: Doc<"threads"> | null): ThreadDoc | null {
+  if (thread === null) {
+    return null;
+  }
+  return publicThread(thread);
+}
+
+function publicThread(thread: Doc<"threads">): ThreadDoc {
+  return omit(thread, ["defaultSystemPrompt", "parentThreadIds", "order"]);
+}
+
+export const getThread = query({
+  args: { threadId: v.id("threads") },
+  handler: async (ctx, args) => {
+    return publicThreadOrNull(await ctx.db.get(args.threadId));
+  },
+  returns: v.union(vThreadDoc, v.null()),
+});
+
+export const listThreadsByUserId = query({
+  args: {
+    userId: v.string(),
+    order: v.optional(v.union(v.literal("asc"), v.literal("desc"))),
+    paginationOpts: v.optional(paginationOptsValidator),
+  },
+  handler: async (ctx, args) => {
+    const threads = await paginator(ctx.db, schema)
+      .query("threads")
+      .withIndex("userId", (q) => q.eq("userId", args.userId))
+      .order(args.order ?? "desc")
+      .paginate(args.paginationOpts ?? { cursor: null, numItems: 100 });
+    return {
+      ...threads,
+      page: threads.page.map(publicThread),
+    };
+  },
+  returns: paginationResultValidator(vThreadDoc),
+});
+
+const vThread = schema.tables.threads.validator;
+
+export const createThread = mutation({
+  args: omit(vThread.fields, ["order", "status"]),
+  handler: async (ctx, args) => {
+    const threadId = await ctx.db.insert("threads", {
+      ...args,
+      status: "active",
+    });
+    return publicThread((await ctx.db.get(threadId))!);
+  },
+  returns: vThreadDoc,
+});
+
+export const updateThread = mutation({
+  args: {
+    threadId: v.id("threads"),
+    patch: v.object(
+      partial(pick(vThread.fields, ["title", "summary", "status"]))
+    ),
+  },
+  handler: async (ctx, args) => {
+    const thread = await ctx.db.get(args.threadId);
+    assert(thread, `Thread ${args.threadId} not found`);
+    await ctx.db.patch(args.threadId, args.patch);
+    return publicThread((await ctx.db.get(args.threadId))!);
+  },
+  returns: vThreadDoc,
+});
+
+// When we expose this, we need to also hide all the messages and steps
+// export const archiveThread = mutation({
+//   args: { threadId: v.id("threads") },
+//   handler: async (ctx, args) => {
+//     const thread = await ctx.db.get(args.threadId);
+//     assert(thread, `Thread ${args.threadId} not found`);
+//     await ctx.db.patch(args.threadId, { status: "archived" });
+//     return publicThread((await ctx.db.get(args.threadId))!);
+//   },
+//   returns: vThreadDoc,
+// });
+
+// TODO: delete thread
+
+const deleteThreadArgs = {
+  threadId: v.id("threads"),
+  cursor: v.optional(v.string()),
+  limit: v.optional(v.number()),
+};
+type DeleteThreadArgs = ObjectType<typeof deleteThreadArgs>;
+const deleteThreadReturns = {
+  cursor: v.string(),
+  isDone: v.boolean(),
+};
+type DeleteThreadReturns = ObjectType<typeof deleteThreadReturns>;
+
+/**
+ * Use this to delete a thread and everything it contains.
+ * It will try to delete all pages synchronously.
+ * If it times out or fails, you'll have to run it again.
+ */
+export const deleteAllForThreadIdSync = action({
+  args: deleteThreadArgs,
+  handler: async (ctx, args) => {
+    let cursor = args.cursor;
+    while (true) {
+      const result = await ctx.runMutation(
+        internal.threads._deletePageForThreadId,
+        { threadId: args.threadId, cursor, limit: args.limit }
+      );
+      if (result.isDone) {
+        break;
+      }
+      cursor = result.cursor;
+    }
+  },
+  returns: v.null(),
+});
+
+export const _deletePageForThreadId = internalMutation({
+  args: deleteThreadArgs,
+  handler: deletePageForThreadIdHandler,
+  returns: deleteThreadReturns,
+});
+
+/**
+ * Use this to delete a thread and everything it contains.
+ * It will continue deleting pages asynchronously.
+ */
+export const deleteAllForThreadIdAsync = mutation({
+  args: deleteThreadArgs,
+  handler: async (ctx, args) => {
+    const result = await deletePageForThreadIdHandler(ctx, args);
+    if (!result.isDone) {
+      await ctx.scheduler.runAfter(0, api.threads.deleteAllForThreadIdAsync, {
+        threadId: args.threadId,
+        cursor: result.cursor,
+      });
+    }
+    return result;
+  },
+  returns: deleteThreadReturns,
+});
+
+async function deletePageForThreadIdHandler(
+  ctx: MutationCtx,
+  args: DeleteThreadArgs
+): Promise<DeleteThreadReturns> {
+  const messages = await paginator(ctx.db, schema)
+    .query("messages")
+    .withIndex("threadId_status_tool_order_stepOrder", (q) =>
+      q.eq("threadId", args.threadId)
+    )
+    .paginate({
+      numItems: args.limit ?? 100,
+      cursor: args.cursor ?? null,
+    });
+  await Promise.all(messages.page.map((m) => deleteMessage(ctx, m)));
+  if (messages.isDone) {
+    await ctx.db.delete(args.threadId);
+  }
+  return {
+    cursor: messages.continueCursor,
+    isDone: messages.isDone,
+  };
+}

package/src/component/users.ts

@@ -0,0 +1,145 @@
+import { paginator } from "convex-helpers/server/pagination";
+import { nullable } from "convex-helpers/validators";
+import { ObjectType } from "convex/values";
+import { internal } from "./_generated/api.js";
+import {
+  action,
+  internalMutation,
+  mutation,
+  MutationCtx,
+  query,
+} from "./_generated/server.js";
+import { schema, v } from "./schema.js";
+import { deleteMessage } from "./messages.js";
+import { paginationOptsValidator } from "convex/server";
+import { stream } from "convex-helpers/server/stream";
+import { paginationResultValidator } from "../validators.js";
+
+// Note: it only searches for users with threads
+export const listUsersWithThreads = query({
+  args: {
+    paginationOpts: paginationOptsValidator,
+  },
+  handler: async (ctx, args) => {
+    const results = await stream(ctx.db, schema)
+      .query("threads")
+      .withIndex("userId", (q) => q)
+      .filterWith(async (q) => !!q.userId)
+      .distinct(["userId"])
+      .paginate(args.paginationOpts);
+    return {
+      ...results,
+      page: results.page.map((t) => t.userId).filter((t): t is string => !!t),
+    };
+  },
+  returns: paginationResultValidator(v.string()),
+});
+
+export const deleteAllForUserId = action({
+  args: { userId: v.string() },
+  handler: async (ctx, args) => {
+    let messagesCursor = null;
+    let threadsCursor = null;
+    let isDone = false;
+    while (!isDone) {
+      const result: {
+        messagesCursor: string;
+        threadsCursor: string | null;
+        isDone: boolean;
+      } = await ctx.runMutation(internal.users._deletePageForUserId, {
+        userId: args.userId,
+        messagesCursor,
+        threadsCursor,
+      });
+      messagesCursor = result.messagesCursor;
+      threadsCursor = result.threadsCursor;
+      isDone = result.isDone;
+    }
+  },
+  returns: v.null(),
+});
+
+export const deleteAllForUserIdAsync = mutation({
+  args: {
+    userId: v.string(),
+  },
+  handler: async (ctx, args) => {
+    const isDone = await deleteAllForUserIdAsyncHandler(ctx, {
+      userId: args.userId,
+      messagesCursor: null,
+      threadsCursor: null,
+    });
+    return isDone;
+  },
+  returns: v.boolean(),
+});
+
+const deleteAllArgs = {
+  userId: v.string(),
+  messagesCursor: nullable(v.string()),
+  threadsCursor: nullable(v.string()),
+};
+type DeleteAllArgs = ObjectType<typeof deleteAllArgs>;
+const deleteAllReturns = {
+  messagesCursor: v.string(),
+  threadsCursor: nullable(v.string()),
+  isDone: v.boolean(),
+};
+type DeleteAllReturns = ObjectType<typeof deleteAllReturns>;
+
+export const _deleteAllForUserIdAsync = internalMutation({
+  args: deleteAllArgs,
+  handler: deleteAllForUserIdAsyncHandler,
+  returns: v.boolean(),
+});
+
+async function deleteAllForUserIdAsyncHandler(
+  ctx: MutationCtx,
+  args: DeleteAllArgs
+): Promise<boolean> {
+  const result = await deletePageForUserId(ctx, args);
+  if (!result.isDone) {
+    await ctx.scheduler.runAfter(0, internal.users._deleteAllForUserIdAsync, {
+      userId: args.userId,
+      messagesCursor: result.messagesCursor,
+      threadsCursor: result.threadsCursor,
+    });
+  }
+  return result.isDone;
+}
+
+export const _deletePageForUserId = internalMutation({
+  args: deleteAllArgs,
+  handler: deletePageForUserId,
+  returns: deleteAllReturns,
+});
+async function deletePageForUserId(
+  ctx: MutationCtx,
+  args: DeleteAllArgs
+): Promise<DeleteAllReturns> {
+  const threads = await paginator(ctx.db, schema)
+    .query("threads")
+    .withIndex("userId", (q) => q.eq("userId", args.userId))
+    .order("desc")
+    .paginate({
+      numItems: 100,
+      cursor: args.threadsCursor ?? null,
+    });
+  await Promise.all(threads.page.map((c) => ctx.db.delete(c._id)));
+  const messages = await paginator(ctx.db, schema)
+    .query("messages")
+    .withIndex("userId_status_tool_order_stepOrder", (q) =>
+      q.eq("userId", args.userId)
+    )
+    .order("desc")
+    .paginate({
+      numItems: 100,
+      cursor: args.messagesCursor ?? null,
+    });
+  await Promise.all(messages.page.map((m) => deleteMessage(ctx, m)));
+  return {
+    messagesCursor: messages.continueCursor,
+    threadsCursor: threads.continueCursor,
+    isDone: messages.isDone,
+  };
+}

package/src/component/vector/index.ts

@@ -65,14 +65,13 @@ export const deleteBatchForThread = mutation({
   handler: async (ctx, args) => {
     const tableName = getVectorTableName(args.vectorDimension);
     const vectors = await mergedStream(
-      ["thread", "memory"].map((kind) =>
+      ["thread", "memory"].map((table) =>
         stream(ctx.db, schema)
           .query(tableName)
-          // eslint-disable-next-line @typescript-eslint/no-explicit-any
-          .withIndex("model_kind_threadId" as any, (q) =>
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            (q.eq("model", args.model) as any)
-              .eq("kind", kind)
+          .withIndex("model_table_threadId", (q) =>
+            q
+              .eq("model", args.model)
+              .eq("table", table)
               .eq("threadId", args.threadId)
           )
       ),

package/src/component/vector/tables.ts

@@ -47,6 +47,21 @@ function table<D extends number>(dimensions: D): Table<D> {
     .index("model_table_threadId", ["model", "table", "threadId"]);
 }
 
+type Table<D extends number> = TableDefinition<
+  VObject<ObjectType<typeof embeddings>, typeof embeddings>,
+  { model_table_threadId: ["model", "table", "threadId", "_creationTime"] },
+  GenericTableSearchIndexes,
+  VectorIndex<D>
+>;
+
+type VectorIndex<D extends number> = {
+  vector: {
+    vectorField: "vector";
+    dimensions: D;
+    filterFields: "model_table_userId" | "model_table_threadId";
+  };
+};
+
 export type VectorSchema = SchemaDefinition<
   { [key in VectorTableName]: Table<128> },
   true
@@ -80,21 +95,6 @@ export const vVectorId = v.union(
   VId<(typeof VectorTableNames)[number]>[]
 >;
 
-type Table<D extends number> = TableDefinition<
-  VObject<ObjectType<typeof embeddings>, typeof embeddings>,
-  { id: ["id"] },
-  GenericTableSearchIndexes,
-  VectorIndex<D>
->;
-
-type VectorIndex<D extends number> = {
-  vector: {
-    vectorField: "vector";
-    dimensions: D;
-    filterFields: string;
-  };
-};
-
 export function getVectorTableName(dimension: VectorDimension) {
   return `embeddings_${dimension}` as VectorTableName;
 }

package/src/mapping.ts

@@ -11,6 +11,7 @@ import {
   type Message as AIMessage,
   type UserContent,
 } from "ai";
+import type { FileUIPart } from "@ai-sdk/ui-utils";
 import { assert } from "convex-helpers";
 import {
   MessageWithMetadata,
@@ -39,9 +40,11 @@ export type SerializedMessage = SerializeUrlsAndUint8Arrays<CoreMessage>;
 export function serializeMessage(
   messageWithId: CoreMessage & { id?: string }
 ): SerializedMessage {
-  const { id: _, ...message } = messageWithId;
+  const { id: _, experimental_providerMetadata, ...message } = messageWithId;
   const content = message.content;
   return {
+    // for backwards compatibility
+    providerOptions: experimental_providerMetadata,
     ...message,
     content: serializeContent(content),
   } as SerializedMessage;
@@ -90,6 +93,7 @@ export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
     provider: metadata.provider,
     providerMetadata: step.providerMetadata,
     reasoning: step.reasoning,
+    reasoningDetails: step.reasoningDetails,
     usage: step.usage,
     warnings: step.warnings,
     finishReason: step.finishReason,
@@ -101,33 +105,41 @@ export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
     step.toolResults.length > 0
       ? step.response.messages.slice(-2)
       : step.response.messages.slice(-1)
-  ).map((message) => ({
-    message: serializeMessage(message),
-    id: message.id,
-    ...(message.role === "tool" ? toolFields : assistantFields),
-    // fileId: message.fileId,
-  }));
+  ).map(
+    (message): MessageWithMetadata => ({
+      message: serializeMessage(message),
+      id: message.id,
+      ...(message.role === "tool" ? toolFields : assistantFields),
+      text: step.text,
+      // fileId: message.fileId,
+      files: step.files.map((file) => ({
+        mimeType: file.mimeType,
+        data: serializeDataOrUrl(file.uint8Array ?? file.base64),
+        // TODO: if the file is big, store it and populate url, fileId
+      })),
+    })
+  );
   return messages;
 }
 
 export function serializeObjectResult(
-  step: GenerateObjectResult<unknown>,
+  result: GenerateObjectResult<unknown>,
   metadata: { model: string; provider: string }
 ): StepWithMessagesWithMetadata {
-  const text = JSON.stringify(step.object);
+  const text = JSON.stringify(result.object);
 
   return {
     messages: [
       {
         message: { role: "assistant" as const, content: text },
-        id: step.response.id,
+        id: result.response.id,
         model: metadata.model,
         provider: metadata.provider,
-        providerMetadata: step.providerMetadata,
-        finishReason: step.finishReason,
+        providerMetadata: result.providerMetadata,
+        finishReason: result.finishReason,
         text,
-        usage: step.usage,
-        warnings: step.warnings,
+        usage: result.usage,
+        warnings: result.warnings,
       },
     ],
     step: {
@@ -136,23 +148,22 @@ export function serializeObjectResult(
       stepType: "initial",
       toolCalls: [],
       toolResults: [],
-      usage: step.usage,
-      warnings: step.warnings,
-      finishReason: step.finishReason,
-      request: step.request,
+      usage: result.usage,
+      warnings: result.warnings,
+      finishReason: result.finishReason,
+      providerMetadata: result.providerMetadata,
+      request: result.request,
       response: {
-        ...step.response,
-        timestamp: step.response.timestamp.getTime(),
+        ...result.response,
+        timestamp: result.response.timestamp.getTime(),
         messages: [
           serializeMessageWithId({
             role: "assistant" as const,
             content: text,
-            id: step.response.id,
+            id: result.response.id,
           }),
         ],
       },
-      providerMetadata: step.providerMetadata,
-      experimental_providerMetadata: step.experimental_providerMetadata,
     },
   };
 }
@@ -161,16 +172,19 @@ export function serializeContent(content: Content): SerializedContent {
   if (typeof content === "string") {
     return content;
   }
-  const serialized = content.map((part) => {
-    switch (part.type) {
-      case "image":
-        return { ...part, image: serializeDataOrUrl(part.image) };
-      case "file":
-        return { ...part, file: serializeDataOrUrl(part.data) };
-      default:
-        return part;
+  const serialized = content.map(
+    ({ experimental_providerMetadata, ...rest }) => {
+      const part = { providerOptions: experimental_providerMetadata, ...rest };
+      switch (part.type) {
+        case "image":
+          return { ...part, image: serializeDataOrUrl(part.image) };
+        case "file":
+          return { ...part, file: serializeDataOrUrl(part.data) };
+        default:
+          return part;
+      }
     }
-  });
+  );
   return serialized as SerializedContent;
 }
 
@@ -222,6 +236,25 @@ function deserializeUrl(urlOrString: string | ArrayBuffer): URL | DataContent {
   return urlOrString;
 }
 
+export function toUIFilePart(file: {
+  data?: ArrayBuffer | string;
+  url?: string;
+  mimeType: string;
+}): FileUIPart {
+  return {
+    type: "file",
+    data:
+      file.data instanceof ArrayBuffer
+        ? encodeBase64(file.data)
+        : file.url ?? file.data ?? "",
+    mimeType: file.mimeType,
+  };
+}
+
+function encodeBase64(data: ArrayBuffer): string {
+  return Buffer.from(data).toString("base64");
+}
+
 export function promptOrMessagesToCoreMessages(args: {
   prompt?: string;
   messages?: CoreMessage[] | AIMessageWithoutId[];