@convex-dev/agent 0.0.1-alpha.4 → 0.0.2

package/src/component/schema.ts

@@ -17,16 +17,17 @@ export const schema = defineSchema({
     // the parent thread(s). There are multiple if the thread is a merging of
     // multiple threads.
     parentThreadIds: v.optional(v.array(v.id("threads"))),
-  }).index("status_userId_order", ["status", "userId", "order"]),
+  }).index("userId_status_order", ["userId", "status", "order"]),
   // TODO: text search on title/ summary
   messages: defineTable({
     id: v.optional(v.string()), // external id, e.g. from Vercel AI SDK
     userId: v.optional(v.string()), // useful for future indexes (text search)
-    threadId: v.optional(v.id("threads")),
+    threadId: v.id("threads"),
     parentMessageId: v.optional(v.id("messages")),
     stepId: v.optional(v.id("steps")),
     agentName: v.optional(v.string()),
     message: v.optional(vMessage),
+    error: v.optional(v.string()),
     model: v.optional(v.string()),
     text: v.optional(v.string()),
     embeddingId: v.optional(vVectorId),
@@ -35,7 +36,7 @@ export const schema = defineSchema({
     // Repeats until a non-tool message.
     // Unset if it's not in a thread.
     order: v.number(),
-    stepOrder: v.optional(v.number()),
+    stepOrder: v.number(),
     fileId: v.optional(v.id("files")),
     status: vMessageStatus,
   })
@@ -92,6 +93,16 @@ export const schema = defineSchema({
       "stepOrder",
     ]),
 
+  memories: defineTable({
+    threadId: v.optional(v.id("threads")),
+    userId: v.optional(v.string()),
+    memory: v.string(),
+    embeddingId: v.optional(vVectorId),
+  })
+    .index("threadId", ["threadId"])
+    .index("userId", ["userId"])
+    .index("embeddingId", ["embeddingId"]),
+
   files: defineTable({
     storageId: v.string(),
     hash: v.string(),

package/src/component/vector/index.ts

@@ -1,7 +1,14 @@
 import { paginator } from "convex-helpers/server/pagination";
 import { v } from "convex/values";
-import { mutation, query } from "../_generated/server";
-import { getVectorTableName, vVectorDimension, vVectorId } from "./tables";
+import { ActionCtx, mutation, MutationCtx, query } from "../_generated/server";
+import {
+  EmbeddingsWithoutDenormalizedFields,
+  getVectorTableName,
+  VectorDimension,
+  vEmbeddingsWithoutDenormalizedFields,
+  vVectorDimension,
+  vVectorId,
+} from "./tables";
 import schema from "../schema";
 import { mergedStream } from "convex-helpers/server/stream";
 import { stream } from "convex-helpers/server/stream";
@@ -10,6 +17,7 @@ export const paginate = query({
   args: {
     vectorDimension: vVectorDimension,
     targetModel: v.string(),
+    table: v.optional(v.string()),
     cursor: v.optional(v.string()),
     limit: v.number(),
   },
@@ -23,8 +31,11 @@ export const paginate = query({
     const vectors = await paginator(ctx.db, schema)
       .query(tableName)
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      .withIndex("model_kind_threadId" as any, (q) =>
-        q.eq("model", args.targetModel)
+      .withIndex("model_table_threadId" as any, (q) =>
+        args.table
+          ? // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (q.eq("model", args.targetModel) as any).eq("table", args.table)
+          : q.eq("model", args.targetModel)
       )
       .paginate({
         cursor: args.cursor ?? null,
@@ -82,36 +93,58 @@ export const deleteBatchForThread = mutation({
 export const insertBatch = mutation({
   args: {
     vectorDimension: vVectorDimension,
-    vectors: v.array(
-      v.object({
-        model: v.string(),
-        kind: v.union(v.literal("thread"), v.literal("memory")),
-        userId: v.optional(v.string()),
-        threadId: v.optional(v.string()),
-        vector: v.array(v.number()),
-      })
-    ),
+    vectors: v.array(vEmbeddingsWithoutDenormalizedFields),
   },
   returns: v.null(),
   handler: async (ctx, args) => {
     await Promise.all(
-      args.vectors.map((v) =>
-        ctx.db.insert(getVectorTableName(args.vectorDimension), {
-          model: v.model,
-          kind: v.kind,
-          userId: v.userId,
-          threadId: v.threadId,
-          vector: v.vector,
-          model_kind_userId: v.userId ? [v.model, v.kind, v.userId] : undefined,
-          model_kind_threadId: v.threadId
-            ? [v.model, v.kind, v.threadId]
-            : undefined,
-        })
-      )
+      args.vectors.map((v) => insertVector(ctx, args.vectorDimension, v))
     );
   },
 });
 
+export async function insertVector(
+  ctx: MutationCtx,
+  dimension: VectorDimension,
+  v: EmbeddingsWithoutDenormalizedFields
+) {
+  return ctx.db.insert(getVectorTableName(dimension), {
+    ...v,
+    model_table_userId: v.userId ? [v.model, v.table, v.userId] : undefined,
+    model_table_threadId: v.threadId
+      ? [v.model, v.table, v.threadId]
+      : undefined,
+  });
+}
+
+export function searchVectors(
+  ctx: ActionCtx,
+  vector: number[],
+  args: {
+    dimension: VectorDimension;
+    model: string;
+    table: string;
+    userId?: string;
+    threadId?: string;
+    limit?: number;
+  }
+) {
+  const tableName = getVectorTableName(args.dimension);
+  return ctx.vectorSearch(tableName, "vector", {
+    vector,
+    // TODO: to support more tables, add more "OR" clauses for each.
+    filter: (q) =>
+      args.userId
+        ? q.eq("model_table_userId", [args.model, args.table, args.userId])
+        : q.eq("model_table_threadId", [
+            args.model,
+            args.table,
+            args.threadId!,
+          ]),
+    limit: args.limit,
+  });
+}
+
 export const updateBatch = mutation({
   args: {
     vectors: v.array(

package/src/component/vector/tables.ts

@@ -1,37 +1,69 @@
+import { omit } from "convex-helpers";
 import { literals } from "convex-helpers/validators";
 import {
   defineTable,
   GenericTableSearchIndexes,
+  SchemaDefinition,
   TableDefinition,
 } from "convex/server";
-import { GenericId, ObjectType, v, VId, VObject, VUnion } from "convex/values";
+import {
+  GenericId,
+  Infer,
+  ObjectType,
+  v,
+  VId,
+  VObject,
+  VUnion,
+} from "convex/values";
 import { QueryCtx } from "../_generated/server";
 
 // We only generate embeddings for non-tool, non-system messages
 const embeddings = {
   model: v.string(),
-  kind: v.union(v.literal("thread"), v.literal("memory")),
+  // What table it's stored in. (usually messages or memories)
+  table: v.string(),
   userId: v.optional(v.string()),
   threadId: v.optional(v.string()),
   // not set for private threads
-  model_kind_userId: v.optional(v.array(v.string())),
-  model_kind_threadId: v.optional(v.array(v.string())),
+  model_table_userId: v.optional(v.array(v.string())),
+  model_table_threadId: v.optional(v.array(v.string())),
   vector: v.array(v.number()),
 };
 
+export const vEmbeddingsWithoutDenormalizedFields = v.object(
+  omit(embeddings, ["model_table_userId", "model_table_threadId"])
+);
+export type EmbeddingsWithoutDenormalizedFields = Infer<
+  typeof vEmbeddingsWithoutDenormalizedFields
+>;
+
 function table<D extends number>(dimensions: D): Table<D> {
   return defineTable(embeddings)
     .vectorIndex("vector", {
       vectorField: "vector",
       dimensions,
-      filterFields: ["model_kind_userId", "model_kind_threadId"],
+      filterFields: ["model_table_userId", "model_table_threadId"],
     })
-    .index("model_kind_threadId", ["model", "kind", "threadId"]);
+    .index("model_table_threadId", ["model", "table", "threadId"]);
 }
 
+export type VectorSchema = SchemaDefinition<
+  { [key in VectorTableName]: Table<128> },
+  true
+>;
+
 export const VectorDimensions = [
   128, 256, 512, 768, 1024, 1536, 2048, 3072, 4096,
 ] as const;
+export function validateVectorDimension(
+  dimension: number
+): asserts dimension is VectorDimension {
+  if (!VectorDimensions.includes(dimension as VectorDimension)) {
+    throw new Error(
+      `Unsupported vector dimension${dimension}. Supported: ${VectorDimensions.join(", ")}`
+    );
+  }
+}
 export type VectorDimension = (typeof VectorDimensions)[number];
 export const VectorTableNames = VectorDimensions.map(
   (d) => `embeddings_${d}`

package/src/mapping.ts

@@ -1,10 +1,9 @@
 import {
   convertToCoreMessages,
   coreMessageSchema,
+  GenerateObjectResult,
   type AssistantContent,
-  type CoreAssistantMessage,
   type CoreMessage,
-  type CoreToolMessage,
   type DataContent,
   type GenerateTextResult,
   type StepResult,
@@ -13,8 +12,12 @@ import {
   type Message as UIMessage,
   type UserContent,
 } from "ai";
-import { MessageWithFileAndId, Step } from "./validators";
 import { assert } from "convex-helpers";
+import {
+  MessageWithFileAndId,
+  Step,
+  StepWithMessagesWithFileAndId,
+} from "./validators";
 
 export type SerializeUrlsAndUint8Arrays<T> = T extends URL
   ? string
@@ -115,6 +118,41 @@ export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
   return messages;
 }
 
+export function serializeObjectResult(
+  result: GenerateObjectResult<unknown>
+): StepWithMessagesWithFileAndId {
+  const text = JSON.stringify(result.object);
+  const serializedMessage = serializeMessageWithId({
+    role: "assistant" as const,
+    content: text,
+    id: result.response.id,
+  });
+
+  const messages = [serializedMessage];
+
+  return {
+    messages,
+    step: {
+      text,
+      isContinued: false,
+      stepType: "initial",
+      toolCalls: [],
+      toolResults: [],
+      usage: result.usage,
+      warnings: result.warnings,
+      finishReason: result.finishReason,
+      request: result.request,
+      response: {
+        ...result.response,
+        timestamp: result.response.timestamp.getTime(),
+        messages,
+      },
+      providerMetadata: result.providerMetadata,
+      experimental_providerMetadata: result.experimental_providerMetadata,
+    },
+  };
+}
+
 export function deserializeContent(content: SerializedContent): Content {
   if (typeof content === "string") {
     return content;
@@ -172,23 +210,26 @@ export function promptOrMessagesToCoreMessages(args: {
   if (args.system) {
     messages.push({ role: "system", content: args.system });
   }
-  if (!args.messages) {
-    assert(args.prompt, "messages or prompt is required");
+  assert(args.prompt || args.messages, "messages or prompt is required");
+  if (args.messages) {
+    if (
+      args.messages.some(
+        (m) =>
+          typeof m === "object" &&
+          m !== null &&
+          (m.role === "data" || // UI-only role
+            "toolInvocations" in m || // UI-specific field
+            "parts" in m || // UI-specific field
+            "experimental_attachments" in m)
+      )
+    ) {
+      messages.push(...convertToCoreMessages(args.messages as UIMessage[]));
+    } else {
+      messages.push(...coreMessageSchema.array().parse(args.messages));
+    }
+  }
+  if (args.prompt) {
     messages.push({ role: "user", content: args.prompt });
-  } else if (
-    args.messages.some(
-      (m) =>
-        typeof m === "object" &&
-        m !== null &&
-        (m.role === "data" || // UI-only role
-          "toolInvocations" in m || // UI-specific field
-          "parts" in m || // UI-specific field
-          "experimental_attachments" in m)
-    )
-  ) {
-    messages.push(...convertToCoreMessages(args.messages as UIMessage[]));
-  } else {
-    messages.push(...coreMessageSchema.array().parse(args.messages));
   }
   assert(messages.length > 0, "Messages must contain at least one message");
   return messages;

package/src/validators.ts

@@ -1,9 +1,12 @@
-import { Infer, v } from "convex/values";
+import { Infer, ObjectType, v } from "convex/values";
+import { vVectorDimension } from "./component/vector/tables";
 
 // const deprecated = v.optional(v.any()) as unknown as VNull<unknown, "optional">;
 
 const providerOptions = v.optional(v.record(v.string(), v.any()));
+export type ProviderOptions = Infer<typeof providerOptions>;
 const experimental_providerMetadata = providerOptions;
+export type ProviderMetadata = Infer<typeof experimental_providerMetadata>;
 
 export const vThreadStatus = v.union(
   v.literal("active"),
@@ -185,6 +188,14 @@ export const vResponse = v.object({
   body: v.optional(v.any()),
 });
 
+export const vResponseWithoutMessages = v.object({
+  id: v.string(),
+  timestamp: v.number(),
+  modelId: v.string(),
+  headers: v.optional(v.record(v.string(), v.string())), // clear these?
+  body: v.optional(v.any()),
+});
+
 export const vFinishReason = v.union(
   v.literal("stop"),
   v.literal("length"),
@@ -243,19 +254,30 @@ export const vStep = v.object({
 });
 export type Step = Infer<typeof vStep>;
 
-export const vStepWithMessagesWithFileAndId = v.object({
+export const vStepWithMessages = v.object({
   step: vStep,
   messages: v.array(vMessageWithFileAndId),
 });
-export type StepWithMessagesWithFileAndId = Infer<
-  typeof vStepWithMessagesWithFileAndId
->;
+export type StepWithMessagesWithFileAndId = Infer<typeof vStepWithMessages>;
 
+export const vObjectResult = v.object({
+  request: vRequest,
+  response: vResponseWithoutMessages,
+  finishReason: vFinishReason,
+  usage: v.optional(v.any()),
+  object: v.any(),
+  error: v.optional(v.string()),
+  warnings: v.optional(v.array(vLanguageModelV1CallWarning)),
+  providerMetadata: providerOptions,
+  experimental_providerMetadata,
+});
+export type ObjectResult = Infer<typeof vObjectResult>;
 export const vSearchOptions = v.object({
   vector: v.optional(v.array(v.number())),
   vectorModel: v.optional(v.string()),
   text: v.optional(v.string()),
   limit: v.number(),
+  vectorScoreThreshold: v.optional(v.number()),
   messageRange: v.optional(v.object({ before: v.number(), after: v.number() })),
 });
 export type SearchOptions = Infer<typeof vSearchOptions>;
@@ -279,15 +301,93 @@ export const vStorageOptions = v.object({
   saveAllOutputMessages: v.optional(v.boolean()),
 });
 
-export const vThreadArgs = v.object({
-  maxSteps: v.optional(v.number()),
+const vPromptFields = {
+  system: v.optional(v.string()),
   prompt: v.optional(v.string()),
   messages: v.optional(v.array(vMessage)),
+};
+
+export const vCallSettingsFields = {
+  maxTokens: v.optional(v.number()),
+  temperature: v.optional(v.number()),
+  topP: v.optional(v.number()),
+  topK: v.optional(v.number()),
+  presencePenalty: v.optional(v.number()),
+  frequencyPenalty: v.optional(v.number()),
+  seed: v.optional(v.number()),
+  maxRetries: v.optional(v.number()),
+  headers: v.optional(v.record(v.string(), v.string())),
+};
+export type CallSettings = ObjectType<typeof vCallSettingsFields>;
+
+export const vTextArgs = v.object({
+  ...vCallSettingsFields,
+  ...vPromptFields,
+  toolChoice: v.optional(
+    v.union(
+      v.literal("auto"),
+      v.literal("none"),
+      v.literal("required"),
+      v.object({
+        type: v.literal("tool"),
+        toolName: v.string(),
+      })
+    )
+  ),
+  maxSteps: v.optional(v.number()),
+  experimental_continueSteps: v.optional(v.boolean()),
+  providerOptions,
+  experimental_providerMetadata,
 });
+export type TextArgs = Infer<typeof vTextArgs>;
 
-export const vObjectArgs = v.object({
-  output: v.optional(v.any()),
-  mode: v.optional(v.literal("json")),
-  prompt: v.optional(v.string()),
-  messages: v.optional(v.array(vMessage)),
+const objectArgsCommonFields = {
+  ...vCallSettingsFields,
+  ...vPromptFields,
+  providerOptions,
+  experimental_providerMetadata,
+};
+
+export const vSafeObjectArgs = v.union(
+  v.object({
+    ...objectArgsCommonFields,
+    output: v.optional(v.literal("object")),
+    schema: v.any(), // JSON schema
+    schemaName: v.optional(v.string()),
+    schemaDescription: v.optional(v.string()),
+    mode: v.optional(
+      v.union(v.literal("auto"), v.literal("json"), v.literal("tool"))
+    ),
+  }),
+  v.object({
+    ...objectArgsCommonFields,
+    output: v.optional(v.literal("array")),
+    schema: v.any(), // JSON schema
+    schemaName: v.optional(v.string()),
+    schemaDescription: v.optional(v.string()),
+    mode: v.optional(
+      v.union(v.literal("auto"), v.literal("json"), v.literal("tool"))
+    ),
+  }),
+  v.object({
+    ...objectArgsCommonFields,
+    output: v.optional(v.literal("enum")),
+    enum: v.array(v.string()),
+    mode: v.optional(
+      v.union(v.literal("auto"), v.literal("json"), v.literal("tool"))
+    ),
+  }),
+  v.object({
+    ...objectArgsCommonFields,
+    output: v.literal("no-schema"),
+    mode: v.optional(v.literal("json")),
+  })
+);
+export type SafeObjectArgs = Infer<typeof vSafeObjectArgs>;
+
+export const vEmbeddingsWithMetadata = v.object({
+  vectors: v.array(v.union(v.array(v.number()), v.null())),
+  dimension: vVectorDimension,
+  model: v.string(),
 });
+export type EmbeddingsWithMetadata = Infer<typeof vEmbeddingsWithMetadata>;