@convex-dev/agent 0.0.1-alpha.0

package/src/component/schema.ts

@@ -0,0 +1,101 @@
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
+import { vChatStatus, vMessage, vMessageStatus, vStep } from "../validators";
+import { typedV } from "convex-helpers/validators";
+import vectorTables, { vVectorId } from "./vector/tables";
+
+export const schema = defineSchema({
+  chats: defineTable({
+    userId: v.optional(v.string()), // Unset for anonymous
+    order: v.optional(v.number()), // within a domain
+    // TODO: is this bubbling up in continue?
+    defaultSystemPrompt: v.optional(v.string()),
+    title: v.optional(v.string()),
+    summary: v.optional(v.string()),
+    status: vChatStatus,
+    // If this is a chat continuation, we can use this to find context from
+    // the parent chat(s). There are multiple if the chat is a merging of
+    // multiple chats.
+    parentChatIds: v.optional(v.array(v.id("chats"))),
+  }).index("status_userId_order", ["status", "userId", "order"]),
+  // TODO: text search on title/ summary
+  messages: defineTable({
+    id: v.optional(v.string()), // external id, e.g. from Vercel AI SDK
+    userId: v.optional(v.string()), // useful for future indexes (text search)
+    chatId: v.id("chats"),
+    threadId: v.optional(v.id("messages")),
+    stepId: v.optional(v.id("steps")),
+    agentName: v.optional(v.string()),
+    message: v.optional(vMessage),
+    model: v.optional(v.string()),
+    text: v.optional(v.string()),
+    embeddingId: v.optional(vVectorId),
+    // TODO: add sub-messages back in? or be able to skip them?
+    tool: v.boolean(),
+    // Repeats until a non-tool message.
+    // Set when the message is finished
+    order: v.optional(v.number()),
+    stepOrder: v.optional(v.number()),
+    fileId: v.optional(v.id("files")),
+    status: vMessageStatus,
+  })
+    // Allows finding successful visible messages in order
+    // Also surface pending messages separately to e.g. stream
+    .index("chatId_status_tool_order_stepOrder", [
+      "chatId",
+      "status",
+      "tool",
+      "order",
+      "stepOrder",
+    ])
+    // Allows finding all threaded messages in order
+    // Allows finding all failed messages to evaluate
+    // .index("status_threadId_order_stepOrder", [
+    //   "status",
+    //   "threadId",
+    //   "order",
+    //   "stepOrder",
+    // ])
+    // Allows text search on message content
+    .searchIndex("text_search", {
+      searchField: "text",
+      filterFields: ["userId", "chatId"],
+    })
+    // Allows finding messages by vector embedding id
+    .index("embeddingId", ["embeddingId"]),
+
+  steps: defineTable({
+    chatId: v.id("chats"),
+    // Could be different from the order if we fail.
+    parentMessageId: v.id("messages"),
+    order: v.number(), // parent message order
+    stepOrder: v.number(), // step order
+    step: vStep,
+    status: vMessageStatus,
+  })
+    .index("status_chatId_order_stepOrder", [
+      "status",
+      "chatId",
+      "order",
+      "stepOrder",
+    ])
+    .index("parentMessageId_order_stepOrder", [
+      "parentMessageId",
+      "order",
+      "stepOrder",
+    ]),
+
+  files: defineTable({
+    storageId: v.string(),
+    hash: v.string(),
+    refcount: v.number(),
+  })
+    .index("hash", ["hash"])
+    .index("refcount", ["refcount"]),
+  ...vectorTables,
+});
+
+export const vv = typedV(schema);
+export { vv as v };
+
+export default schema;

package/src/component/setup.test.ts

@@ -0,0 +1,5 @@
+/// <reference types="vite/client" />
+import { test } from "vitest";
+export const modules = import.meta.glob("./**/*.*s");
+
+test("setup", () => {});

package/src/component/vector/tables.ts

@@ -0,0 +1,92 @@
+import { literals } from "convex-helpers/validators";
+import {
+  defineTable,
+  GenericTableSearchIndexes,
+  TableDefinition,
+} from "convex/server";
+import { GenericId, ObjectType, v, VId, VObject, VUnion } from "convex/values";
+import { QueryCtx } from "../_generated/server";
+
+// We only generate embeddings for non-tool, non-system messages
+const embeddings = {
+  model: v.string(),
+  kind: v.union(v.literal("chat"), v.literal("memory")),
+  userId: v.optional(v.string()),
+  chatId: v.optional(v.string()),
+  // not set for private chats
+  model_kind_userId: v.optional(v.array(v.string())),
+  model_kind_chatId: v.optional(v.array(v.string())),
+  vector: v.array(v.number()),
+};
+
+function table<D extends number>(dimensions: D): Table<D> {
+  return defineTable(embeddings).vectorIndex("vector", {
+    vectorField: "vector",
+    dimensions,
+    filterFields: ["model_kind_userId", "model_kind_chatId"],
+  });
+}
+
+export const VectorDimensions = [
+  128, 256, 512, 768, 1024, 1536, 2048, 3072, 4096,
+] as const;
+export type VectorDimension = (typeof VectorDimensions)[number];
+export const VectorTableNames = VectorDimensions.map(
+  (d) => `embeddings_${d}`
+) as `embeddings_${(typeof VectorDimensions)[number]}`[];
+export type VectorTableName = (typeof VectorTableNames)[number];
+export type VectorTableId = GenericId<(typeof VectorTableNames)[number]>;
+
+export const vVectorDimension = literals(...VectorDimensions);
+export const vVectorTableName = literals(...VectorTableNames);
+export const vVectorId = v.union(
+  ...VectorTableNames.map((name) => v.id(name))
+) as VUnion<
+  GenericId<(typeof VectorTableNames)[number]>,
+  VId<(typeof VectorTableNames)[number]>[]
+>;
+
+type Table<D extends number> = TableDefinition<
+  VObject<ObjectType<typeof embeddings>, typeof embeddings>,
+  { id: ["id"] },
+  GenericTableSearchIndexes,
+  VectorIndex<D>
+>;
+
+type VectorIndex<D extends number> = {
+  vector: {
+    vectorField: "vector";
+    dimensions: D;
+    filterFields: string;
+  };
+};
+
+export function getVectorTableName(dimension: VectorDimension) {
+  return `embeddings_${dimension}` as VectorTableName;
+}
+export function getVectorIdInfo(ctx: QueryCtx, id: VectorTableId) {
+  for (const dimension of VectorDimensions) {
+    const tableName = getVectorTableName(dimension);
+    if (ctx.db.normalizeId(tableName, id)) {
+      return { tableName, dimension };
+    }
+  }
+  throw new Error(`Unknown vector table id: ${id}`);
+}
+
+const tables: {
+  [K in keyof typeof VectorDimensions &
+    number as `embeddings_${(typeof VectorDimensions)[K]}`]: Table<
+    (typeof VectorDimensions)[K]
+  >;
+} = Object.fromEntries(
+  VectorDimensions.map((dimensions) => [
+    `embeddings_${dimensions}`,
+    table(dimensions),
+  ])
+) as Record<
+  `embeddings_${(typeof VectorDimensions)[number]}`,
+  Table<(typeof VectorDimensions)[number]>
+>;
+
+export default tables;

package/src/mapping.ts

@@ -0,0 +1,160 @@
+import type {
+  AssistantContent,
+  CoreAssistantMessage,
+  CoreMessage,
+  CoreToolMessage,
+  DataContent,
+  GenerateTextResult,
+  StepResult,
+  ToolContent,
+  ToolSet,
+  UserContent,
+} from "ai";
+import { MessageWithFileAndId, Step } from "./validators";
+
+export type SerializeUrlsAndUint8Arrays<T> = T extends URL
+  ? string
+  : T extends Uint8Array | ArrayBufferLike
+    ? ArrayBuffer
+    : T extends Array<infer Inner>
+      ? Array<SerializeUrlsAndUint8Arrays<Inner>>
+      : // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        T extends Record<string, any>
+        ? { [K in keyof T]: SerializeUrlsAndUint8Arrays<T[K]> }
+        : T;
+
+export type Content = UserContent | AssistantContent | ToolContent;
+export type SerializedContent = SerializeUrlsAndUint8Arrays<Content>;
+
+export type SerializedMessage = SerializeUrlsAndUint8Arrays<CoreMessage>;
+
+export function serializeMessage(
+  messageWithId: CoreMessage & { id?: string }
+): SerializedMessage {
+  const { id: _, ...message } = messageWithId;
+  const content = message.content;
+  return {
+    ...message,
+    content: serializeContent(content),
+  } as SerializedMessage;
+}
+
+export function serializeMessageWithId(
+  messageWithId: CoreMessage & { id?: string }
+): { message: SerializedMessage; id: string | undefined } {
+  return { message: serializeMessage(messageWithId), id: messageWithId.id };
+}
+
+export function serializeContent(content: Content): SerializedContent {
+  if (typeof content === "string") {
+    return content;
+  }
+  const serialized = content.map((part) => {
+    switch (part.type) {
+      case "image":
+        return { ...part, image: serializeDataOrUrl(part.image) };
+      case "file":
+        return { ...part, file: serializeDataOrUrl(part.data) };
+      default:
+        return part;
+    }
+  });
+  return serialized as SerializedContent;
+}
+
+export function serializeResponse<TOOLS extends ToolSet, OUTPUT>(
+  response: GenerateTextResult<TOOLS, OUTPUT>["response"]
+): { message: SerializedMessage; id?: string }[] {
+  const { id, timestamp, modelId, headers, messages, body } = response;
+  // TODO: what to do about all the rest?
+  // Store body?
+  return messages.map((m) => serializeMessageWithId(m));
+}
+
+export function serializeStep<TOOLS extends ToolSet>(
+  step: StepResult<TOOLS>
+): Step {
+  const content = step.response?.messages.map((message) => {
+    return serializeMessageWithId(message);
+  });
+  const timestamp = step.response?.timestamp.getTime();
+  const response = {
+    ...step.response,
+    messages: content,
+    timestamp,
+    headers: {},
+  };
+  return {
+    ...step,
+    response,
+  };
+}
+
+export function serializeNewMessagesInStep<TOOLS extends ToolSet>(
+  step: StepResult<TOOLS>
+): MessageWithFileAndId[] {
+  const messages: MessageWithFileAndId[] = [];
+  if (step.text) {
+    messages.push(serializeMessageWithId(step.response.messages.at(-1)!));
+  }
+  // TODO: get ID from regular messages
+  if (step.toolCalls || step.toolResults) {
+    messages.push(
+      ...step.response.messages
+        .slice(
+          0,
+          -((step.toolCalls?.length ?? 0) + (step.toolResults?.length ?? 0))
+        )
+        .map((m) => serializeMessageWithId(m))
+    );
+  }
+  return messages;
+}
+
+export function deserializeContent(content: SerializedContent): Content {
+  if (typeof content === "string") {
+    return content;
+  }
+  return content.map((part) => {
+    switch (part.type) {
+      case "image":
+        return { ...part, file: deserializeUrl(part.image) };
+      case "file":
+        return { ...part, file: deserializeUrl(part.data) };
+      default:
+        return part;
+    }
+  }) as Content;
+}
+
+// TODO: store in file storage if it's big
+function serializeDataOrUrl(
+  dataOrUrl: DataContent | URL
+): ArrayBuffer | string {
+  if (typeof dataOrUrl === "string") {
+    return dataOrUrl;
+  }
+  if (dataOrUrl instanceof ArrayBuffer) {
+    return dataOrUrl; // Already an ArrayBuffer
+  }
+  if (dataOrUrl instanceof URL) {
+    return dataOrUrl.toString();
+  }
+  return dataOrUrl.buffer.slice(
+    dataOrUrl.byteOffset,
+    dataOrUrl.byteOffset + dataOrUrl.byteLength
+  ) as ArrayBuffer;
+}
+
+function deserializeUrl(urlOrString: string | ArrayBuffer): URL | DataContent {
+  if (typeof urlOrString === "string") {
+    if (
+      urlOrString.startsWith("http://") ||
+      urlOrString.startsWith("https://")
+    ) {
+      return new URL(urlOrString);
+    }
+    return urlOrString;
+  }
+  return urlOrString;
+}

package/src/react/index.ts

@@ -0,0 +1,8 @@
+// This is where React components go.
+if (typeof window === "undefined") {
+  throw new Error("this is frontend code, but it's running somewhere else!");
+}
+
+export function subtract(a: number, b: number): number {
+  return a - b;
+}

package/src/shared.ts

@@ -0,0 +1,35 @@
+import { CoreMessage } from "ai";
+import { Message } from "./validators";
+
+export function isTool(message: Message | CoreMessage) {
+  return (
+    message.role === "tool" ||
+    (message.role === "assistant" &&
+      Array.isArray(message.content) &&
+      message.content.some((c) => c.type === "tool-call"))
+  );
+}
+
+export function extractText(message: Message | CoreMessage) {
+  switch (message.role) {
+    case "user":
+      if (typeof message.content === "string") {
+        return message.content;
+      }
+      return message.content
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join("");
+    case "assistant":
+      if (typeof message.content === "string") {
+        return message.content;
+      }
+      return message.content
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join("");
+  }
+  return undefined;
+}
+
+export const DEFAULT_MESSAGE_RANGE = { before: 2, after: 1 };

package/src/validators.test.ts

@@ -0,0 +1,101 @@
+import { Infer } from "convex/values";
+import { expectTypeOf, test } from "vitest";
+import {
+  vAssistantContent,
+  vAssistantMessage,
+  vFilePart,
+  vImagePart,
+  vMessage,
+  vReasoningPart,
+  vRedactedReasoningPart,
+  vSystemMessage,
+  vTextPart,
+  vToolCallPart,
+  vToolContent,
+  vToolMessage,
+  vUserMessage,
+} from "./validators";
+import { vUserContent } from "./validators";
+import {
+  AssistantContent,
+  CoreAssistantMessage,
+  CoreMessage,
+  CoreSystemMessage,
+  CoreToolMessage,
+  CoreUserMessage,
+  FilePart,
+  ImagePart,
+  TextPart,
+  ToolCallPart,
+  ToolContent,
+  UserContent,
+} from "ai";
+import { SerializeUrlsAndUint8Arrays } from "./mapping";
+
+// type assertion
+type OurUserContent = SerializeUrlsAndUint8Arrays<UserContent>;
+expectTypeOf<OurUserContent>().toExtend<Infer<typeof vUserContent>>();
+expectTypeOf<Infer<typeof vUserContent>>().toExtend<OurUserContent>();
+
+type OurAssistantContent = SerializeUrlsAndUint8Arrays<AssistantContent>;
+expectTypeOf<OurAssistantContent>().toExtend<Infer<typeof vAssistantContent>>();
+expectTypeOf<Infer<typeof vAssistantContent>>().toExtend<OurAssistantContent>();
+
+type OurToolContent = SerializeUrlsAndUint8Arrays<ToolContent>;
+expectTypeOf<OurToolContent>().toExtend<Infer<typeof vToolContent>>();
+expectTypeOf<Infer<typeof vToolContent>>().toExtend<OurToolContent>();
+
+expectTypeOf<Infer<typeof vToolCallPart>>().toExtend<ToolCallPart>();
+expectTypeOf<ToolCallPart>().toExtend<Infer<typeof vToolCallPart>>();
+
+type OurTextPart = SerializeUrlsAndUint8Arrays<TextPart>;
+expectTypeOf<OurTextPart>().toExtend<Infer<typeof vTextPart>>();
+expectTypeOf<Infer<typeof vTextPart>>().toExtend<OurTextPart>();
+
+type OurImagePart = SerializeUrlsAndUint8Arrays<ImagePart>;
+expectTypeOf<OurImagePart>().toExtend<Infer<typeof vImagePart>>();
+expectTypeOf<Infer<typeof vImagePart>>().toExtend<OurImagePart>();
+
+type OurFilePart = SerializeUrlsAndUint8Arrays<FilePart>;
+expectTypeOf<OurFilePart>().toExtend<Infer<typeof vFilePart>>();
+expectTypeOf<Infer<typeof vFilePart>>().toExtend<OurFilePart>();
+
+// narrow to the type
+type ReasoningPart = AssistantContent[number] & { type: "reasoning" } & object;
+type OurReasoningPart = SerializeUrlsAndUint8Arrays<ReasoningPart>;
+expectTypeOf<OurReasoningPart>().toExtend<Infer<typeof vReasoningPart>>();
+expectTypeOf<Infer<typeof vReasoningPart>>().toExtend<OurReasoningPart>();
+
+// narrow to the type
+type RedactedReasoningPart = AssistantContent[number] & {
+  type: "redacted-reasoning";
+} & object;
+type OurRedactedReasoningPart =
+  SerializeUrlsAndUint8Arrays<RedactedReasoningPart>;
+expectTypeOf<OurRedactedReasoningPart>().toExtend<
+  Infer<typeof vRedactedReasoningPart>
+>();
+expectTypeOf<
+  Infer<typeof vRedactedReasoningPart>
+>().toExtend<OurRedactedReasoningPart>();
+
+// test("noop", () => {
+type OurUserMessage = SerializeUrlsAndUint8Arrays<CoreUserMessage>;
+expectTypeOf<OurUserMessage>().toExtend<Infer<typeof vUserMessage>>();
+expectTypeOf<Infer<typeof vUserMessage>>().toExtend<OurUserMessage>();
+
+type OurAssistantMessage = SerializeUrlsAndUint8Arrays<CoreAssistantMessage>;
+expectTypeOf<OurAssistantMessage>().toExtend<Infer<typeof vAssistantMessage>>();
+expectTypeOf<Infer<typeof vAssistantMessage>>().toExtend<OurAssistantMessage>();
+
+expectTypeOf<Infer<typeof vToolMessage>>().toExtend<CoreToolMessage>();
+expectTypeOf<CoreToolMessage>().toExtend<Infer<typeof vToolMessage>>();
+
+expectTypeOf<Infer<typeof vSystemMessage>>().toExtend<CoreSystemMessage>();
+expectTypeOf<CoreSystemMessage>().toExtend<Infer<typeof vSystemMessage>>();
+
+type OurMessage = SerializeUrlsAndUint8Arrays<CoreMessage>;
+expectTypeOf<OurMessage>().toExtend<Infer<typeof vMessage>>();
+expectTypeOf<Infer<typeof vMessage>>().toExtend<OurMessage>();
+
+test("noop", () => { });