@trestleinc/replicate 1.1.2 → 1.2.0-preview.1

package/src/component/schema.ts

@@ -4,29 +4,46 @@ import { v } from "convex/values";
 export default defineSchema({
   documents: defineTable({
     collection: v.string(),
-    documentId: v.string(),
-    crdtBytes: v.bytes(),
+    document: v.string(),
+    bytes: v.bytes(),
     seq: v.number(),
   })
     .index("by_collection", ["collection"])
-    .index("by_collection_document", ["collection", "documentId"])
+    .index("by_document", ["collection", "document"])
     .index("by_seq", ["collection", "seq"]),
 
   snapshots: defineTable({
     collection: v.string(),
-    documentId: v.string(),
-    snapshotBytes: v.bytes(),
-    stateVector: v.bytes(),
-    snapshotSeq: v.number(),
-    createdAt: v.number(),
-  }).index("by_document", ["collection", "documentId"]),
+    document: v.string(),
+    bytes: v.bytes(),
+    vector: v.bytes(),
+    seq: v.number(),
+    created: v.number(),
+  }).index("by_document", ["collection", "document"]),
 
-  peers: defineTable({
+  sessions: defineTable({
     collection: v.string(),
-    peerId: v.string(),
-    lastSyncedSeq: v.number(),
-    lastSeenAt: v.number(),
+    document: v.string(),
+    client: v.string(),
+    vector: v.optional(v.bytes()),
+    connected: v.boolean(),
+    seq: v.number(),
+    seen: v.number(),
+    user: v.optional(v.string()),
+    profile: v.optional(v.object({
+      name: v.optional(v.string()),
+      color: v.optional(v.string()),
+      avatar: v.optional(v.string()),
+    })),
+    cursor: v.optional(v.object({
+      anchor: v.any(),
+      head: v.any(),
+      field: v.optional(v.string()),
+    })),
+    timeout: v.optional(v.id("_scheduled_functions")),
   })
     .index("by_collection", ["collection"])
-    .index("by_collection_peer", ["collection", "peerId"]),
+    .index("by_document", ["collection", "document"])
+    .index("by_client", ["collection", "document", "client"])
+    .index("by_connected", ["collection", "document", "connected"]),
 });

package/src/server/collection.ts

@@ -0,0 +1,98 @@
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from "convex/server";
+import { Replicate } from "$/server/replicate";
+import type { CompactionConfig } from "$/shared/types";
+
+export interface CollectionOptions<T extends object> {
+  compaction?: Partial<CompactionConfig>;
+  hooks?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    evalMark?: (ctx: GenericMutationCtx<GenericDataModel>, client: string) => void | Promise<void>;
+    evalLeave?: (ctx: GenericMutationCtx<GenericDataModel>, client: string) => void | Promise<void>;
+    evalCompact?: (
+      ctx: GenericMutationCtx<GenericDataModel>,
+      document: string,
+    ) => void | Promise<void>;
+    onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
+    onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    transform?: (docs: T[]) => T[] | Promise<T[]>;
+  };
+}
+
+function createCollection<T extends object>(
+  component: any,
+  name: string,
+  options?: CollectionOptions<T>,
+) {
+  return createCollectionInternal<T>(component, name, options);
+}
+
+export const collection = {
+  create: createCollection,
+} as const;
+
+function createCollectionInternal<T extends object>(
+  component: any,
+  name: string,
+  options?: CollectionOptions<T>,
+) {
+  const storage = new Replicate<T>(component, name, options?.compaction);
+
+  const hooks = options?.hooks;
+
+  return {
+    __collection: name,
+
+    stream: storage.createStreamQuery({
+      evalRead: hooks?.evalRead,
+      onStream: hooks?.onStream,
+    }),
+
+    material: storage.createSSRQuery({
+      evalRead: hooks?.evalRead,
+      transform: hooks?.transform,
+    }),
+
+    recovery: storage.createRecoveryQuery({
+      evalRead: hooks?.evalRead,
+    }),
+
+    insert: storage.createInsertMutation({
+      evalWrite: hooks?.evalWrite,
+      onInsert: hooks?.onInsert,
+    }),
+
+    update: storage.createUpdateMutation({
+      evalWrite: hooks?.evalWrite,
+      onUpdate: hooks?.onUpdate,
+    }),
+
+    remove: storage.createRemoveMutation({
+      evalRemove: hooks?.evalRemove,
+      onRemove: hooks?.onRemove,
+    }),
+
+    mark: storage.createMarkMutation({
+      evalWrite: hooks?.evalMark,
+    }),
+
+    compact: storage.createCompactMutation({
+      evalWrite: hooks?.evalCompact,
+    }),
+
+    sessions: storage.createSessionsQuery({
+      evalRead: hooks?.evalRead,
+    }),
+
+    cursors: storage.createCursorsQuery({
+      evalRead: hooks?.evalRead,
+    }),
+
+    leave: storage.createLeaveMutation({
+      evalWrite: hooks?.evalLeave,
+    }),
+  };
+}

package/src/server/index.ts

@@ -1,5 +1,5 @@
-export { replicate } from "$/server/builder";
-export type { ReplicateConfig } from "$/server/builder";
+export { collection } from "$/server/collection";
+export type { CollectionOptions } from "$/server/collection";
 
 import { table, prose } from "$/server/schema";
 

package/src/server/{storage.ts → replicate.ts}

@@ -34,32 +34,34 @@ export class Replicate<T extends object> {
 
     return queryGeneric({
       args: {
-        cursor: v.number(),
+        seq: v.number(),
         limit: v.optional(v.number()),
-        sizeThreshold: v.optional(v.number()),
+        threshold: v.optional(v.number()),
       },
       returns: v.object({
         changes: v.array(
           v.object({
-            documentId: v.string(),
-            crdtBytes: v.bytes(),
+            document: v.string(),
+            bytes: v.bytes(),
             seq: v.number(),
-            operationType: v.string(),
+            type: v.string(),
           }),
         ),
-        cursor: v.number(),
-        hasMore: v.boolean(),
-        compact: v.optional(v.string()),
+        seq: v.number(),
+        more: v.boolean(),
+        compact: v.optional(v.object({
+          documents: v.array(v.string()),
+        })),
       }),
       handler: async (ctx, args) => {
         if (opts?.evalRead) {
           await opts.evalRead(ctx, collection);
         }
-        const result = await ctx.runQuery(component.public.stream, {
+        const result = await ctx.runQuery(component.mutations.stream, {
           collection,
-          cursor: args.cursor,
+          seq: args.seq,
           limit: args.limit,
-          sizeThreshold: args.sizeThreshold,
+          threshold: args.threshold,
         });
 
         if (opts?.onStream) {
@@ -83,9 +85,12 @@ export class Replicate<T extends object> {
       args: {},
       returns: v.object({
         documents: v.any(),
-        cursor: v.optional(v.number()),
         count: v.number(),
-        crdtBytes: v.optional(v.bytes()),
+        crdt: v.optional(v.record(v.string(), v.object({
+          bytes: v.bytes(),
+          seq: v.number(),
+        }))),
+        cursor: v.optional(v.number()),
       }),
       handler: async (ctx) => {
         if (opts?.evalRead) {
@@ -98,24 +103,35 @@ export class Replicate<T extends object> {
 
         const response: {
           documents: T[];
-          cursor?: number;
           count: number;
-          crdtBytes?: ArrayBuffer;
+          crdt?: Record<string, { bytes: ArrayBuffer; seq: number }>;
+          cursor?: number;
         } = {
           documents: docs,
           count: docs.length,
         };
 
-        if (opts?.includeCRDTState) {
-          const crdtState = await ctx.runQuery(component.public.getInitialState, {
-            collection,
-          });
-
-          if (crdtState) {
-            response.crdtBytes = crdtState.crdtBytes;
-            response.cursor = crdtState.cursor;
+        if (opts?.includeCRDTState && docs.length > 0) {
+          const crdt: Record<string, { bytes: ArrayBuffer; seq: number }> = {};
+          let maxSeq = 0;
+
+          for (const doc of docs) {
+            const docId = (doc as { id: string }).id;
+            const state = await ctx.runQuery(component.mutations.getDocumentState, {
+              collection,
+              document: docId,
+            });
+
+            if (state) {
+              crdt[docId] = { bytes: state.bytes, seq: state.seq };
+              maxSeq = Math.max(maxSeq, state.seq);
+            }
           }
+
+          response.crdt = crdt;
+          response.cursor = maxSeq;
         }
+
         return response;
       },
     });
@@ -130,30 +146,30 @@ export class Replicate<T extends object> {
 
     return mutationGeneric({
       args: {
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
-        materializedDoc: v.any(),
+        document: v.string(),
+        bytes: v.bytes(),
+        material: v.any(),
       },
       returns: v.object({
         success: v.boolean(),
         seq: v.number(),
       }),
       handler: async (ctx, args) => {
-        const doc = args.materializedDoc as T;
+        const doc = args.material as T;
 
         if (opts?.evalWrite) {
           await opts.evalWrite(ctx, doc);
         }
 
-        const result = await ctx.runMutation(component.public.insertDocument, {
+        const result = await ctx.runMutation(component.mutations.insertDocument, {
           collection,
-          documentId: args.documentId,
-          crdtBytes: args.crdtBytes,
+          document: args.document,
+          bytes: args.bytes,
         });
 
         await ctx.db.insert(collection, {
-          id: args.documentId,
-          ...(args.materializedDoc as object),
+          id: args.document,
+          ...(args.material as object),
           timestamp: Date.now(),
         });
 
@@ -178,35 +194,35 @@ export class Replicate<T extends object> {
 
     return mutationGeneric({
       args: {
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
-        materializedDoc: v.any(),
+        document: v.string(),
+        bytes: v.bytes(),
+        material: v.any(),
       },
       returns: v.object({
         success: v.boolean(),
         seq: v.number(),
       }),
       handler: async (ctx, args) => {
-        const doc = args.materializedDoc as T;
+        const doc = args.material as T;
 
         if (opts?.evalWrite) {
           await opts.evalWrite(ctx, doc);
         }
 
-        const result = await ctx.runMutation(component.public.updateDocument, {
+        const result = await ctx.runMutation(component.mutations.updateDocument, {
           collection,
-          documentId: args.documentId,
-          crdtBytes: args.crdtBytes,
+          document: args.document,
+          bytes: args.bytes,
         });
 
         const existing = await ctx.db
           .query(collection)
-          .withIndex("by_doc_id", q => q.eq("id", args.documentId))
+          .withIndex("by_doc_id", q => q.eq("id", args.document))
           .first();
 
         if (existing) {
           await ctx.db.patch(existing._id, {
-            ...(args.materializedDoc as object),
+            ...(args.material as object),
             timestamp: Date.now(),
           });
         }
@@ -232,28 +248,27 @@ export class Replicate<T extends object> {
 
     return mutationGeneric({
       args: {
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
+        document: v.string(),
+        bytes: v.bytes(),
       },
       returns: v.object({
         success: v.boolean(),
         seq: v.number(),
       }),
       handler: async (ctx, args) => {
-        const documentId = args.documentId;
         if (opts?.evalRemove) {
-          await opts.evalRemove(ctx, documentId);
+          await opts.evalRemove(ctx, args.document);
         }
 
-        const result = await ctx.runMutation(component.public.deleteDocument, {
+        const result = await ctx.runMutation(component.mutations.deleteDocument, {
           collection,
-          documentId: documentId,
-          crdtBytes: args.crdtBytes,
+          document: args.document,
+          bytes: args.bytes,
         });
 
         const existing = await ctx.db
           .query(collection)
-          .withIndex("by_doc_id", q => q.eq("id", documentId))
+          .withIndex("by_doc_id", q => q.eq("id", args.document))
           .first();
 
         if (existing) {
@@ -261,7 +276,7 @@ export class Replicate<T extends object> {
         }
 
         if (opts?.onRemove) {
-          await opts.onRemove(ctx, documentId);
+          await opts.onRemove(ctx, args.document);
         }
 
         return {
@@ -273,26 +288,150 @@ export class Replicate<T extends object> {
   }
 
   createMarkMutation(opts?: {
-    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, peerId: string) => void | Promise<void>;
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, client: string) => void | Promise<void>;
   }) {
     const component = this.component;
     const collection = this.collectionName;
 
     return mutationGeneric({
       args: {
-        peerId: v.string(),
-        syncedSeq: v.number(),
+        document: v.string(),
+        client: v.string(),
+        seq: v.optional(v.number()),
+        vector: v.optional(v.bytes()),
+        user: v.optional(v.string()),
+        profile: v.optional(v.object({
+          name: v.optional(v.string()),
+          color: v.optional(v.string()),
+          avatar: v.optional(v.string()),
+        })),
+        cursor: v.optional(v.object({
+          anchor: v.any(),
+          head: v.any(),
+          field: v.optional(v.string()),
+        })),
+        interval: v.optional(v.number()),
       },
       returns: v.null(),
       handler: async (ctx, args) => {
         if (opts?.evalWrite) {
-          await opts.evalWrite(ctx, args.peerId);
+          await opts.evalWrite(ctx, args.client);
         }
 
-        await ctx.runMutation(component.public.mark, {
+        await ctx.runMutation(component.mutations.mark, {
           collection,
-          peerId: args.peerId,
-          syncedSeq: args.syncedSeq,
+          document: args.document,
+          client: args.client,
+          seq: args.seq,
+          vector: args.vector,
+          user: args.user,
+          profile: args.profile,
+          cursor: args.cursor,
+          interval: args.interval,
+        });
+
+        return null;
+      },
+    });
+  }
+
+  createSessionsQuery(opts?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return queryGeneric({
+      args: {
+        document: v.string(),
+        connected: v.optional(v.boolean()),
+        exclude: v.optional(v.string()),
+        group: v.optional(v.boolean()),
+      },
+      returns: v.array(v.object({
+        client: v.string(),
+        document: v.string(),
+        user: v.optional(v.string()),
+        profile: v.optional(v.any()),
+        cursor: v.optional(v.object({
+          anchor: v.any(),
+          head: v.any(),
+          field: v.optional(v.string()),
+        })),
+        seen: v.number(),
+      })),
+      handler: async (ctx, args) => {
+        if (opts?.evalRead) {
+          await opts.evalRead(ctx, collection);
+        }
+
+        return await ctx.runQuery(component.mutations.sessions, {
+          collection,
+          document: args.document,
+          connected: args.connected,
+          exclude: args.exclude,
+          group: args.group,
+        });
+      },
+    });
+  }
+
+  createCursorsQuery(opts?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return queryGeneric({
+      args: {
+        document: v.string(),
+        exclude: v.optional(v.string()),
+      },
+      returns: v.array(v.object({
+        client: v.string(),
+        user: v.optional(v.string()),
+        profile: v.optional(v.any()),
+        cursor: v.object({
+          anchor: v.any(),
+          head: v.any(),
+          field: v.optional(v.string()),
+        }),
+      })),
+      handler: async (ctx, args) => {
+        if (opts?.evalRead) {
+          await opts.evalRead(ctx, collection);
+        }
+
+        return await ctx.runQuery(component.mutations.cursors, {
+          collection,
+          document: args.document,
+          exclude: args.exclude,
+        });
+      },
+    });
+  }
+
+  createLeaveMutation(opts?: {
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, client: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return mutationGeneric({
+      args: {
+        document: v.string(),
+        client: v.string(),
+      },
+      returns: v.null(),
+      handler: async (ctx, args) => {
+        if (opts?.evalWrite) {
+          await opts.evalWrite(ctx, args.client);
+        }
+
+        await ctx.runMutation(component.mutations.leave, {
+          collection,
+          document: args.document,
+          client: args.client,
         });
 
         return null;
@@ -303,7 +442,7 @@ export class Replicate<T extends object> {
   createCompactMutation(opts?: {
     evalWrite?: (
       ctx: GenericMutationCtx<GenericDataModel>,
-      documentId: string,
+      document: string,
     ) => void | Promise<void>;
   }) {
     const component = this.component;
@@ -311,55 +450,55 @@ export class Replicate<T extends object> {
 
     return mutationGeneric({
       args: {
-        documentId: v.string(),
-        snapshotBytes: v.bytes(),
-        stateVector: v.bytes(),
-        peerTimeout: v.optional(v.number()),
+        document: v.string(),
       },
       returns: v.object({
         success: v.boolean(),
         removed: v.number(),
         retained: v.number(),
+        size: v.number(),
       }),
       handler: async (ctx, args) => {
         if (opts?.evalWrite) {
-          await opts.evalWrite(ctx, args.documentId);
+          await opts.evalWrite(ctx, args.document);
         }
 
-        return await ctx.runMutation(component.public.compact, {
+        return await ctx.runMutation(component.mutations.compact, {
           collection,
-          documentId: args.documentId,
-          snapshotBytes: args.snapshotBytes,
-          stateVector: args.stateVector,
-          peerTimeout: args.peerTimeout,
+          document: args.document,
         });
       },
     });
   }
 
   createRecoveryQuery(opts?: {
-    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    evalRead?: (
+      ctx: GenericQueryCtx<GenericDataModel>,
+      collection: string,
+      document: string,
+    ) => void | Promise<void>;
   }) {
     const component = this.component;
     const collection = this.collectionName;
 
     return queryGeneric({
       args: {
-        clientStateVector: v.bytes(),
+        document: v.string(),
+        vector: v.bytes(),
       },
       returns: v.object({
         diff: v.optional(v.bytes()),
-        serverStateVector: v.bytes(),
-        cursor: v.number(),
+        vector: v.bytes(),
       }),
       handler: async (ctx, args) => {
         if (opts?.evalRead) {
-          await opts.evalRead(ctx, collection);
+          await opts.evalRead(ctx, collection, args.document);
         }
 
-        return await ctx.runQuery(component.public.recovery, {
+        return await ctx.runQuery(component.mutations.recovery, {
           collection,
-          clientStateVector: args.clientStateVector,
+          document: args.document,
+          vector: args.vector,
         });
       },
     });