@trestleinc/replicate 1.1.0 → 1.1.2-preview.0

package/src/component/schema.ts

@@ -1,29 +1,32 @@
-import { defineSchema, defineTable } from 'convex/server';
-import { v } from 'convex/values';
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
 
 export default defineSchema({
   documents: defineTable({
     collection: v.string(),
     documentId: v.string(),
     crdtBytes: v.bytes(),
-    version: v.number(),
-    timestamp: v.number(),
+    seq: v.number(),
   })
-    .index('by_collection', ['collection'])
-    .index('by_collection_document_version', ['collection', 'documentId', 'version'])
-    .index('by_timestamp', ['collection', 'timestamp']),
+    .index("by_collection", ["collection"])
+    .index("by_collection_document", ["collection", "documentId"])
+    .index("by_seq", ["collection", "seq"]),
 
   snapshots: defineTable({
     collection: v.string(),
     documentId: v.string(),
     snapshotBytes: v.bytes(),
-    latestCompactionTimestamp: v.number(),
+    stateVector: v.bytes(),
+    snapshotSeq: v.number(),
     createdAt: v.number(),
-    metadata: v.optional(
-      v.object({
-        deltaCount: v.number(),
-        totalSize: v.number(),
-      })
-    ),
-  }).index('by_document', ['collection', 'documentId']),
+  }).index("by_document", ["collection", "documentId"]),
+
+  peers: defineTable({
+    collection: v.string(),
+    peerId: v.string(),
+    lastSyncedSeq: v.number(),
+    lastSeenAt: v.number(),
+  })
+    .index("by_collection", ["collection"])
+    .index("by_collection_peer", ["collection", "peerId"]),
 });

package/src/server/builder.ts

@@ -1,17 +1,22 @@
-import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
-import { Replicate } from '$/server/storage.js';
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from "convex/server";
+import { Replicate } from "$/server/storage";
+import type { CompactionConfig } from "$/shared/types";
 
 /**
  * Configuration for replicate handlers (without component - used with factory pattern).
  */
 export interface ReplicateConfig<T extends object> {
   collection: string;
-  /** Size threshold for auto-compaction (default: 5MB). Set to 0 to disable. */
-  compaction?: { threshold?: number };
+  compaction?: Partial<CompactionConfig>;
   hooks?: {
     evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
     evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
     evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    evalMark?: (ctx: GenericMutationCtx<GenericDataModel>, peerId: string) => void | Promise<void>;
+    evalCompact?: (
+      ctx: GenericMutationCtx<GenericDataModel>,
+      documentId: string,
+    ) => void | Promise<void>;
     onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
     onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
     onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
@@ -48,11 +53,11 @@ export function replicate(component: any) {
  * Internal implementation for replicate.
  */
 function replicateInternal<T extends object>(component: any, config: ReplicateConfig<T>) {
-  const storage = new Replicate<T>(component, config.collection, {
-    threshold: config.compaction?.threshold,
-  });
+  const storage = new Replicate<T>(component, config.collection, config.compaction);
 
   return {
+    __collection: config.collection,
+
     stream: storage.createStreamQuery({
       evalRead: config.hooks?.evalRead,
       onStream: config.hooks?.onStream,
@@ -81,5 +86,13 @@ function replicateInternal<T extends object>(component: any, config: ReplicateCo
       evalRemove: config.hooks?.evalRemove,
       onRemove: config.hooks?.onRemove,
     }),
+
+    mark: storage.createMarkMutation({
+      evalWrite: config.hooks?.evalMark,
+    }),
+
+    compact: storage.createCompactMutation({
+      evalWrite: config.hooks?.evalCompact,
+    }),
   };
 }

package/src/server/index.ts

@@ -1,11 +1,9 @@
-export { replicate } from '$/server/builder.js';
-export type { ReplicateConfig } from '$/server/builder.js';
+export { replicate } from "$/server/builder";
+export type { ReplicateConfig } from "$/server/builder";
 
-import { table, prose } from '$/server/schema.js';
+import { table, prose } from "$/server/schema";
 
 export const schema = {
   table,
   prose,
 } as const;
-
-export type { ReplicationFields } from '$/server/schema.js';

package/src/server/schema.ts

@@ -1,14 +1,14 @@
-import { defineTable } from 'convex/server';
-import { v } from 'convex/values';
+import { defineTable } from "convex/server";
+import { v } from "convex/values";
 
 /** Fields automatically added to replicated tables */
-export type ReplicationFields = {
+export interface ReplicationFields {
   timestamp: number;
-};
+}
 
 export const prose = () =>
   v.object({
-    type: v.literal('doc'),
+    type: v.literal("doc"),
     content: v.optional(v.array(v.any())),
   });
 

package/src/server/storage.ts

@@ -1,13 +1,29 @@
-import { v } from 'convex/values';
-import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
-import { queryGeneric, mutationGeneric } from 'convex/server';
+import { v } from "convex/values";
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from "convex/server";
+import { queryGeneric, mutationGeneric } from "convex/server";
+import { type CompactionConfig, parseSize, parseDuration } from "$/shared/types";
+
+const BYTES_PER_MB = 1024 * 1024;
+const MS_PER_HOUR = 60 * 60 * 1000;
+const DEFAULT_SIZE_THRESHOLD_5MB = 5 * BYTES_PER_MB;
+const DEFAULT_PEER_TIMEOUT_24H = 24 * MS_PER_HOUR;
 
 export class Replicate<T extends object> {
+  private sizeThreshold: number;
+  private peerTimeout: number;
+
   constructor(
     public component: any,
     public collectionName: string,
-    private options?: { threshold?: number }
-  ) {}
+    compaction?: Partial<CompactionConfig>,
+  ) {
+    this.sizeThreshold = compaction?.sizeThreshold
+      ? parseSize(compaction.sizeThreshold)
+      : DEFAULT_SIZE_THRESHOLD_5MB;
+    this.peerTimeout = compaction?.peerTimeout
+      ? parseDuration(compaction.peerTimeout)
+      : DEFAULT_PEER_TIMEOUT_24H;
+  }
 
   createStreamQuery(opts?: {
     evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
@@ -18,22 +34,22 @@ export class Replicate<T extends object> {
 
     return queryGeneric({
       args: {
-        checkpoint: v.object({ lastModified: v.number() }),
+        cursor: v.number(),
         limit: v.optional(v.number()),
-        vector: v.optional(v.bytes()),
+        sizeThreshold: v.optional(v.number()),
       },
       returns: v.object({
         changes: v.array(
           v.object({
-            documentId: v.optional(v.string()),
+            documentId: v.string(),
             crdtBytes: v.bytes(),
-            version: v.number(),
-            timestamp: v.number(),
+            seq: v.number(),
             operationType: v.string(),
-          })
+          }),
         ),
-        checkpoint: v.object({ lastModified: v.number() }),
+        cursor: v.number(),
         hasMore: v.boolean(),
+        compact: v.optional(v.string()),
       }),
       handler: async (ctx, args) => {
         if (opts?.evalRead) {
@@ -41,9 +57,9 @@ export class Replicate<T extends object> {
         }
         const result = await ctx.runQuery(component.public.stream, {
           collection,
-          checkpoint: args.checkpoint,
+          cursor: args.cursor,
           limit: args.limit,
-          vector: args.vector,
+          sizeThreshold: args.sizeThreshold,
         });
 
         if (opts?.onStream) {
@@ -67,7 +83,7 @@ export class Replicate<T extends object> {
       args: {},
       returns: v.object({
         documents: v.any(),
-        checkpoint: v.optional(v.object({ lastModified: v.number() })),
+        cursor: v.optional(v.number()),
         count: v.number(),
         crdtBytes: v.optional(v.bytes()),
       }),
@@ -80,17 +96,13 @@ export class Replicate<T extends object> {
           docs = await opts.transform(docs);
         }
 
-        const latestTimestamp =
-          docs.length > 0 ? Math.max(...docs.map((doc: any) => doc.timestamp || 0)) : 0;
-
         const response: {
           documents: T[];
-          checkpoint?: { lastModified: number };
+          cursor?: number;
           count: number;
           crdtBytes?: ArrayBuffer;
         } = {
           documents: docs,
-          checkpoint: latestTimestamp > 0 ? { lastModified: latestTimestamp } : undefined,
           count: docs.length,
         };
 
@@ -101,7 +113,7 @@ export class Replicate<T extends object> {
 
           if (crdtState) {
             response.crdtBytes = crdtState.crdtBytes;
-            response.checkpoint = crdtState.checkpoint;
+            response.cursor = crdtState.cursor;
           }
         }
         return response;
@@ -115,7 +127,6 @@ export class Replicate<T extends object> {
   }) {
     const component = this.component;
     const collection = this.collectionName;
-    const threshold = this.options?.threshold;
 
     return mutationGeneric({
       args: {
@@ -125,7 +136,7 @@ export class Replicate<T extends object> {
       },
       returns: v.object({
         success: v.boolean(),
-        metadata: v.any(),
+        seq: v.number(),
       }),
       handler: async (ctx, args) => {
         const doc = args.materializedDoc as T;
@@ -134,13 +145,10 @@ export class Replicate<T extends object> {
           await opts.evalWrite(ctx, doc);
         }
 
-        const version = Date.now();
-        await ctx.runMutation(component.public.insertDocument, {
+        const result = await ctx.runMutation(component.public.insertDocument, {
           collection,
           documentId: args.documentId,
           crdtBytes: args.crdtBytes,
-          version,
-          threshold,
         });
 
         await ctx.db.insert(collection, {
@@ -155,11 +163,7 @@ export class Replicate<T extends object> {
 
         return {
           success: true,
-          metadata: {
-            documentId: args.documentId,
-            timestamp: Date.now(),
-            collection,
-          },
+          seq: result.seq,
         };
       },
     });
@@ -171,7 +175,6 @@ export class Replicate<T extends object> {
   }) {
     const component = this.component;
     const collection = this.collectionName;
-    const threshold = this.options?.threshold;
 
     return mutationGeneric({
       args: {
@@ -181,7 +184,7 @@ export class Replicate<T extends object> {
       },
       returns: v.object({
         success: v.boolean(),
-        metadata: v.any(),
+        seq: v.number(),
       }),
       handler: async (ctx, args) => {
         const doc = args.materializedDoc as T;
@@ -190,22 +193,19 @@ export class Replicate<T extends object> {
           await opts.evalWrite(ctx, doc);
         }
 
-        const version = Date.now();
-        await ctx.runMutation(component.public.updateDocument, {
+        const result = await ctx.runMutation(component.public.updateDocument, {
           collection,
           documentId: args.documentId,
           crdtBytes: args.crdtBytes,
-          version,
-          threshold,
         });
 
         const existing = await ctx.db
           .query(collection)
-          .withIndex('by_doc_id', (q) => q.eq('id', args.documentId))
+          .withIndex("by_doc_id", q => q.eq("id", args.documentId))
           .first();
 
         if (existing) {
-          await ctx.db.patch(collection, existing._id, {
+          await ctx.db.patch(existing._id, {
             ...(args.materializedDoc as object),
             timestamp: Date.now(),
           });
@@ -217,11 +217,7 @@ export class Replicate<T extends object> {
 
         return {
           success: true,
-          metadata: {
-            documentId: args.documentId,
-            timestamp: Date.now(),
-            collection,
-          },
+          seq: result.seq,
         };
       },
     });
@@ -233,7 +229,6 @@ export class Replicate<T extends object> {
   }) {
     const component = this.component;
     const collection = this.collectionName;
-    const threshold = this.options?.threshold;
 
     return mutationGeneric({
       args: {
@@ -242,30 +237,27 @@ export class Replicate<T extends object> {
       },
       returns: v.object({
         success: v.boolean(),
-        metadata: v.any(),
+        seq: v.number(),
       }),
       handler: async (ctx, args) => {
-        const documentId = args.documentId as string;
+        const documentId = args.documentId;
         if (opts?.evalRemove) {
           await opts.evalRemove(ctx, documentId);
         }
 
-        const version = Date.now();
-        await ctx.runMutation(component.public.deleteDocument, {
+        const result = await ctx.runMutation(component.public.deleteDocument, {
           collection,
           documentId: documentId,
           crdtBytes: args.crdtBytes,
-          version,
-          threshold,
         });
 
         const existing = await ctx.db
           .query(collection)
-          .withIndex('by_doc_id', (q) => q.eq('id', documentId))
+          .withIndex("by_doc_id", q => q.eq("id", documentId))
           .first();
 
         if (existing) {
-          await ctx.db.delete(collection, existing._id);
+          await ctx.db.delete(existing._id);
         }
 
         if (opts?.onRemove) {
@@ -274,16 +266,77 @@ export class Replicate<T extends object> {
 
         return {
           success: true,
-          metadata: {
-            documentId: documentId,
-            timestamp: Date.now(),
-            collection,
-          },
+          seq: result.seq,
         };
       },
     });
   }
 
+  createMarkMutation(opts?: {
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, peerId: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return mutationGeneric({
+      args: {
+        peerId: v.string(),
+        syncedSeq: v.number(),
+      },
+      returns: v.null(),
+      handler: async (ctx, args) => {
+        if (opts?.evalWrite) {
+          await opts.evalWrite(ctx, args.peerId);
+        }
+
+        await ctx.runMutation(component.public.mark, {
+          collection,
+          peerId: args.peerId,
+          syncedSeq: args.syncedSeq,
+        });
+
+        return null;
+      },
+    });
+  }
+
+  createCompactMutation(opts?: {
+    evalWrite?: (
+      ctx: GenericMutationCtx<GenericDataModel>,
+      documentId: string,
+    ) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return mutationGeneric({
+      args: {
+        documentId: v.string(),
+        snapshotBytes: v.bytes(),
+        stateVector: v.bytes(),
+        peerTimeout: v.optional(v.number()),
+      },
+      returns: v.object({
+        success: v.boolean(),
+        removed: v.number(),
+        retained: v.number(),
+      }),
+      handler: async (ctx, args) => {
+        if (opts?.evalWrite) {
+          await opts.evalWrite(ctx, args.documentId);
+        }
+
+        return await ctx.runMutation(component.public.compact, {
+          collection,
+          documentId: args.documentId,
+          snapshotBytes: args.snapshotBytes,
+          stateVector: args.stateVector,
+          peerTimeout: args.peerTimeout,
+        });
+      },
+    });
+  }
+
   createRecoveryQuery(opts?: {
     evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
   }) {
@@ -297,6 +350,7 @@ export class Replicate<T extends object> {
       returns: v.object({
         diff: v.optional(v.bytes()),
         serverStateVector: v.bytes(),
+        cursor: v.number(),
       }),
       handler: async (ctx, args) => {
         if (opts?.evalRead) {

package/src/shared/index.ts

@@ -1,5 +1,5 @@
-/**
- * Shared types re-exported for top-level access
- */
-export type { FragmentValue, XmlFragmentJSON, XmlNodeJSON } from "./types.js";
-export { OperationType } from "./types.js";
+export type {
+  ProseValue,
+  XmlFragmentJSON,
+  XmlNodeJSON,
+} from "./types.js";

package/src/shared/types.ts

@@ -17,6 +17,17 @@ export interface XmlFragmentJSON {
   content?: XmlNodeJSON[];
 }
 
+declare const PROSE_BRAND: unique symbol;
+
+/**
+ * Branded prose type for Zod schemas.
+ * Extends XmlFragmentJSON with a unique brand for type-level detection.
+ * Use the `prose()` helper from `@trestleinc/replicate/client` to create this type.
+ */
+export interface ProseValue extends XmlFragmentJSON {
+  readonly [PROSE_BRAND]: typeof PROSE_BRAND;
+}
+
 /** ProseMirror node structure */
 export interface XmlNodeJSON {
   type: string;
@@ -33,20 +44,46 @@ export enum OperationType {
 }
 
 /**
- * Extract field names from T where the value type is XmlFragmentJSON.
- * Used for type-safe prose field configuration.
- *
- * @example
- * ```typescript
- * interface Notebook {
- *   id: string;
- *   title: string;
- *   content: XmlFragmentJSON;
- * }
- *
- * type Fields = ProseFields<Notebook>; // 'content'
- * ```
+ * Extract prose field names from T (fields typed as ProseValue).
+ * Used internally for type-safe prose field operations.
  */
 export type ProseFields<T> = {
-  [K in keyof T]: T[K] extends XmlFragmentJSON ? K : never;
+  [K in keyof T]: T[K] extends ProseValue ? K : never;
 }[keyof T];
+
+type SizeUnit = "kb" | "mb" | "gb";
+export type Size = `${number}${SizeUnit}`;
+
+type DurationUnit = "m" | "h" | "d";
+export type Duration = `${number}${DurationUnit}`;
+
+export interface CompactionConfig {
+  sizeThreshold: Size;
+  peerTimeout: Duration;
+}
+
+const SIZE_MULTIPLIERS: Record<SizeUnit, number> = {
+  kb: 1024,
+  mb: 1024 ** 2,
+  gb: 1024 ** 3,
+};
+
+const DURATION_MULTIPLIERS: Record<DurationUnit, number> = {
+  m: 60_000,
+  h: 3_600_000,
+  d: 86_400_000,
+};
+
+export function parseSize(s: Size): number {
+  const match = /^(\d+)(kb|mb|gb)$/i.exec(s);
+  if (!match) throw new Error(`Invalid size: ${s}`);
+  const [, num, unit] = match;
+  return parseInt(num) * SIZE_MULTIPLIERS[unit.toLowerCase() as SizeUnit];
+}
+
+export function parseDuration(s: Duration): number {
+  const match = /^(\d+)(m|h|d)$/i.exec(s);
+  if (!match) throw new Error(`Invalid duration: ${s}`);
+  const [, num, unit] = match;
+  return parseInt(num) * DURATION_MULTIPLIERS[unit.toLowerCase() as DurationUnit];
+}

package/dist/client/collection.d.ts

@@ -1,96 +0,0 @@
-import * as Y from 'yjs';
-import type { Persistence } from '$/client/persistence/types.js';
-import type { ConvexClient } from 'convex/browser';
-import type { FunctionReference } from 'convex/server';
-import type { CollectionConfig, Collection } from '@tanstack/db';
-import type { ProseFields } from '$/shared/types.js';
-/** Server-rendered material data for SSR hydration */
-export type Materialized<T> = {
-    documents: ReadonlyArray<T>;
-    checkpoint?: {
-        lastModified: number;
-    };
-    count?: number;
-    crdtBytes?: ArrayBuffer;
-};
-/** Configuration for creating a Convex-backed collection */
-export interface ConvexCollectionOptionsConfig<T extends object> {
-    getKey: (item: T) => string | number;
-    material?: Materialized<T>;
-    convexClient: ConvexClient;
-    api: {
-        stream: FunctionReference<'query'>;
-        insert: FunctionReference<'mutation'>;
-        update: FunctionReference<'mutation'>;
-        remove: FunctionReference<'mutation'>;
-        recovery: FunctionReference<'query'>;
-        material?: FunctionReference<'query'>;
-        [key: string]: any;
-    };
-    collection: string;
-    /** Fields that contain prose (rich text) content stored as Y.XmlFragment */
-    prose: Array<ProseFields<T>>;
-    /** Undo capture timeout in ms. Changes within this window merge into one undo. Default: 500 */
-    undoCaptureTimeout?: number;
-    /** Persistence provider for Y.Doc and key-value storage */
-    persistence: Persistence;
-}
-/** Editor binding for BlockNote/TipTap collaboration */
-export interface EditorBinding {
-    /** The Y.XmlFragment bound to the editor */
-    readonly fragment: Y.XmlFragment;
-    /** Provider stub for BlockNote compatibility */
-    readonly provider: {
-        readonly awareness: null;
-    };
-    /** Current sync state - true if unsent changes exist */
-    readonly pending: boolean;
-    /** Subscribe to pending state changes. Returns unsubscribe function. */
-    onPendingChange(callback: (pending: boolean) => void): () => void;
-    /** Undo the last content edit */
-    undo(): void;
-    /** Redo the last undone edit */
-    redo(): void;
-    /** Check if undo is available */
-    canUndo(): boolean;
-    /** Check if redo is available */
-    canRedo(): boolean;
-}
-/** Utilities exposed on collection.utils */
-interface ConvexCollectionUtils<T extends object> {
-    /**
-     * Get an editor binding for a prose field.
-     * Waits for Y.Doc to be ready (IndexedDB loaded) before returning.
-     * @param documentId - The document ID
-     * @param field - The prose field name (must be in `prose` config)
-     * @returns Promise resolving to EditorBinding
-     */
-    prose(documentId: string, field: ProseFields<T>): Promise<EditorBinding>;
-}
-/** Extended collection with prose field utilities */
-export interface ConvexCollection<T extends object> extends Collection<T> {
-    /** Utilities for prose field operations */
-    utils: ConvexCollectionUtils<T>;
-}
-/**
- * Create TanStack DB collection options with Convex + Yjs replication.
- *
- * @example
- * ```typescript
- * const options = convexCollectionOptions<Task>({
- *   getKey: (t) => t.id,
- *   convexClient,
- *   api: { stream: api.tasks.stream, insert: api.tasks.insert, ... },
- *   collection: 'tasks',
- * });
- * const collection = createCollection(options);
- * ```
- */
-export declare function convexCollectionOptions<T extends object>({ getKey, material, convexClient, api, collection, prose: proseFields, undoCaptureTimeout, persistence, }: ConvexCollectionOptionsConfig<T>): CollectionConfig<T> & {
-    _convexClient: ConvexClient;
-    _collection: string;
-    _proseFields: Array<ProseFields<T>>;
-    _persistence: Persistence;
-    utils: ConvexCollectionUtils<T>;
-};
-export {};