@trestleinc/replicate 0.1.0 → 1.1.0

package/src/server/builder.ts

@@ -0,0 +1,85 @@
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
+import { Replicate } from '$/server/storage.js';
+
+/**
+ * Configuration for replicate handlers (without component - used with factory pattern).
+ */
+export interface ReplicateConfig<T extends object> {
+  collection: string;
+  /** Size threshold for auto-compaction (default: 5MB). Set to 0 to disable. */
+  compaction?: { threshold?: number };
+  hooks?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
+    onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    transform?: (docs: T[]) => T[] | Promise<T[]>;
+  };
+}
+
+/**
+ * Create a replicate function bound to your component. Call this once in your
+ * convex/replicate.ts file, then use the returned function for all collections.
+ *
+ * @example
+ * ```typescript
+ * // convex/replicate.ts (create once)
+ * import { replicate } from '@trestleinc/replicate/server';
+ * import { components } from './_generated/api';
+ *
+ * export const tasks = replicate(components.replicate)<Task>({ collection: 'tasks' });
+ *
+ * // Or bind once and reuse:
+ * const r = replicate(components.replicate);
+ * export const tasks = r<Task>({ collection: 'tasks' });
+ * export const notebooks = r<Notebook>({ collection: 'notebooks' });
+ * ```
+ */
+export function replicate(component: any) {
+  return function boundReplicate<T extends object>(config: ReplicateConfig<T>) {
+    return replicateInternal<T>(component, config);
+  };
+}
+
+/**
+ * Internal implementation for replicate.
+ */
+function replicateInternal<T extends object>(component: any, config: ReplicateConfig<T>) {
+  const storage = new Replicate<T>(component, config.collection, {
+    threshold: config.compaction?.threshold,
+  });
+
+  return {
+    stream: storage.createStreamQuery({
+      evalRead: config.hooks?.evalRead,
+      onStream: config.hooks?.onStream,
+    }),
+
+    material: storage.createSSRQuery({
+      evalRead: config.hooks?.evalRead,
+      transform: config.hooks?.transform,
+    }),
+
+    recovery: storage.createRecoveryQuery({
+      evalRead: config.hooks?.evalRead,
+    }),
+
+    insert: storage.createInsertMutation({
+      evalWrite: config.hooks?.evalWrite,
+      onInsert: config.hooks?.onInsert,
+    }),
+
+    update: storage.createUpdateMutation({
+      evalWrite: config.hooks?.evalWrite,
+      onUpdate: config.hooks?.onUpdate,
+    }),
+
+    remove: storage.createRemoveMutation({
+      evalRemove: config.hooks?.evalRemove,
+      onRemove: config.hooks?.onRemove,
+    }),
+  };
+}

package/src/server/index.ts

@@ -1,26 +1,11 @@
-/**
- * Server-side utilities for Convex backend.
- * Import this in your Convex functions (convex/*.ts files).
- *
- * @example
- * ```typescript
- * // convex/tasks.ts
- * import {
- *   insertDocumentHelper,
- *   updateDocumentHelper,
- *   deleteDocumentHelper,
- *   streamHelper,
- * } from '@trestleinc/replicate/server';
- * ```
- */
+export { replicate } from '$/server/builder.js';
+export type { ReplicateConfig } from '$/server/builder.js';
 
-// Replication helpers for mutations/queries
-export {
-  insertDocumentHelper,
-  updateDocumentHelper,
-  deleteDocumentHelper,
-  streamHelper,
-} from './replication.js';
+import { table, prose } from '$/server/schema.js';
 
-// Schema utilities
-export { replicatedTable, type ReplicationFields } from './schema.js';
+export const schema = {
+  table,
+  prose,
+} as const;
+
+export type { ReplicationFields } from '$/server/schema.js';

package/src/server/schema.ts

@@ -1,97 +1,41 @@
-/**
- * Schema utilities for defining replicated tables.
- * Automatically adds replication metadata fields so users don't have to.
- *
- * @example
- * ```typescript
- * // convex/schema.ts
- * import { defineSchema } from 'convex/server';
- * import { v } from 'convex/values';
- * import { replicatedTable } from '@trestleinc/replicate/server';
- *
- * export default defineSchema({
- *   tasks: replicatedTable(
- *     {
- *       id: v.string(),
- *       text: v.string(),
- *       isCompleted: v.boolean(),
- *     },
- *     (table) => table
- *       .index('by_id', ['id'])
- *       .index('by_timestamp', ['timestamp'])
- *   ),
- * });
- * ```
- */
-
 import { defineTable } from 'convex/server';
 import { v } from 'convex/values';
 
-/**
- * Internal replication metadata fields added to every replicated table.
- * These are managed automatically by the replication layer.
- */
+/** Fields automatically added to replicated tables */
 export type ReplicationFields = {
-  /** Version number for conflict resolution */
-  version: number;
-  /** Last modification timestamp (Unix ms) */
   timestamp: number;
 };
 
+export const prose = () =>
+  v.object({
+    type: v.literal('doc'),
+    content: v.optional(v.array(v.any())),
+  });
+
 /**
- * Wraps a table definition to automatically add replication metadata fields.
- *
- * Users define their business logic fields, and we inject:
- * - `version` - For conflict resolution and CRDT versioning
- * - `timestamp` - For incremental sync and change tracking
- *
- * Enables:
- * - Dual-storage architecture (CRDT component + main table)
- * - Conflict-free replication across clients
- * - Hard delete support with CRDT history preservation
- * - Event sourcing via component storage
- *
- * @param userFields - User's business logic fields (id, text, etc.)
- * @param applyIndexes - Optional callback to add indexes to the table
- * @returns TableDefinition with replication fields injected
+ * Define a table with automatic timestamp field for replication.
+ * All replicated tables must have an `id` field and define a `by_doc_id` index.
  *
  * @example
  * ```typescript
- * // Simple table with hard delete support
- * tasks: replicatedTable({
- *   id: v.string(),
- *   text: v.string(),
- * })
- *
- * // With indexes
- * tasks: replicatedTable(
- *   {
- *     id: v.string(),
- *     text: v.string(),
- *   },
- *   (table) => table
- *     .index('by_id', ['id'])
- *     .index('by_timestamp', ['timestamp'])
- * )
+ * // convex/schema.ts
+ * export default defineSchema({
+ *   tasks: table(
+ *     { id: v.string(), text: v.string(), isCompleted: v.boolean() },
+ *     (t) => t.index('by_doc_id', ['id']).index('by_completed', ['isCompleted'])
+ *   ),
+ * });
  * ```
  */
-export function replicatedTable(
-  userFields: Record<string, any>,
-  applyIndexes?: (table: any) => any
-): any {
-  // Create table with user fields + replication metadata
-  const tableWithMetadata = defineTable({
+export function table(userFields: Record<string, any>, applyIndexes?: (table: any) => any): any {
+  const tbl = defineTable({
     ...userFields,
-
-    // Injected replication fields (hidden from user's mental model)
-    version: v.number(),
     timestamp: v.number(),
   });
 
-  // Apply user-defined indexes if provided
   if (applyIndexes) {
-    return applyIndexes(tableWithMetadata);
+    return applyIndexes(tbl);
   }
 
-  return tableWithMetadata;
+  return tbl;
 }

package/src/server/storage.ts

@@ -0,0 +1,313 @@
+import { v } from 'convex/values';
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
+import { queryGeneric, mutationGeneric } from 'convex/server';
+
+export class Replicate<T extends object> {
+  constructor(
+    public component: any,
+    public collectionName: string,
+    private options?: { threshold?: number }
+  ) {}
+
+  createStreamQuery(opts?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return queryGeneric({
+      args: {
+        checkpoint: v.object({ lastModified: v.number() }),
+        limit: v.optional(v.number()),
+        vector: v.optional(v.bytes()),
+      },
+      returns: v.object({
+        changes: v.array(
+          v.object({
+            documentId: v.optional(v.string()),
+            crdtBytes: v.bytes(),
+            version: v.number(),
+            timestamp: v.number(),
+            operationType: v.string(),
+          })
+        ),
+        checkpoint: v.object({ lastModified: v.number() }),
+        hasMore: v.boolean(),
+      }),
+      handler: async (ctx, args) => {
+        if (opts?.evalRead) {
+          await opts.evalRead(ctx, collection);
+        }
+        const result = await ctx.runQuery(component.public.stream, {
+          collection,
+          checkpoint: args.checkpoint,
+          limit: args.limit,
+          vector: args.vector,
+        });
+
+        if (opts?.onStream) {
+          await opts.onStream(ctx, result);
+        }
+
+        return result;
+      },
+    });
+  }
+
+  createSSRQuery(opts?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    transform?: (docs: T[]) => T[] | Promise<T[]>;
+    includeCRDTState?: boolean;
+  }) {
+    const collection = this.collectionName;
+    const component = this.component;
+
+    return queryGeneric({
+      args: {},
+      returns: v.object({
+        documents: v.any(),
+        checkpoint: v.optional(v.object({ lastModified: v.number() })),
+        count: v.number(),
+        crdtBytes: v.optional(v.bytes()),
+      }),
+      handler: async (ctx) => {
+        if (opts?.evalRead) {
+          await opts.evalRead(ctx, collection);
+        }
+        let docs = (await ctx.db.query(collection).collect()) as T[];
+        if (opts?.transform) {
+          docs = await opts.transform(docs);
+        }
+
+        const latestTimestamp =
+          docs.length > 0 ? Math.max(...docs.map((doc: any) => doc.timestamp || 0)) : 0;
+
+        const response: {
+          documents: T[];
+          checkpoint?: { lastModified: number };
+          count: number;
+          crdtBytes?: ArrayBuffer;
+        } = {
+          documents: docs,
+          checkpoint: latestTimestamp > 0 ? { lastModified: latestTimestamp } : undefined,
+          count: docs.length,
+        };
+
+        if (opts?.includeCRDTState) {
+          const crdtState = await ctx.runQuery(component.public.getInitialState, {
+            collection,
+          });
+
+          if (crdtState) {
+            response.crdtBytes = crdtState.crdtBytes;
+            response.checkpoint = crdtState.checkpoint;
+          }
+        }
+        return response;
+      },
+    });
+  }
+
+  createInsertMutation(opts?: {
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+    const threshold = this.options?.threshold;
+
+    return mutationGeneric({
+      args: {
+        documentId: v.string(),
+        crdtBytes: v.bytes(),
+        materializedDoc: v.any(),
+      },
+      returns: v.object({
+        success: v.boolean(),
+        metadata: v.any(),
+      }),
+      handler: async (ctx, args) => {
+        const doc = args.materializedDoc as T;
+
+        if (opts?.evalWrite) {
+          await opts.evalWrite(ctx, doc);
+        }
+
+        const version = Date.now();
+        await ctx.runMutation(component.public.insertDocument, {
+          collection,
+          documentId: args.documentId,
+          crdtBytes: args.crdtBytes,
+          version,
+          threshold,
+        });
+
+        await ctx.db.insert(collection, {
+          id: args.documentId,
+          ...(args.materializedDoc as object),
+          timestamp: Date.now(),
+        });
+
+        if (opts?.onInsert) {
+          await opts.onInsert(ctx, doc);
+        }
+
+        return {
+          success: true,
+          metadata: {
+            documentId: args.documentId,
+            timestamp: Date.now(),
+            collection,
+          },
+        };
+      },
+    });
+  }
+
+  createUpdateMutation(opts?: {
+    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+    const threshold = this.options?.threshold;
+
+    return mutationGeneric({
+      args: {
+        documentId: v.string(),
+        crdtBytes: v.bytes(),
+        materializedDoc: v.any(),
+      },
+      returns: v.object({
+        success: v.boolean(),
+        metadata: v.any(),
+      }),
+      handler: async (ctx, args) => {
+        const doc = args.materializedDoc as T;
+
+        if (opts?.evalWrite) {
+          await opts.evalWrite(ctx, doc);
+        }
+
+        const version = Date.now();
+        await ctx.runMutation(component.public.updateDocument, {
+          collection,
+          documentId: args.documentId,
+          crdtBytes: args.crdtBytes,
+          version,
+          threshold,
+        });
+
+        const existing = await ctx.db
+          .query(collection)
+          .withIndex('by_doc_id', (q) => q.eq('id', args.documentId))
+          .first();
+
+        if (existing) {
+          await ctx.db.patch(collection, existing._id, {
+            ...(args.materializedDoc as object),
+            timestamp: Date.now(),
+          });
+        }
+
+        if (opts?.onUpdate) {
+          await opts.onUpdate(ctx, doc);
+        }
+
+        return {
+          success: true,
+          metadata: {
+            documentId: args.documentId,
+            timestamp: Date.now(),
+            collection,
+          },
+        };
+      },
+    });
+  }
+
+  createRemoveMutation(opts?: {
+    evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+    const threshold = this.options?.threshold;
+
+    return mutationGeneric({
+      args: {
+        documentId: v.string(),
+        crdtBytes: v.bytes(),
+      },
+      returns: v.object({
+        success: v.boolean(),
+        metadata: v.any(),
+      }),
+      handler: async (ctx, args) => {
+        const documentId = args.documentId as string;
+        if (opts?.evalRemove) {
+          await opts.evalRemove(ctx, documentId);
+        }
+
+        const version = Date.now();
+        await ctx.runMutation(component.public.deleteDocument, {
+          collection,
+          documentId: documentId,
+          crdtBytes: args.crdtBytes,
+          version,
+          threshold,
+        });
+
+        const existing = await ctx.db
+          .query(collection)
+          .withIndex('by_doc_id', (q) => q.eq('id', documentId))
+          .first();
+
+        if (existing) {
+          await ctx.db.delete(collection, existing._id);
+        }
+
+        if (opts?.onRemove) {
+          await opts.onRemove(ctx, documentId);
+        }
+
+        return {
+          success: true,
+          metadata: {
+            documentId: documentId,
+            timestamp: Date.now(),
+            collection,
+          },
+        };
+      },
+    });
+  }
+
+  createRecoveryQuery(opts?: {
+    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+  }) {
+    const component = this.component;
+    const collection = this.collectionName;
+
+    return queryGeneric({
+      args: {
+        clientStateVector: v.bytes(),
+      },
+      returns: v.object({
+        diff: v.optional(v.bytes()),
+        serverStateVector: v.bytes(),
+      }),
+      handler: async (ctx, args) => {
+        if (opts?.evalRead) {
+          await opts.evalRead(ctx, collection);
+        }
+
+        return await ctx.runQuery(component.public.recovery, {
+          collection,
+          clientStateVector: args.clientStateVector,
+        });
+      },
+    });
+  }
+}

package/src/shared/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Shared types re-exported for top-level access
+ */
+export type { FragmentValue, XmlFragmentJSON, XmlNodeJSON } from "./types.js";
+export { OperationType } from "./types.js";

package/src/shared/types.ts

@@ -0,0 +1,52 @@
+/**
+ * Shared types for @trestleinc/replicate
+ *
+ * These types are used across client, server, and component code.
+ * They are safe to import in any environment (browser, Node.js, Convex).
+ */
+
+/** Marker used during insert/update to signal a fragment field */
+export interface FragmentValue {
+  __xmlFragment: true;
+  content?: XmlFragmentJSON;
+}
+
+/** ProseMirror-compatible JSON for XmlFragment serialization */
+export interface XmlFragmentJSON {
+  type: "doc";
+  content?: XmlNodeJSON[];
+}
+
+/** ProseMirror node structure */
+export interface XmlNodeJSON {
+  type: string;
+  attrs?: Record<string, unknown>;
+  content?: XmlNodeJSON[];
+  text?: string;
+  marks?: { type: string; attrs?: Record<string, unknown> }[];
+}
+
+/** Operation type for streaming changes */
+export enum OperationType {
+  Delta = "delta",
+  Snapshot = "snapshot",
+}
+
+/**
+ * Extract field names from T where the value type is XmlFragmentJSON.
+ * Used for type-safe prose field configuration.
+ *
+ * @example
+ * ```typescript
+ * interface Notebook {
+ *   id: string;
+ *   title: string;
+ *   content: XmlFragmentJSON;
+ * }
+ *
+ * type Fields = ProseFields<Notebook>; // 'content'
+ * ```
+ */
+export type ProseFields<T> = {
+  [K in keyof T]: T[K] extends XmlFragmentJSON ? K : never;
+}[keyof T];