@trestleinc/replicate 0.1.0 → 1.1.0

package/dist/server/builder.d.ts

@@ -0,0 +1,94 @@
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
+/**
+ * Configuration for replicate handlers (without component - used with factory pattern).
+ */
+export interface ReplicateConfig<T extends object> {
+    collection: string;
+    /** Size threshold for auto-compaction (default: 5MB). Set to 0 to disable. */
+    compaction?: {
+        threshold?: number;
+    };
+    hooks?: {
+        evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+        evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+        evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+        onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
+        onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+        onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+        onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+        transform?: (docs: T[]) => T[] | Promise<T[]>;
+    };
+}
+/**
+ * Create a replicate function bound to your component. Call this once in your
+ * convex/replicate.ts file, then use the returned function for all collections.
+ *
+ * @example
+ * ```typescript
+ * // convex/replicate.ts (create once)
+ * import { replicate } from '@trestleinc/replicate/server';
+ * import { components } from './_generated/api';
+ *
+ * export const tasks = replicate(components.replicate)<Task>({ collection: 'tasks' });
+ *
+ * // Or bind once and reuse:
+ * const r = replicate(components.replicate);
+ * export const tasks = r<Task>({ collection: 'tasks' });
+ * export const notebooks = r<Notebook>({ collection: 'notebooks' });
+ * ```
+ */
+export declare function replicate(component: any): <T extends object>(config: ReplicateConfig<T>) => {
+    stream: import("convex/server").RegisteredQuery<"public", {
+        limit?: number | undefined;
+        vector?: ArrayBuffer | undefined;
+        checkpoint: {
+            lastModified: number;
+        };
+    }, Promise<any>>;
+    material: import("convex/server").RegisteredQuery<"public", {}, Promise<{
+        documents: T[];
+        checkpoint?: {
+            lastModified: number;
+        } | undefined;
+        count: number;
+        crdtBytes?: ArrayBuffer;
+    }>>;
+    recovery: import("convex/server").RegisteredQuery<"public", {
+        clientStateVector: ArrayBuffer;
+    }, Promise<any>>;
+    insert: import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+        materializedDoc: any;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+    update: import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+        materializedDoc: any;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+    remove: import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+};

package/dist/server/index.d.ts

@@ -1,17 +1,14 @@
-/**
- * Server-side utilities for Convex backend.
- * Import this in your Convex functions (convex/*.ts files).
- *
- * @example
- * ```typescript
- * // convex/tasks.ts
- * import {
- *   insertDocumentHelper,
- *   updateDocumentHelper,
- *   deleteDocumentHelper,
- *   streamHelper,
- * } from '@trestleinc/replicate/server';
- * ```
- */
-export { insertDocumentHelper, updateDocumentHelper, deleteDocumentHelper, streamHelper, } from './replication.js';
-export { replicatedTable, type ReplicationFields } from './schema.js';
+export { replicate } from '$/server/builder.js';
+export type { ReplicateConfig } from '$/server/builder.js';
+import { table } from '$/server/schema.js';
+export declare const schema: {
+    readonly table: typeof table;
+    readonly prose: () => import("convex/values").VObject<{
+        content?: any[] | undefined;
+        type: "doc";
+    }, {
+        type: import("convex/values").VLiteral<"doc", "required">;
+        content: import("convex/values").VArray<any[] | undefined, import("convex/values").VAny<any, "required", string>, "optional">;
+    }, "required", "type" | "content">;
+};
+export type { ReplicationFields } from '$/server/schema.js';

package/dist/server/schema.d.ts

@@ -1,73 +1,27 @@
+/** Fields automatically added to replicated tables */
+export type ReplicationFields = {
+    timestamp: number;
+};
+export declare const prose: () => import("convex/values").VObject<{
+    content?: any[] | undefined;
+    type: "doc";
+}, {
+    type: import("convex/values").VLiteral<"doc", "required">;
+    content: import("convex/values").VArray<any[] | undefined, import("convex/values").VAny<any, "required", string>, "optional">;
+}, "required", "type" | "content">;
 /**
- * Schema utilities for defining replicated tables.
- * Automatically adds replication metadata fields so users don't have to.
+ * Define a table with automatic timestamp field for replication.
+ * All replicated tables must have an `id` field and define a `by_doc_id` index.
  *
  * @example
  * ```typescript
  * // convex/schema.ts
- * import { defineSchema } from 'convex/server';
- * import { v } from 'convex/values';
- * import { replicatedTable } from '@trestleinc/replicate/server';
- *
  * export default defineSchema({
- *   tasks: replicatedTable(
- *     {
- *       id: v.string(),
- *       text: v.string(),
- *       isCompleted: v.boolean(),
- *     },
- *     (table) => table
- *       .index('by_id', ['id'])
- *       .index('by_timestamp', ['timestamp'])
+ *   tasks: table(
+ *     { id: v.string(), text: v.string(), isCompleted: v.boolean() },
+ *     (t) => t.index('by_doc_id', ['id']).index('by_completed', ['isCompleted'])
  *   ),
  * });
  * ```
  */
-/**
- * Internal replication metadata fields added to every replicated table.
- * These are managed automatically by the replication layer.
- */
-export type ReplicationFields = {
-    /** Version number for conflict resolution */
-    version: number;
-    /** Last modification timestamp (Unix ms) */
-    timestamp: number;
-};
-/**
- * Wraps a table definition to automatically add replication metadata fields.
- *
- * Users define their business logic fields, and we inject:
- * - `version` - For conflict resolution and CRDT versioning
- * - `timestamp` - For incremental sync and change tracking
- *
- * Enables:
- * - Dual-storage architecture (CRDT component + main table)
- * - Conflict-free replication across clients
- * - Hard delete support with CRDT history preservation
- * - Event sourcing via component storage
- *
- * @param userFields - User's business logic fields (id, text, etc.)
- * @param applyIndexes - Optional callback to add indexes to the table
- * @returns TableDefinition with replication fields injected
- *
- * @example
- * ```typescript
- * // Simple table with hard delete support
- * tasks: replicatedTable({
- *   id: v.string(),
- *   text: v.string(),
- * })
- *
- * // With indexes
- * tasks: replicatedTable(
- *   {
- *     id: v.string(),
- *     text: v.string(),
- *   },
- *   (table) => table
- *     .index('by_id', ['id'])
- *     .index('by_timestamp', ['timestamp'])
- * )
- * ```
- */
-export declare function replicatedTable(userFields: Record<string, any>, applyIndexes?: (table: any) => any): any;
+export declare function table(userFields: Record<string, any>, applyIndexes?: (table: any) => any): any;

package/dist/server/storage.d.ts

@@ -0,0 +1,80 @@
+import type { GenericMutationCtx, GenericQueryCtx, GenericDataModel } from 'convex/server';
+export declare class Replicate<T extends object> {
+    component: any;
+    collectionName: string;
+    private options?;
+    constructor(component: any, collectionName: string, options?: {
+        threshold?: number;
+    } | undefined);
+    createStreamQuery(opts?: {
+        evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+        onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
+    }): import("convex/server").RegisteredQuery<"public", {
+        limit?: number | undefined;
+        vector?: ArrayBuffer | undefined;
+        checkpoint: {
+            lastModified: number;
+        };
+    }, Promise<any>>;
+    createSSRQuery(opts?: {
+        evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+        transform?: (docs: T[]) => T[] | Promise<T[]>;
+        includeCRDTState?: boolean;
+    }): import("convex/server").RegisteredQuery<"public", {}, Promise<{
+        documents: T[];
+        checkpoint?: {
+            lastModified: number;
+        };
+        count: number;
+        crdtBytes?: ArrayBuffer;
+    }>>;
+    createInsertMutation(opts?: {
+        evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+        onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    }): import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+        materializedDoc: any;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+    createUpdateMutation(opts?: {
+        evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+        onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
+    }): import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+        materializedDoc: any;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+    createRemoveMutation(opts?: {
+        evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+        onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
+    }): import("convex/server").RegisteredMutation<"public", {
+        documentId: string;
+        crdtBytes: ArrayBuffer;
+    }, Promise<{
+        success: boolean;
+        metadata: {
+            documentId: string;
+            timestamp: number;
+            collection: string;
+        };
+    }>>;
+    createRecoveryQuery(opts?: {
+        evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
+    }): import("convex/server").RegisteredQuery<"public", {
+        clientStateVector: ArrayBuffer;
+    }, Promise<any>>;
+}

package/dist/server.js

@@ -1,96 +1,281 @@
-import { defineTable } from "convex/server";
 import { v } from "convex/values";
-function cleanDocument(doc) {
-    return Object.fromEntries(Object.entries(doc).filter(([_, value])=>null != value));
+import { defineTable, mutationGeneric, queryGeneric } from "convex/server";
+class Replicate {
+    component;
+    collectionName;
+    options;
+    constructor(component, collectionName, options){
+        this.component = component;
+        this.collectionName = collectionName;
+        this.options = options;
+    }
+    createStreamQuery(opts) {
+        const component = this.component;
+        const collection = this.collectionName;
+        return queryGeneric({
+            args: {
+                checkpoint: v.object({
+                    lastModified: v.number()
+                }),
+                limit: v.optional(v.number()),
+                vector: v.optional(v.bytes())
+            },
+            returns: v.object({
+                changes: v.array(v.object({
+                    documentId: v.optional(v.string()),
+                    crdtBytes: v.bytes(),
+                    version: v.number(),
+                    timestamp: v.number(),
+                    operationType: v.string()
+                })),
+                checkpoint: v.object({
+                    lastModified: v.number()
+                }),
+                hasMore: v.boolean()
+            }),
+            handler: async (ctx, args)=>{
+                if (opts?.evalRead) await opts.evalRead(ctx, collection);
+                const result = await ctx.runQuery(component.public.stream, {
+                    collection,
+                    checkpoint: args.checkpoint,
+                    limit: args.limit,
+                    vector: args.vector
+                });
+                if (opts?.onStream) await opts.onStream(ctx, result);
+                return result;
+            }
+        });
+    }
+    createSSRQuery(opts) {
+        const collection = this.collectionName;
+        const component = this.component;
+        return queryGeneric({
+            args: {},
+            returns: v.object({
+                documents: v.any(),
+                checkpoint: v.optional(v.object({
+                    lastModified: v.number()
+                })),
+                count: v.number(),
+                crdtBytes: v.optional(v.bytes())
+            }),
+            handler: async (ctx)=>{
+                if (opts?.evalRead) await opts.evalRead(ctx, collection);
+                let docs = await ctx.db.query(collection).collect();
+                if (opts?.transform) docs = await opts.transform(docs);
+                const latestTimestamp = docs.length > 0 ? Math.max(...docs.map((doc)=>doc.timestamp || 0)) : 0;
+                const response = {
+                    documents: docs,
+                    checkpoint: latestTimestamp > 0 ? {
+                        lastModified: latestTimestamp
+                    } : void 0,
+                    count: docs.length
+                };
+                if (opts?.includeCRDTState) {
+                    const crdtState = await ctx.runQuery(component.public.getInitialState, {
+                        collection
+                    });
+                    if (crdtState) {
+                        response.crdtBytes = crdtState.crdtBytes;
+                        response.checkpoint = crdtState.checkpoint;
+                    }
+                }
+                return response;
+            }
+        });
+    }
+    createInsertMutation(opts) {
+        const component = this.component;
+        const collection = this.collectionName;
+        const threshold = this.options?.threshold;
+        return mutationGeneric({
+            args: {
+                documentId: v.string(),
+                crdtBytes: v.bytes(),
+                materializedDoc: v.any()
+            },
+            returns: v.object({
+                success: v.boolean(),
+                metadata: v.any()
+            }),
+            handler: async (ctx, args)=>{
+                const doc = args.materializedDoc;
+                if (opts?.evalWrite) await opts.evalWrite(ctx, doc);
+                const version = Date.now();
+                await ctx.runMutation(component.public.insertDocument, {
+                    collection,
+                    documentId: args.documentId,
+                    crdtBytes: args.crdtBytes,
+                    version,
+                    threshold
+                });
+                await ctx.db.insert(collection, {
+                    id: args.documentId,
+                    ...args.materializedDoc,
+                    timestamp: Date.now()
+                });
+                if (opts?.onInsert) await opts.onInsert(ctx, doc);
+                return {
+                    success: true,
+                    metadata: {
+                        documentId: args.documentId,
+                        timestamp: Date.now(),
+                        collection
+                    }
+                };
+            }
+        });
+    }
+    createUpdateMutation(opts) {
+        const component = this.component;
+        const collection = this.collectionName;
+        const threshold = this.options?.threshold;
+        return mutationGeneric({
+            args: {
+                documentId: v.string(),
+                crdtBytes: v.bytes(),
+                materializedDoc: v.any()
+            },
+            returns: v.object({
+                success: v.boolean(),
+                metadata: v.any()
+            }),
+            handler: async (ctx, args)=>{
+                const doc = args.materializedDoc;
+                if (opts?.evalWrite) await opts.evalWrite(ctx, doc);
+                const version = Date.now();
+                await ctx.runMutation(component.public.updateDocument, {
+                    collection,
+                    documentId: args.documentId,
+                    crdtBytes: args.crdtBytes,
+                    version,
+                    threshold
+                });
+                const existing = await ctx.db.query(collection).withIndex('by_doc_id', (q)=>q.eq('id', args.documentId)).first();
+                if (existing) await ctx.db.patch(collection, existing._id, {
+                    ...args.materializedDoc,
+                    timestamp: Date.now()
+                });
+                if (opts?.onUpdate) await opts.onUpdate(ctx, doc);
+                return {
+                    success: true,
+                    metadata: {
+                        documentId: args.documentId,
+                        timestamp: Date.now(),
+                        collection
+                    }
+                };
+            }
+        });
+    }
+    createRemoveMutation(opts) {
+        const component = this.component;
+        const collection = this.collectionName;
+        const threshold = this.options?.threshold;
+        return mutationGeneric({
+            args: {
+                documentId: v.string(),
+                crdtBytes: v.bytes()
+            },
+            returns: v.object({
+                success: v.boolean(),
+                metadata: v.any()
+            }),
+            handler: async (ctx, args)=>{
+                const documentId = args.documentId;
+                if (opts?.evalRemove) await opts.evalRemove(ctx, documentId);
+                const version = Date.now();
+                await ctx.runMutation(component.public.deleteDocument, {
+                    collection,
+                    documentId: documentId,
+                    crdtBytes: args.crdtBytes,
+                    version,
+                    threshold
+                });
+                const existing = await ctx.db.query(collection).withIndex('by_doc_id', (q)=>q.eq('id', documentId)).first();
+                if (existing) await ctx.db.delete(collection, existing._id);
+                if (opts?.onRemove) await opts.onRemove(ctx, documentId);
+                return {
+                    success: true,
+                    metadata: {
+                        documentId: documentId,
+                        timestamp: Date.now(),
+                        collection
+                    }
+                };
+            }
+        });
+    }
+    createRecoveryQuery(opts) {
+        const component = this.component;
+        const collection = this.collectionName;
+        return queryGeneric({
+            args: {
+                clientStateVector: v.bytes()
+            },
+            returns: v.object({
+                diff: v.optional(v.bytes()),
+                serverStateVector: v.bytes()
+            }),
+            handler: async (ctx, args)=>{
+                if (opts?.evalRead) await opts.evalRead(ctx, collection);
+                return await ctx.runQuery(component.public.recovery, {
+                    collection,
+                    clientStateVector: args.clientStateVector
+                });
+            }
+        });
+    }
 }
-async function insertDocumentHelper(ctx, components, tableName, args) {
-    const timestamp = Date.now();
-    await ctx.runMutation(components.replicate.public.insertDocument, {
-        collectionName: tableName,
-        documentId: args.id,
-        crdtBytes: args.crdtBytes,
-        version: args.version
-    });
-    const db = ctx.db;
-    const cleanDoc = cleanDocument(args.materializedDoc);
-    await db.insert(tableName, {
-        id: args.id,
-        ...cleanDoc,
-        version: args.version,
-        timestamp
-    });
-    return {
-        success: true,
-        metadata: {
-            documentId: args.id,
-            timestamp,
-            version: args.version,
-            collectionName: tableName
-        }
+function replicate(component) {
+    return function(config) {
+        return replicateInternal(component, config);
     };
 }
-async function updateDocumentHelper(ctx, components, tableName, args) {
-    const timestamp = Date.now();
-    await ctx.runMutation(components.replicate.public.updateDocument, {
-        collectionName: tableName,
-        documentId: args.id,
-        crdtBytes: args.crdtBytes,
-        version: args.version
-    });
-    const db = ctx.db;
-    const existing = await db.query(tableName).withIndex('by_user_id', (q)=>q.eq('id', args.id)).first();
-    if (!existing) throw new Error(`Document ${args.id} not found in table ${tableName}`);
-    const cleanDoc = cleanDocument(args.materializedDoc);
-    await db.patch(existing._id, {
-        ...cleanDoc,
-        version: args.version,
-        timestamp
+function replicateInternal(component, config) {
+    const storage = new Replicate(component, config.collection, {
+        threshold: config.compaction?.threshold
     });
     return {
-        success: true,
-        metadata: {
-            documentId: args.id,
-            timestamp,
-            version: args.version,
-            collectionName: tableName
-        }
+        stream: storage.createStreamQuery({
+            evalRead: config.hooks?.evalRead,
+            onStream: config.hooks?.onStream
+        }),
+        material: storage.createSSRQuery({
+            evalRead: config.hooks?.evalRead,
+            transform: config.hooks?.transform
+        }),
+        recovery: storage.createRecoveryQuery({
+            evalRead: config.hooks?.evalRead
+        }),
+        insert: storage.createInsertMutation({
+            evalWrite: config.hooks?.evalWrite,
+            onInsert: config.hooks?.onInsert
+        }),
+        update: storage.createUpdateMutation({
+            evalWrite: config.hooks?.evalWrite,
+            onUpdate: config.hooks?.onUpdate
+        }),
+        remove: storage.createRemoveMutation({
+            evalRemove: config.hooks?.evalRemove,
+            onRemove: config.hooks?.onRemove
+        })
     };
 }
-async function deleteDocumentHelper(ctx, components, tableName, args) {
-    const timestamp = Date.now();
-    await ctx.runMutation(components.replicate.public.deleteDocument, {
-        collectionName: tableName,
-        documentId: args.id,
-        crdtBytes: args.crdtBytes,
-        version: args.version
+const prose = ()=>v.object({
+        type: v.literal('doc'),
+        content: v.optional(v.array(v.any()))
     });
-    const db = ctx.db;
-    const existing = await db.query(tableName).withIndex('by_user_id', (q)=>q.eq('id', args.id)).first();
-    if (existing) await db.delete(existing._id);
-    return {
-        success: true,
-        metadata: {
-            documentId: args.id,
-            timestamp,
-            version: args.version,
-            collectionName: tableName
-        }
-    };
-}
-async function streamHelper(ctx, components, tableName, args) {
-    return ctx.runQuery(components.replicate.public.stream, {
-        collectionName: tableName,
-        checkpoint: args.checkpoint,
-        limit: args.limit
-    });
-}
-function replicatedTable(userFields, applyIndexes) {
-    const tableWithMetadata = defineTable({
+function table(userFields, applyIndexes) {
+    const tbl = defineTable({
         ...userFields,
-        version: v.number(),
         timestamp: v.number()
     });
-    if (applyIndexes) return applyIndexes(tableWithMetadata);
-    return tableWithMetadata;
+    if (applyIndexes) return applyIndexes(tbl);
+    return tbl;
 }
-export { deleteDocumentHelper, insertDocumentHelper, replicatedTable, streamHelper, updateDocumentHelper };
+const schema = {
+    table: table,
+    prose: prose
+};
+export { replicate, schema };

package/dist/shared/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Shared types re-exported for top-level access
+ */
+export type { FragmentValue, XmlFragmentJSON, XmlNodeJSON } from "./types.js";
+export { OperationType } from "./types.js";

package/dist/shared/index.js

@@ -0,0 +1,2 @@
+import { OperationType } from "./types.js";
+export { OperationType };