@trestleinc/replicate 0.1.0 → 1.1.0

package/dist/component/logger.js

@@ -0,0 +1,30 @@
+class ComponentLogger {
+    category;
+    constructor(category){
+        this.category = category;
+    }
+    format(level, message, context) {
+        const prefix = `[${this.category.join(':')}]`;
+        const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+        return `${prefix} ${level}: ${message}${contextStr}`;
+    }
+    debug(message, context) {
+        console.log(this.format('DEBUG', message, context));
+    }
+    info(message, context) {
+        console.log(this.format('INFO', message, context));
+    }
+    warn(message, context) {
+        console.warn(this.format('WARN', message, context));
+    }
+    error(message, context) {
+        console.error(this.format('ERROR', message, context));
+    }
+}
+function getLogger(category) {
+    return new ComponentLogger([
+        'component',
+        ...category
+    ]);
+}
+export { getLogger };

package/dist/component/public.d.ts

@@ -1,87 +1,39 @@
-/**
- * Insert a new document with CRDT bytes (Yjs format).
- * Appends delta to event log (event sourcing pattern).
- *
- * @param collectionName - Collection identifier
- * @param documentId - Unique document identifier
- * @param crdtBytes - ArrayBuffer containing Yjs CRDT bytes (delta)
- * @param version - CRDT version number
- */
+import { OperationType } from '$/shared/types.js';
+export { OperationType };
 export declare const insertDocument: import("convex/server").RegisteredMutation<"public", {
-    collectionName: string;
+    threshold?: number | undefined;
+    collection: string;
     documentId: string;
     crdtBytes: ArrayBuffer;
     version: number;
 }, Promise<{
     success: boolean;
+    compacted: boolean;
 }>>;
-/**
- * Update an existing document with new CRDT bytes (Yjs format).
- * Appends delta to event log (event sourcing pattern).
- *
- * @param collectionName - Collection identifier
- * @param documentId - Unique document identifier
- * @param crdtBytes - ArrayBuffer containing Yjs CRDT bytes (delta)
- * @param version - CRDT version number
- */
 export declare const updateDocument: import("convex/server").RegisteredMutation<"public", {
-    collectionName: string;
+    threshold?: number | undefined;
+    collection: string;
     documentId: string;
     crdtBytes: ArrayBuffer;
     version: number;
 }, Promise<{
     success: boolean;
+    compacted: boolean;
 }>>;
-/**
- * Delete a document from CRDT storage.
- * Appends deletion delta to event log (preserves history).
- *
- * @param collectionName - Collection identifier
- * @param documentId - Unique document identifier
- * @param crdtBytes - ArrayBuffer containing Yjs deletion delta
- * @param version - CRDT version number
- */
 export declare const deleteDocument: import("convex/server").RegisteredMutation<"public", {
-    collectionName: string;
+    threshold?: number | undefined;
+    collection: string;
     documentId: string;
     crdtBytes: ArrayBuffer;
     version: number;
 }, Promise<{
     success: boolean;
+    compacted: boolean;
 }>>;
-/**
- * Get complete event history for a document.
- * Returns all CRDT deltas in chronological order.
- *
- * Used for:
- * - Future recovery features (client-side)
- * - Audit trails
- * - Debugging
- *
- * @param collectionName - Collection identifier
- * @param documentId - Unique document identifier
- */
-export declare const getDocumentHistory: import("convex/server").RegisteredQuery<"public", {
-    collectionName: string;
-    documentId: string;
-}, Promise<{
-    crdtBytes: ArrayBuffer;
-    version: number;
-    timestamp: number;
-    operationType: string;
-}[]>>;
-/**
- * Stream CRDT changes for incremental replication.
- * Returns Yjs CRDT bytes for documents modified since the checkpoint.
- * Can be used for both polling (awaitReplication) and subscriptions (live updates).
- *
- * @param collectionName - Collection identifier
- * @param checkpoint - Last replication checkpoint
- * @param limit - Maximum number of changes to return (default: 100)
- */
 export declare const stream: import("convex/server").RegisteredQuery<"public", {
     limit?: number | undefined;
-    collectionName: string;
+    vector?: ArrayBuffer | undefined;
+    collection: string;
     checkpoint: {
         lastModified: number;
     };
@@ -91,9 +43,32 @@ export declare const stream: import("convex/server").RegisteredQuery<"public", {
         crdtBytes: ArrayBuffer;
         version: number;
         timestamp: number;
+        operationType: OperationType;
     }[];
     checkpoint: {
         lastModified: number;
     };
     hasMore: boolean;
 }>>;
+export declare const getInitialState: import("convex/server").RegisteredQuery<"public", {
+    collection: string;
+}, Promise<{
+    crdtBytes: ArrayBuffer;
+    checkpoint: {
+        lastModified: number;
+    };
+} | null>>;
+/**
+ * Recovery query for state vector based sync.
+ * Client sends its state vector, server computes and returns the diff.
+ */
+export declare const recovery: import("convex/server").RegisteredQuery<"public", {
+    collection: string;
+    clientStateVector: ArrayBuffer;
+}, Promise<{
+    serverStateVector: ArrayBuffer;
+    diff?: undefined;
+} | {
+    diff: ArrayBuffer | undefined;
+    serverStateVector: ArrayBuffer;
+}>>;

package/dist/component/public.js

@@ -1,112 +1,163 @@
+import { Doc, applyUpdateV2, diffUpdateV2, encodeStateVector, encodeStateVectorFromUpdateV2, mergeUpdatesV2 } from "yjs";
 import { v } from "convex/values";
 import { mutation, query } from "./_generated/server.js";
+import { getLogger } from "./logger.js";
+import { OperationType } from "../shared/types.js";
+const DEFAULT_SIZE_THRESHOLD = 5000000;
+async function _maybeCompactDocument(ctx, collection, documentId, threshold = DEFAULT_SIZE_THRESHOLD) {
+    const logger = getLogger([
+        'compaction'
+    ]);
+    const deltas = await ctx.db.query('documents').withIndex('by_collection_document_version', (q)=>q.eq('collection', collection).eq('documentId', documentId)).collect();
+    const totalSize = deltas.reduce((sum, d)=>sum + d.crdtBytes.byteLength, 0);
+    if (totalSize < threshold) return null;
+    logger.info('Auto-compacting document', {
+        collection,
+        documentId,
+        deltaCount: deltas.length,
+        totalSize,
+        threshold
+    });
+    const sorted = deltas.sort((a, b)=>a.timestamp - b.timestamp);
+    const updates = sorted.map((d)=>new Uint8Array(d.crdtBytes));
+    const compactedState = mergeUpdatesV2(updates);
+    const testDoc = new Doc({
+        guid: `${collection}:${documentId}`
+    });
+    try {
+        applyUpdateV2(testDoc, compactedState);
+    } catch (error) {
+        logger.error('Compacted state validation failed', {
+            collection,
+            documentId,
+            error: String(error)
+        });
+        testDoc.destroy();
+        return null;
+    }
+    testDoc.destroy();
+    const existingSnapshot = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', collection).eq('documentId', documentId)).first();
+    if (existingSnapshot) await ctx.db.delete('snapshots', existingSnapshot._id);
+    await ctx.db.insert('snapshots', {
+        collection,
+        documentId,
+        snapshotBytes: compactedState.buffer,
+        latestCompactionTimestamp: sorted[sorted.length - 1].timestamp,
+        createdAt: Date.now(),
+        metadata: {
+            deltaCount: deltas.length,
+            totalSize
+        }
+    });
+    for (const delta of sorted)await ctx.db.delete('documents', delta._id);
+    logger.info('Auto-compaction completed', {
+        collection,
+        documentId,
+        deltasCompacted: deltas.length,
+        snapshotSize: compactedState.length
+    });
+    return {
+        deltasCompacted: deltas.length,
+        snapshotSize: compactedState.length
+    };
+}
 const insertDocument = mutation({
     args: {
-        collectionName: v.string(),
+        collection: v.string(),
         documentId: v.string(),
         crdtBytes: v.bytes(),
-        version: v.number()
+        version: v.number(),
+        threshold: v.optional(v.number())
     },
     returns: v.object({
-        success: v.boolean()
+        success: v.boolean(),
+        compacted: v.optional(v.boolean())
     }),
     handler: async (ctx, args)=>{
         await ctx.db.insert('documents', {
-            collectionName: args.collectionName,
+            collection: args.collection,
             documentId: args.documentId,
             crdtBytes: args.crdtBytes,
             version: args.version,
-            timestamp: Date.now(),
-            operationType: 'insert'
+            timestamp: Date.now()
         });
+        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
         return {
-            success: true
+            success: true,
+            compacted: null !== compactionResult
         };
     }
 });
 const updateDocument = mutation({
     args: {
-        collectionName: v.string(),
+        collection: v.string(),
         documentId: v.string(),
         crdtBytes: v.bytes(),
-        version: v.number()
+        version: v.number(),
+        threshold: v.optional(v.number())
     },
     returns: v.object({
-        success: v.boolean()
+        success: v.boolean(),
+        compacted: v.optional(v.boolean())
     }),
     handler: async (ctx, args)=>{
         await ctx.db.insert('documents', {
-            collectionName: args.collectionName,
+            collection: args.collection,
             documentId: args.documentId,
             crdtBytes: args.crdtBytes,
             version: args.version,
-            timestamp: Date.now(),
-            operationType: 'update'
+            timestamp: Date.now()
         });
+        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
         return {
-            success: true
+            success: true,
+            compacted: null !== compactionResult
         };
     }
 });
 const deleteDocument = mutation({
     args: {
-        collectionName: v.string(),
+        collection: v.string(),
         documentId: v.string(),
         crdtBytes: v.bytes(),
-        version: v.number()
+        version: v.number(),
+        threshold: v.optional(v.number())
     },
     returns: v.object({
-        success: v.boolean()
+        success: v.boolean(),
+        compacted: v.optional(v.boolean())
     }),
     handler: async (ctx, args)=>{
         await ctx.db.insert('documents', {
-            collectionName: args.collectionName,
+            collection: args.collection,
             documentId: args.documentId,
             crdtBytes: args.crdtBytes,
             version: args.version,
-            timestamp: Date.now(),
-            operationType: 'delete'
+            timestamp: Date.now()
         });
+        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
         return {
-            success: true
+            success: true,
+            compacted: null !== compactionResult
         };
     }
 });
-const getDocumentHistory = query({
-    args: {
-        collectionName: v.string(),
-        documentId: v.string()
-    },
-    returns: v.array(v.object({
-        crdtBytes: v.bytes(),
-        version: v.number(),
-        timestamp: v.number(),
-        operationType: v.string()
-    })),
-    handler: async (ctx, args)=>{
-        const deltas = await ctx.db.query('documents').withIndex('by_collection_document_version', (q)=>q.eq('collectionName', args.collectionName).eq('documentId', args.documentId)).order('asc').collect();
-        return deltas.map((d)=>({
-                crdtBytes: d.crdtBytes,
-                version: d.version,
-                timestamp: d.timestamp,
-                operationType: d.operationType
-            }));
-    }
-});
 const stream = query({
     args: {
-        collectionName: v.string(),
+        collection: v.string(),
         checkpoint: v.object({
             lastModified: v.number()
         }),
+        vector: v.optional(v.bytes()),
         limit: v.optional(v.number())
     },
     returns: v.object({
         changes: v.array(v.object({
-            documentId: v.string(),
+            documentId: v.optional(v.string()),
             crdtBytes: v.bytes(),
             version: v.number(),
-            timestamp: v.number()
+            timestamp: v.number(),
+            operationType: v.string()
         })),
         checkpoint: v.object({
             lastModified: v.number()
@@ -115,21 +166,144 @@ const stream = query({
     }),
     handler: async (ctx, args)=>{
         const limit = args.limit ?? 100;
-        const documents = await ctx.db.query('documents').withIndex('by_timestamp', (q)=>q.eq('collectionName', args.collectionName).gt('timestamp', args.checkpoint.lastModified)).order('asc').take(limit);
-        const changes = documents.map((doc)=>({
-                documentId: doc.documentId,
-                crdtBytes: doc.crdtBytes,
-                version: doc.version,
-                timestamp: doc.timestamp
-            }));
-        const newCheckpoint = {
-            lastModified: documents.length > 0 ? documents[documents.length - 1]?.timestamp ?? args.checkpoint.lastModified : args.checkpoint.lastModified
+        const documents = await ctx.db.query('documents').withIndex('by_timestamp', (q)=>q.eq('collection', args.collection).gt('timestamp', args.checkpoint.lastModified)).order('asc').take(limit);
+        if (documents.length > 0) {
+            const changes = documents.map((doc)=>({
+                    documentId: doc.documentId,
+                    crdtBytes: doc.crdtBytes,
+                    version: doc.version,
+                    timestamp: doc.timestamp,
+                    operationType: OperationType.Delta
+                }));
+            const newCheckpoint = {
+                lastModified: documents[documents.length - 1]?.timestamp ?? args.checkpoint.lastModified
+            };
+            return {
+                changes,
+                checkpoint: newCheckpoint,
+                hasMore: documents.length === limit
+            };
+        }
+        const oldestDelta = await ctx.db.query('documents').withIndex('by_timestamp', (q)=>q.eq('collection', args.collection)).order('asc').first();
+        if (oldestDelta && args.checkpoint.lastModified < oldestDelta.timestamp) {
+            const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
+            if (0 === snapshots.length) throw new Error(`Disparity detected but no snapshots available for collection: ${args.collection}. Client checkpoint: ${args.checkpoint.lastModified}, Oldest delta: ${oldestDelta.timestamp}`);
+            const changes = snapshots.map((snapshot)=>({
+                    documentId: snapshot.documentId,
+                    crdtBytes: snapshot.snapshotBytes,
+                    version: 0,
+                    timestamp: snapshot.createdAt,
+                    operationType: OperationType.Snapshot
+                }));
+            const latestTimestamp = Math.max(...snapshots.map((s)=>s.latestCompactionTimestamp));
+            return {
+                changes,
+                checkpoint: {
+                    lastModified: latestTimestamp
+                },
+                hasMore: false
+            };
+        }
+        return {
+            changes: [],
+            checkpoint: args.checkpoint,
+            hasMore: false
         };
+    }
+});
+const getInitialState = query({
+    args: {
+        collection: v.string()
+    },
+    returns: v.union(v.object({
+        crdtBytes: v.bytes(),
+        checkpoint: v.object({
+            lastModified: v.number()
+        })
+    }), v["null"]()),
+    handler: async (ctx, args)=>{
+        const logger = getLogger([
+            'ssr'
+        ]);
+        const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
+        const deltas = await ctx.db.query('documents').withIndex('by_collection', (q)=>q.eq('collection', args.collection)).collect();
+        if (0 === snapshots.length && 0 === deltas.length) {
+            logger.info('No initial state available - collection is empty', {
+                collection: args.collection
+            });
+            return null;
+        }
+        const updates = [];
+        let latestTimestamp = 0;
+        for (const snapshot of snapshots){
+            updates.push(new Uint8Array(snapshot.snapshotBytes));
+            latestTimestamp = Math.max(latestTimestamp, snapshot.latestCompactionTimestamp);
+        }
+        const sorted = deltas.sort((a, b)=>a.timestamp - b.timestamp);
+        for (const delta of sorted){
+            updates.push(new Uint8Array(delta.crdtBytes));
+            latestTimestamp = Math.max(latestTimestamp, delta.timestamp);
+        }
+        logger.info('Reconstructing initial state', {
+            collection: args.collection,
+            snapshotCount: snapshots.length,
+            deltaCount: deltas.length
+        });
+        const merged = mergeUpdatesV2(updates);
+        logger.info('Initial state reconstructed', {
+            collection: args.collection,
+            originalSize: updates.reduce((sum, u)=>sum + u.byteLength, 0),
+            mergedSize: merged.byteLength
+        });
+        return {
+            crdtBytes: merged.buffer,
+            checkpoint: {
+                lastModified: latestTimestamp
+            }
+        };
+    }
+});
+const recovery = query({
+    args: {
+        collection: v.string(),
+        clientStateVector: v.bytes()
+    },
+    returns: v.object({
+        diff: v.optional(v.bytes()),
+        serverStateVector: v.bytes()
+    }),
+    handler: async (ctx, args)=>{
+        const logger = getLogger([
+            'recovery'
+        ]);
+        const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
+        const deltas = await ctx.db.query('documents').withIndex('by_collection', (q)=>q.eq('collection', args.collection)).collect();
+        if (0 === snapshots.length && 0 === deltas.length) {
+            const emptyDoc = new Doc();
+            const emptyVector = encodeStateVector(emptyDoc);
+            emptyDoc.destroy();
+            return {
+                serverStateVector: emptyVector.buffer
+            };
+        }
+        const updates = [];
+        for (const snapshot of snapshots)updates.push(new Uint8Array(snapshot.snapshotBytes));
+        for (const delta of deltas)updates.push(new Uint8Array(delta.crdtBytes));
+        const mergedState = mergeUpdatesV2(updates);
+        const clientVector = new Uint8Array(args.clientStateVector);
+        const diff = diffUpdateV2(mergedState, clientVector);
+        const serverVector = encodeStateVectorFromUpdateV2(mergedState);
+        logger.info('Recovery sync computed', {
+            collection: args.collection,
+            snapshotCount: snapshots.length,
+            deltaCount: deltas.length,
+            diffSize: diff.byteLength,
+            hasDiff: diff.byteLength > 0
+        });
         return {
-            changes,
-            checkpoint: newCheckpoint,
-            hasMore: documents.length === limit
+            diff: diff.byteLength > 0 ? diff.buffer : void 0,
+            serverStateVector: serverVector.buffer
         };
     }
 });
-export { deleteDocument, getDocumentHistory, insertDocument, stream, updateDocument };
+export { OperationType, deleteDocument, getInitialState, insertDocument, recovery, stream, updateDocument };

package/dist/component/schema.d.ts

@@ -1,22 +1,46 @@
 declare const _default: import("convex/server").SchemaDefinition<{
     documents: import("convex/server").TableDefinition<import("convex/values").VObject<{
-        collectionName: string;
+        collection: string;
         documentId: string;
         crdtBytes: ArrayBuffer;
         version: number;
         timestamp: number;
-        operationType: string;
     }, {
-        collectionName: import("convex/values").VString<string, "required">;
+        collection: import("convex/values").VString<string, "required">;
         documentId: import("convex/values").VString<string, "required">;
         crdtBytes: import("convex/values").VBytes<ArrayBuffer, "required">;
         version: import("convex/values").VFloat64<number, "required">;
         timestamp: import("convex/values").VFloat64<number, "required">;
-        operationType: import("convex/values").VString<string, "required">;
-    }, "required", "collectionName" | "documentId" | "crdtBytes" | "version" | "timestamp" | "operationType">, {
-        by_collection: ["collectionName", "_creationTime"];
-        by_collection_document_version: ["collectionName", "documentId", "version", "_creationTime"];
-        by_timestamp: ["collectionName", "timestamp", "_creationTime"];
+    }, "required", "collection" | "documentId" | "crdtBytes" | "version" | "timestamp">, {
+        by_collection: ["collection", "_creationTime"];
+        by_collection_document_version: ["collection", "documentId", "version", "_creationTime"];
+        by_timestamp: ["collection", "timestamp", "_creationTime"];
+    }, {}, {}>;
+    snapshots: import("convex/server").TableDefinition<import("convex/values").VObject<{
+        metadata?: {
+            deltaCount: number;
+            totalSize: number;
+        } | undefined;
+        collection: string;
+        documentId: string;
+        createdAt: number;
+        snapshotBytes: ArrayBuffer;
+        latestCompactionTimestamp: number;
+    }, {
+        collection: import("convex/values").VString<string, "required">;
+        documentId: import("convex/values").VString<string, "required">;
+        snapshotBytes: import("convex/values").VBytes<ArrayBuffer, "required">;
+        latestCompactionTimestamp: import("convex/values").VFloat64<number, "required">;
+        createdAt: import("convex/values").VFloat64<number, "required">;
+        metadata: import("convex/values").VObject<{
+            deltaCount: number;
+            totalSize: number;
+        } | undefined, {
+            deltaCount: import("convex/values").VFloat64<number, "required">;
+            totalSize: import("convex/values").VFloat64<number, "required">;
+        }, "optional", "deltaCount" | "totalSize">;
+    }, "required", "collection" | "documentId" | "createdAt" | "metadata" | "snapshotBytes" | "latestCompactionTimestamp" | "metadata.deltaCount" | "metadata.totalSize">, {
+        by_document: ["collection", "documentId", "_creationTime"];
     }, {}, {}>;
 }, true>;
 export default _default;

package/dist/component/schema.js

@@ -2,21 +2,34 @@ import { defineSchema, defineTable } from "convex/server";
 import { v } from "convex/values";
 const schema = defineSchema({
     documents: defineTable({
-        collectionName: v.string(),
+        collection: v.string(),
         documentId: v.string(),
         crdtBytes: v.bytes(),
         version: v.number(),
-        timestamp: v.number(),
-        operationType: v.string()
+        timestamp: v.number()
     }).index('by_collection', [
-        'collectionName'
+        'collection'
     ]).index('by_collection_document_version', [
-        'collectionName',
+        'collection',
         'documentId',
         'version'
     ]).index('by_timestamp', [
-        'collectionName',
+        'collection',
         'timestamp'
+    ]),
+    snapshots: defineTable({
+        collection: v.string(),
+        documentId: v.string(),
+        snapshotBytes: v.bytes(),
+        latestCompactionTimestamp: v.number(),
+        createdAt: v.number(),
+        metadata: v.optional(v.object({
+            deltaCount: v.number(),
+            totalSize: v.number()
+        }))
+    }).index('by_document', [
+        'collection',
+        'documentId'
     ])
 });
 export { schema as default };