@trestleinc/replicate 1.1.1 → 1.1.2-preview.0

package/dist/component/public.js

@@ -1,309 +1,325 @@
-import { Doc, applyUpdateV2, diffUpdateV2, encodeStateVector, encodeStateVectorFromUpdateV2, mergeUpdatesV2 } from "yjs";
-import { v } from "convex/values";
-import { mutation, query } from "./_generated/server.js";
 import { getLogger } from "./logger.js";
-import { OperationType } from "../shared/types.js";
-const DEFAULT_SIZE_THRESHOLD = 5000000;
-async function _maybeCompactDocument(ctx, collection, documentId, threshold = DEFAULT_SIZE_THRESHOLD) {
-    const logger = getLogger([
-        'compaction'
-    ]);
-    const deltas = await ctx.db.query('documents').withIndex('by_collection_document_version', (q)=>q.eq('collection', collection).eq('documentId', documentId)).collect();
-    const totalSize = deltas.reduce((sum, d)=>sum + d.crdtBytes.byteLength, 0);
-    if (totalSize < threshold) return null;
-    logger.info('Auto-compacting document', {
-        collection,
-        documentId,
-        deltaCount: deltas.length,
-        totalSize,
-        threshold
-    });
-    const sorted = deltas.sort((a, b)=>a.timestamp - b.timestamp);
-    const updates = sorted.map((d)=>new Uint8Array(d.crdtBytes));
-    const compactedState = mergeUpdatesV2(updates);
-    const testDoc = new Doc({
-        guid: `${collection}:${documentId}`
-    });
-    try {
-        applyUpdateV2(testDoc, compactedState);
-    } catch (error) {
-        logger.error('Compacted state validation failed', {
-            collection,
-            documentId,
-            error: String(error)
-        });
-        testDoc.destroy();
-        return null;
-    }
-    testDoc.destroy();
-    const existingSnapshot = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', collection).eq('documentId', documentId)).first();
-    if (existingSnapshot) await ctx.db.delete('snapshots', existingSnapshot._id);
-    await ctx.db.insert('snapshots', {
-        collection,
-        documentId,
-        snapshotBytes: compactedState.buffer,
-        latestCompactionTimestamp: sorted[sorted.length - 1].timestamp,
-        createdAt: Date.now(),
-        metadata: {
-            deltaCount: deltas.length,
-            totalSize
-        }
-    });
-    for (const delta of sorted)await ctx.db.delete('documents', delta._id);
-    logger.info('Auto-compaction completed', {
-        collection,
-        documentId,
-        deltasCompacted: deltas.length,
-        snapshotSize: compactedState.length
-    });
-    return {
-        deltasCompacted: deltas.length,
-        snapshotSize: compactedState.length
-    };
+import { mutation, query } from "./_generated/server.js";
+import { OperationType } from "./shared/types.js";
+import * as Y from "yjs";
+import { v } from "convex/values";
+
+//#region src/component/public.ts
+const DEFAULT_SIZE_THRESHOLD = 5e6;
+const DEFAULT_PEER_TIMEOUT = 300 * 1e3;
+async function getNextSeq(ctx, collection) {
+	return ((await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", collection)).order("desc").first())?.seq ?? 0) + 1;
 }
 const insertDocument = mutation({
-    args: {
-        collection: v.string(),
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
-        version: v.number(),
-        threshold: v.optional(v.number())
-    },
-    returns: v.object({
-        success: v.boolean(),
-        compacted: v.optional(v.boolean())
-    }),
-    handler: async (ctx, args)=>{
-        await ctx.db.insert('documents', {
-            collection: args.collection,
-            documentId: args.documentId,
-            crdtBytes: args.crdtBytes,
-            version: args.version,
-            timestamp: Date.now()
-        });
-        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
-        return {
-            success: true,
-            compacted: null !== compactionResult
-        };
-    }
+	args: {
+		collection: v.string(),
+		documentId: v.string(),
+		crdtBytes: v.bytes()
+	},
+	returns: v.object({
+		success: v.boolean(),
+		seq: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const seq = await getNextSeq(ctx, args.collection);
+		await ctx.db.insert("documents", {
+			collection: args.collection,
+			documentId: args.documentId,
+			crdtBytes: args.crdtBytes,
+			seq
+		});
+		return {
+			success: true,
+			seq
+		};
+	}
 });
 const updateDocument = mutation({
-    args: {
-        collection: v.string(),
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
-        version: v.number(),
-        threshold: v.optional(v.number())
-    },
-    returns: v.object({
-        success: v.boolean(),
-        compacted: v.optional(v.boolean())
-    }),
-    handler: async (ctx, args)=>{
-        await ctx.db.insert('documents', {
-            collection: args.collection,
-            documentId: args.documentId,
-            crdtBytes: args.crdtBytes,
-            version: args.version,
-            timestamp: Date.now()
-        });
-        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
-        return {
-            success: true,
-            compacted: null !== compactionResult
-        };
-    }
+	args: {
+		collection: v.string(),
+		documentId: v.string(),
+		crdtBytes: v.bytes()
+	},
+	returns: v.object({
+		success: v.boolean(),
+		seq: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const seq = await getNextSeq(ctx, args.collection);
+		await ctx.db.insert("documents", {
+			collection: args.collection,
+			documentId: args.documentId,
+			crdtBytes: args.crdtBytes,
+			seq
+		});
+		return {
+			success: true,
+			seq
+		};
+	}
 });
 const deleteDocument = mutation({
-    args: {
-        collection: v.string(),
-        documentId: v.string(),
-        crdtBytes: v.bytes(),
-        version: v.number(),
-        threshold: v.optional(v.number())
-    },
-    returns: v.object({
-        success: v.boolean(),
-        compacted: v.optional(v.boolean())
-    }),
-    handler: async (ctx, args)=>{
-        await ctx.db.insert('documents', {
-            collection: args.collection,
-            documentId: args.documentId,
-            crdtBytes: args.crdtBytes,
-            version: args.version,
-            timestamp: Date.now()
-        });
-        const compactionResult = await _maybeCompactDocument(ctx, args.collection, args.documentId, args.threshold ?? DEFAULT_SIZE_THRESHOLD);
-        return {
-            success: true,
-            compacted: null !== compactionResult
-        };
-    }
+	args: {
+		collection: v.string(),
+		documentId: v.string(),
+		crdtBytes: v.bytes()
+	},
+	returns: v.object({
+		success: v.boolean(),
+		seq: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const seq = await getNextSeq(ctx, args.collection);
+		await ctx.db.insert("documents", {
+			collection: args.collection,
+			documentId: args.documentId,
+			crdtBytes: args.crdtBytes,
+			seq
+		});
+		return {
+			success: true,
+			seq
+		};
+	}
+});
+const mark = mutation({
+	args: {
+		collection: v.string(),
+		peerId: v.string(),
+		syncedSeq: v.number()
+	},
+	returns: v.null(),
+	handler: async (ctx, args) => {
+		const existing = await ctx.db.query("peers").withIndex("by_collection_peer", (q) => q.eq("collection", args.collection).eq("peerId", args.peerId)).first();
+		if (existing) await ctx.db.patch(existing._id, {
+			lastSyncedSeq: Math.max(existing.lastSyncedSeq, args.syncedSeq),
+			lastSeenAt: Date.now()
+		});
+		else await ctx.db.insert("peers", {
+			collection: args.collection,
+			peerId: args.peerId,
+			lastSyncedSeq: args.syncedSeq,
+			lastSeenAt: Date.now()
+		});
+		return null;
+	}
+});
+const compact = mutation({
+	args: {
+		collection: v.string(),
+		documentId: v.string(),
+		snapshotBytes: v.bytes(),
+		stateVector: v.bytes(),
+		peerTimeout: v.optional(v.number())
+	},
+	returns: v.object({
+		success: v.boolean(),
+		removed: v.number(),
+		retained: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const logger = getLogger(["compaction"]);
+		const now = Date.now();
+		const peerCutoff = now - (args.peerTimeout ?? DEFAULT_PEER_TIMEOUT);
+		const deltas = await ctx.db.query("documents").withIndex("by_collection_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).collect();
+		const activePeers = await ctx.db.query("peers").withIndex("by_collection", (q) => q.eq("collection", args.collection)).filter((q) => q.gt(q.field("lastSeenAt"), peerCutoff)).collect();
+		const minSyncedSeq = activePeers.length > 0 ? Math.min(...activePeers.map((p) => p.lastSyncedSeq)) : Infinity;
+		const existingSnapshot = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).first();
+		if (existingSnapshot) await ctx.db.delete(existingSnapshot._id);
+		const snapshotSeq = deltas.length > 0 ? Math.max(...deltas.map((d) => d.seq)) : 0;
+		await ctx.db.insert("snapshots", {
+			collection: args.collection,
+			documentId: args.documentId,
+			snapshotBytes: args.snapshotBytes,
+			stateVector: args.stateVector,
+			snapshotSeq,
+			createdAt: now
+		});
+		let removed = 0;
+		for (const delta of deltas) if (delta.seq < minSyncedSeq) {
+			await ctx.db.delete(delta._id);
+			removed++;
+		}
+		logger.info("Compaction completed", {
+			collection: args.collection,
+			documentId: args.documentId,
+			removed,
+			retained: deltas.length - removed,
+			activePeers: activePeers.length,
+			minSyncedSeq
+		});
+		return {
+			success: true,
+			removed,
+			retained: deltas.length - removed
+		};
+	}
 });
 const stream = query({
-    args: {
-        collection: v.string(),
-        checkpoint: v.object({
-            lastModified: v.number()
-        }),
-        vector: v.optional(v.bytes()),
-        limit: v.optional(v.number())
-    },
-    returns: v.object({
-        changes: v.array(v.object({
-            documentId: v.optional(v.string()),
-            crdtBytes: v.bytes(),
-            version: v.number(),
-            timestamp: v.number(),
-            operationType: v.string()
-        })),
-        checkpoint: v.object({
-            lastModified: v.number()
-        }),
-        hasMore: v.boolean()
-    }),
-    handler: async (ctx, args)=>{
-        const limit = args.limit ?? 100;
-        const documents = await ctx.db.query('documents').withIndex('by_timestamp', (q)=>q.eq('collection', args.collection).gt('timestamp', args.checkpoint.lastModified)).order('asc').take(limit);
-        if (documents.length > 0) {
-            const changes = documents.map((doc)=>({
-                    documentId: doc.documentId,
-                    crdtBytes: doc.crdtBytes,
-                    version: doc.version,
-                    timestamp: doc.timestamp,
-                    operationType: OperationType.Delta
-                }));
-            const newCheckpoint = {
-                lastModified: documents[documents.length - 1]?.timestamp ?? args.checkpoint.lastModified
-            };
-            return {
-                changes,
-                checkpoint: newCheckpoint,
-                hasMore: documents.length === limit
-            };
-        }
-        const oldestDelta = await ctx.db.query('documents').withIndex('by_timestamp', (q)=>q.eq('collection', args.collection)).order('asc').first();
-        if (oldestDelta && args.checkpoint.lastModified < oldestDelta.timestamp) {
-            const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
-            if (0 === snapshots.length) throw new Error(`Disparity detected but no snapshots available for collection: ${args.collection}. Client checkpoint: ${args.checkpoint.lastModified}, Oldest delta: ${oldestDelta.timestamp}`);
-            const changes = snapshots.map((snapshot)=>({
-                    documentId: snapshot.documentId,
-                    crdtBytes: snapshot.snapshotBytes,
-                    version: 0,
-                    timestamp: snapshot.createdAt,
-                    operationType: OperationType.Snapshot
-                }));
-            const latestTimestamp = Math.max(...snapshots.map((s)=>s.latestCompactionTimestamp));
-            return {
-                changes,
-                checkpoint: {
-                    lastModified: latestTimestamp
-                },
-                hasMore: false
-            };
-        }
-        return {
-            changes: [],
-            checkpoint: args.checkpoint,
-            hasMore: false
-        };
-    }
+	args: {
+		collection: v.string(),
+		cursor: v.number(),
+		limit: v.optional(v.number()),
+		sizeThreshold: v.optional(v.number())
+	},
+	returns: v.object({
+		changes: v.array(v.object({
+			documentId: v.string(),
+			crdtBytes: v.bytes(),
+			seq: v.number(),
+			operationType: v.string()
+		})),
+		cursor: v.number(),
+		hasMore: v.boolean(),
+		compact: v.optional(v.string())
+	}),
+	handler: async (ctx, args) => {
+		const limit = args.limit ?? 100;
+		const sizeThreshold = args.sizeThreshold ?? DEFAULT_SIZE_THRESHOLD;
+		const documents = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection).gt("seq", args.cursor)).order("asc").take(limit);
+		if (documents.length > 0) {
+			const changes = documents.map((doc) => ({
+				documentId: doc.documentId,
+				crdtBytes: doc.crdtBytes,
+				seq: doc.seq,
+				operationType: OperationType.Delta
+			}));
+			const newCursor = documents[documents.length - 1]?.seq ?? args.cursor;
+			let compactHint;
+			const allDocs = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
+			const sizeByDocument = /* @__PURE__ */ new Map();
+			for (const doc of allDocs) {
+				const current = sizeByDocument.get(doc.documentId) ?? 0;
+				sizeByDocument.set(doc.documentId, current + doc.crdtBytes.byteLength);
+			}
+			for (const [docId, size] of sizeByDocument) if (size > sizeThreshold) {
+				compactHint = docId;
+				break;
+			}
+			return {
+				changes,
+				cursor: newCursor,
+				hasMore: documents.length === limit,
+				compact: compactHint
+			};
+		}
+		const oldestDelta = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection)).order("asc").first();
+		if (oldestDelta && args.cursor < oldestDelta.seq) {
+			const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
+			if (snapshots.length === 0) throw new Error(`Disparity detected but no snapshots available for collection: ${args.collection}. Client cursor: ${args.cursor}, Oldest delta seq: ${oldestDelta.seq}`);
+			return {
+				changes: snapshots.map((snapshot) => ({
+					documentId: snapshot.documentId,
+					crdtBytes: snapshot.snapshotBytes,
+					seq: snapshot.snapshotSeq,
+					operationType: OperationType.Snapshot
+				})),
+				cursor: Math.max(...snapshots.map((s) => s.snapshotSeq)),
+				hasMore: false,
+				compact: void 0
+			};
+		}
+		return {
+			changes: [],
+			cursor: args.cursor,
+			hasMore: false,
+			compact: void 0
+		};
+	}
 });
 const getInitialState = query({
-    args: {
-        collection: v.string()
-    },
-    returns: v.union(v.object({
-        crdtBytes: v.bytes(),
-        checkpoint: v.object({
-            lastModified: v.number()
-        })
-    }), v["null"]()),
-    handler: async (ctx, args)=>{
-        const logger = getLogger([
-            'ssr'
-        ]);
-        const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
-        const deltas = await ctx.db.query('documents').withIndex('by_collection', (q)=>q.eq('collection', args.collection)).collect();
-        if (0 === snapshots.length && 0 === deltas.length) {
-            logger.info('No initial state available - collection is empty', {
-                collection: args.collection
-            });
-            return null;
-        }
-        const updates = [];
-        let latestTimestamp = 0;
-        for (const snapshot of snapshots){
-            updates.push(new Uint8Array(snapshot.snapshotBytes));
-            latestTimestamp = Math.max(latestTimestamp, snapshot.latestCompactionTimestamp);
-        }
-        const sorted = deltas.sort((a, b)=>a.timestamp - b.timestamp);
-        for (const delta of sorted){
-            updates.push(new Uint8Array(delta.crdtBytes));
-            latestTimestamp = Math.max(latestTimestamp, delta.timestamp);
-        }
-        logger.info('Reconstructing initial state', {
-            collection: args.collection,
-            snapshotCount: snapshots.length,
-            deltaCount: deltas.length
-        });
-        const merged = mergeUpdatesV2(updates);
-        logger.info('Initial state reconstructed', {
-            collection: args.collection,
-            originalSize: updates.reduce((sum, u)=>sum + u.byteLength, 0),
-            mergedSize: merged.byteLength
-        });
-        return {
-            crdtBytes: merged.buffer,
-            checkpoint: {
-                lastModified: latestTimestamp
-            }
-        };
-    }
+	args: { collection: v.string() },
+	returns: v.union(v.object({
+		crdtBytes: v.bytes(),
+		cursor: v.number()
+	}), v.null()),
+	handler: async (ctx, args) => {
+		const logger = getLogger(["ssr"]);
+		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
+		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
+		if (snapshots.length === 0 && deltas.length === 0) {
+			logger.info("No initial state available - collection is empty", { collection: args.collection });
+			return null;
+		}
+		const updates = [];
+		let latestSeq = 0;
+		for (const snapshot of snapshots) {
+			updates.push(new Uint8Array(snapshot.snapshotBytes));
+			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
+		}
+		const sorted = deltas.sort((a, b) => a.seq - b.seq);
+		for (const delta of sorted) {
+			updates.push(new Uint8Array(delta.crdtBytes));
+			latestSeq = Math.max(latestSeq, delta.seq);
+		}
+		logger.info("Reconstructing initial state", {
+			collection: args.collection,
+			snapshotCount: snapshots.length,
+			deltaCount: deltas.length
+		});
+		const merged = Y.mergeUpdatesV2(updates);
+		logger.info("Initial state reconstructed", {
+			collection: args.collection,
+			originalSize: updates.reduce((sum, u) => sum + u.byteLength, 0),
+			mergedSize: merged.byteLength
+		});
+		return {
+			crdtBytes: merged.buffer,
+			cursor: latestSeq
+		};
+	}
 });
 const recovery = query({
-    args: {
-        collection: v.string(),
-        clientStateVector: v.bytes()
-    },
-    returns: v.object({
-        diff: v.optional(v.bytes()),
-        serverStateVector: v.bytes()
-    }),
-    handler: async (ctx, args)=>{
-        const logger = getLogger([
-            'recovery'
-        ]);
-        const snapshots = await ctx.db.query('snapshots').withIndex('by_document', (q)=>q.eq('collection', args.collection)).collect();
-        const deltas = await ctx.db.query('documents').withIndex('by_collection', (q)=>q.eq('collection', args.collection)).collect();
-        if (0 === snapshots.length && 0 === deltas.length) {
-            const emptyDoc = new Doc();
-            const emptyVector = encodeStateVector(emptyDoc);
-            emptyDoc.destroy();
-            return {
-                serverStateVector: emptyVector.buffer
-            };
-        }
-        const updates = [];
-        for (const snapshot of snapshots)updates.push(new Uint8Array(snapshot.snapshotBytes));
-        for (const delta of deltas)updates.push(new Uint8Array(delta.crdtBytes));
-        const mergedState = mergeUpdatesV2(updates);
-        const clientVector = new Uint8Array(args.clientStateVector);
-        const diff = diffUpdateV2(mergedState, clientVector);
-        const serverVector = encodeStateVectorFromUpdateV2(mergedState);
-        logger.info('Recovery sync computed', {
-            collection: args.collection,
-            snapshotCount: snapshots.length,
-            deltaCount: deltas.length,
-            diffSize: diff.byteLength,
-            hasDiff: diff.byteLength > 0
-        });
-        return {
-            diff: diff.byteLength > 0 ? diff.buffer : void 0,
-            serverStateVector: serverVector.buffer
-        };
-    }
+	args: {
+		collection: v.string(),
+		clientStateVector: v.bytes()
+	},
+	returns: v.object({
+		diff: v.optional(v.bytes()),
+		serverStateVector: v.bytes(),
+		cursor: v.number()
+	}),
+	handler: async (ctx, args) => {
+		const logger = getLogger(["recovery"]);
+		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
+		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
+		if (snapshots.length === 0 && deltas.length === 0) {
+			const emptyDoc = new Y.Doc();
+			const emptyVector = Y.encodeStateVector(emptyDoc);
+			emptyDoc.destroy();
+			return {
+				serverStateVector: emptyVector.buffer,
+				cursor: 0
+			};
+		}
+		const updates = [];
+		let latestSeq = 0;
+		for (const snapshot of snapshots) {
+			updates.push(new Uint8Array(snapshot.snapshotBytes));
+			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
+		}
+		for (const delta of deltas) {
+			updates.push(new Uint8Array(delta.crdtBytes));
+			latestSeq = Math.max(latestSeq, delta.seq);
+		}
+		const mergedState = Y.mergeUpdatesV2(updates);
+		const clientVector = new Uint8Array(args.clientStateVector);
+		const diff = Y.diffUpdateV2(mergedState, clientVector);
+		const serverVector = Y.encodeStateVectorFromUpdateV2(mergedState);
+		logger.info("Recovery sync computed", {
+			collection: args.collection,
+			snapshotCount: snapshots.length,
+			deltaCount: deltas.length,
+			diffSize: diff.byteLength,
+			hasDiff: diff.byteLength > 0
+		});
+		return {
+			diff: diff.byteLength > 0 ? diff.buffer : void 0,
+			serverStateVector: serverVector.buffer,
+			cursor: latestSeq
+		};
+	}
 });
-export { OperationType, deleteDocument, getInitialState, insertDocument, recovery, stream, updateDocument };
+
+//#endregion
+export { OperationType, compact, deleteDocument, getInitialState, insertDocument, mark, recovery, stream, updateDocument };