@trestleinc/replicate 1.1.2 → 1.2.0-preview.0

package/dist/component/public.js

@@ -1,325 +0,0 @@
-import { getLogger } from "./logger.js";
-import { mutation, query } from "./_generated/server.js";
-import { OperationType } from "./shared/types.js";
-import * as Y from "yjs";
-import { v } from "convex/values";
-
-//#region src/component/public.ts
-const DEFAULT_SIZE_THRESHOLD = 5e6;
-const DEFAULT_PEER_TIMEOUT = 300 * 1e3;
-async function getNextSeq(ctx, collection) {
-	return ((await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", collection)).order("desc").first())?.seq ?? 0) + 1;
-}
-const insertDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const updateDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const deleteDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const mark = mutation({
-	args: {
-		collection: v.string(),
-		peerId: v.string(),
-		syncedSeq: v.number()
-	},
-	returns: v.null(),
-	handler: async (ctx, args) => {
-		const existing = await ctx.db.query("peers").withIndex("by_collection_peer", (q) => q.eq("collection", args.collection).eq("peerId", args.peerId)).first();
-		if (existing) await ctx.db.patch(existing._id, {
-			lastSyncedSeq: Math.max(existing.lastSyncedSeq, args.syncedSeq),
-			lastSeenAt: Date.now()
-		});
-		else await ctx.db.insert("peers", {
-			collection: args.collection,
-			peerId: args.peerId,
-			lastSyncedSeq: args.syncedSeq,
-			lastSeenAt: Date.now()
-		});
-		return null;
-	}
-});
-const compact = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		snapshotBytes: v.bytes(),
-		stateVector: v.bytes(),
-		peerTimeout: v.optional(v.number())
-	},
-	returns: v.object({
-		success: v.boolean(),
-		removed: v.number(),
-		retained: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["compaction"]);
-		const now = Date.now();
-		const peerCutoff = now - (args.peerTimeout ?? DEFAULT_PEER_TIMEOUT);
-		const deltas = await ctx.db.query("documents").withIndex("by_collection_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).collect();
-		const activePeers = await ctx.db.query("peers").withIndex("by_collection", (q) => q.eq("collection", args.collection)).filter((q) => q.gt(q.field("lastSeenAt"), peerCutoff)).collect();
-		const minSyncedSeq = activePeers.length > 0 ? Math.min(...activePeers.map((p) => p.lastSyncedSeq)) : Infinity;
-		const existingSnapshot = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).first();
-		if (existingSnapshot) await ctx.db.delete(existingSnapshot._id);
-		const snapshotSeq = deltas.length > 0 ? Math.max(...deltas.map((d) => d.seq)) : 0;
-		await ctx.db.insert("snapshots", {
-			collection: args.collection,
-			documentId: args.documentId,
-			snapshotBytes: args.snapshotBytes,
-			stateVector: args.stateVector,
-			snapshotSeq,
-			createdAt: now
-		});
-		let removed = 0;
-		for (const delta of deltas) if (delta.seq < minSyncedSeq) {
-			await ctx.db.delete(delta._id);
-			removed++;
-		}
-		logger.info("Compaction completed", {
-			collection: args.collection,
-			documentId: args.documentId,
-			removed,
-			retained: deltas.length - removed,
-			activePeers: activePeers.length,
-			minSyncedSeq
-		});
-		return {
-			success: true,
-			removed,
-			retained: deltas.length - removed
-		};
-	}
-});
-const stream = query({
-	args: {
-		collection: v.string(),
-		cursor: v.number(),
-		limit: v.optional(v.number()),
-		sizeThreshold: v.optional(v.number())
-	},
-	returns: v.object({
-		changes: v.array(v.object({
-			documentId: v.string(),
-			crdtBytes: v.bytes(),
-			seq: v.number(),
-			operationType: v.string()
-		})),
-		cursor: v.number(),
-		hasMore: v.boolean(),
-		compact: v.optional(v.string())
-	}),
-	handler: async (ctx, args) => {
-		const limit = args.limit ?? 100;
-		const sizeThreshold = args.sizeThreshold ?? DEFAULT_SIZE_THRESHOLD;
-		const documents = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection).gt("seq", args.cursor)).order("asc").take(limit);
-		if (documents.length > 0) {
-			const changes = documents.map((doc) => ({
-				documentId: doc.documentId,
-				crdtBytes: doc.crdtBytes,
-				seq: doc.seq,
-				operationType: OperationType.Delta
-			}));
-			const newCursor = documents[documents.length - 1]?.seq ?? args.cursor;
-			let compactHint;
-			const allDocs = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-			const sizeByDocument = /* @__PURE__ */ new Map();
-			for (const doc of allDocs) {
-				const current = sizeByDocument.get(doc.documentId) ?? 0;
-				sizeByDocument.set(doc.documentId, current + doc.crdtBytes.byteLength);
-			}
-			for (const [docId, size] of sizeByDocument) if (size > sizeThreshold) {
-				compactHint = docId;
-				break;
-			}
-			return {
-				changes,
-				cursor: newCursor,
-				hasMore: documents.length === limit,
-				compact: compactHint
-			};
-		}
-		const oldestDelta = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection)).order("asc").first();
-		if (oldestDelta && args.cursor < oldestDelta.seq) {
-			const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-			if (snapshots.length === 0) throw new Error(`Disparity detected but no snapshots available for collection: ${args.collection}. Client cursor: ${args.cursor}, Oldest delta seq: ${oldestDelta.seq}`);
-			return {
-				changes: snapshots.map((snapshot) => ({
-					documentId: snapshot.documentId,
-					crdtBytes: snapshot.snapshotBytes,
-					seq: snapshot.snapshotSeq,
-					operationType: OperationType.Snapshot
-				})),
-				cursor: Math.max(...snapshots.map((s) => s.snapshotSeq)),
-				hasMore: false,
-				compact: void 0
-			};
-		}
-		return {
-			changes: [],
-			cursor: args.cursor,
-			hasMore: false,
-			compact: void 0
-		};
-	}
-});
-const getInitialState = query({
-	args: { collection: v.string() },
-	returns: v.union(v.object({
-		crdtBytes: v.bytes(),
-		cursor: v.number()
-	}), v.null()),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["ssr"]);
-		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-		if (snapshots.length === 0 && deltas.length === 0) {
-			logger.info("No initial state available - collection is empty", { collection: args.collection });
-			return null;
-		}
-		const updates = [];
-		let latestSeq = 0;
-		for (const snapshot of snapshots) {
-			updates.push(new Uint8Array(snapshot.snapshotBytes));
-			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
-		}
-		const sorted = deltas.sort((a, b) => a.seq - b.seq);
-		for (const delta of sorted) {
-			updates.push(new Uint8Array(delta.crdtBytes));
-			latestSeq = Math.max(latestSeq, delta.seq);
-		}
-		logger.info("Reconstructing initial state", {
-			collection: args.collection,
-			snapshotCount: snapshots.length,
-			deltaCount: deltas.length
-		});
-		const merged = Y.mergeUpdatesV2(updates);
-		logger.info("Initial state reconstructed", {
-			collection: args.collection,
-			originalSize: updates.reduce((sum, u) => sum + u.byteLength, 0),
-			mergedSize: merged.byteLength
-		});
-		return {
-			crdtBytes: merged.buffer,
-			cursor: latestSeq
-		};
-	}
-});
-const recovery = query({
-	args: {
-		collection: v.string(),
-		clientStateVector: v.bytes()
-	},
-	returns: v.object({
-		diff: v.optional(v.bytes()),
-		serverStateVector: v.bytes(),
-		cursor: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["recovery"]);
-		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-		if (snapshots.length === 0 && deltas.length === 0) {
-			const emptyDoc = new Y.Doc();
-			const emptyVector = Y.encodeStateVector(emptyDoc);
-			emptyDoc.destroy();
-			return {
-				serverStateVector: emptyVector.buffer,
-				cursor: 0
-			};
-		}
-		const updates = [];
-		let latestSeq = 0;
-		for (const snapshot of snapshots) {
-			updates.push(new Uint8Array(snapshot.snapshotBytes));
-			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
-		}
-		for (const delta of deltas) {
-			updates.push(new Uint8Array(delta.crdtBytes));
-			latestSeq = Math.max(latestSeq, delta.seq);
-		}
-		const mergedState = Y.mergeUpdatesV2(updates);
-		const clientVector = new Uint8Array(args.clientStateVector);
-		const diff = Y.diffUpdateV2(mergedState, clientVector);
-		const serverVector = Y.encodeStateVectorFromUpdateV2(mergedState);
-		logger.info("Recovery sync computed", {
-			collection: args.collection,
-			snapshotCount: snapshots.length,
-			deltaCount: deltas.length,
-			diffSize: diff.byteLength,
-			hasDiff: diff.byteLength > 0
-		});
-		return {
-			diff: diff.byteLength > 0 ? diff.buffer : void 0,
-			serverStateVector: serverVector.buffer,
-			cursor: latestSeq
-		};
-	}
-});
-
-//#endregion
-export { OperationType, compact, deleteDocument, getInitialState, insertDocument, mark, recovery, stream, updateDocument };

package/dist/component/schema.d.ts

@@ -1,54 +0,0 @@
-import * as convex_server7 from "convex/server";
-import * as convex_values0 from "convex/values";
-
-//#region src/component/schema.d.ts
-declare const _default: convex_server7.SchemaDefinition<{
-  documents: convex_server7.TableDefinition<convex_values0.VObject<{
-    collection: string;
-    documentId: string;
-    crdtBytes: ArrayBuffer;
-    seq: number;
-  }, {
-    collection: convex_values0.VString<string, "required">;
-    documentId: convex_values0.VString<string, "required">;
-    crdtBytes: convex_values0.VBytes<ArrayBuffer, "required">;
-    seq: convex_values0.VFloat64<number, "required">;
-  }, "required", "collection" | "documentId" | "crdtBytes" | "seq">, {
-    by_collection: ["collection", "_creationTime"];
-    by_collection_document: ["collection", "documentId", "_creationTime"];
-    by_seq: ["collection", "seq", "_creationTime"];
-  }, {}, {}>;
-  snapshots: convex_server7.TableDefinition<convex_values0.VObject<{
-    collection: string;
-    documentId: string;
-    snapshotBytes: ArrayBuffer;
-    stateVector: ArrayBuffer;
-    snapshotSeq: number;
-    createdAt: number;
-  }, {
-    collection: convex_values0.VString<string, "required">;
-    documentId: convex_values0.VString<string, "required">;
-    snapshotBytes: convex_values0.VBytes<ArrayBuffer, "required">;
-    stateVector: convex_values0.VBytes<ArrayBuffer, "required">;
-    snapshotSeq: convex_values0.VFloat64<number, "required">;
-    createdAt: convex_values0.VFloat64<number, "required">;
-  }, "required", "collection" | "documentId" | "snapshotBytes" | "stateVector" | "snapshotSeq" | "createdAt">, {
-    by_document: ["collection", "documentId", "_creationTime"];
-  }, {}, {}>;
-  peers: convex_server7.TableDefinition<convex_values0.VObject<{
-    collection: string;
-    peerId: string;
-    lastSyncedSeq: number;
-    lastSeenAt: number;
-  }, {
-    collection: convex_values0.VString<string, "required">;
-    peerId: convex_values0.VString<string, "required">;
-    lastSyncedSeq: convex_values0.VFloat64<number, "required">;
-    lastSeenAt: convex_values0.VFloat64<number, "required">;
-  }, "required", "collection" | "peerId" | "lastSyncedSeq" | "lastSeenAt">, {
-    by_collection: ["collection", "_creationTime"];
-    by_collection_peer: ["collection", "peerId", "_creationTime"];
-  }, {}, {}>;
-}, true>;
-//#endregion
-export { _default as default };

package/dist/component/schema.js

@@ -1,29 +0,0 @@
-import { defineSchema, defineTable } from "convex/server";
-import { v } from "convex/values";
-
-//#region src/component/schema.ts
-var schema_default = defineSchema({
-	documents: defineTable({
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes(),
-		seq: v.number()
-	}).index("by_collection", ["collection"]).index("by_collection_document", ["collection", "documentId"]).index("by_seq", ["collection", "seq"]),
-	snapshots: defineTable({
-		collection: v.string(),
-		documentId: v.string(),
-		snapshotBytes: v.bytes(),
-		stateVector: v.bytes(),
-		snapshotSeq: v.number(),
-		createdAt: v.number()
-	}).index("by_document", ["collection", "documentId"]),
-	peers: defineTable({
-		collection: v.string(),
-		peerId: v.string(),
-		lastSyncedSeq: v.number(),
-		lastSeenAt: v.number()
-	}).index("by_collection", ["collection"]).index("by_collection_peer", ["collection", "peerId"])
-});
-
-//#endregion
-export { schema_default as default };

package/dist/component/shared/types.d.ts

@@ -1,9 +0,0 @@
-//#region src/shared/types.d.ts
-
-/** Operation type for streaming changes */
-declare enum OperationType {
-  Delta = "delta",
-  Snapshot = "snapshot",
-}
-//#endregion
-export { OperationType };

package/dist/component/shared/types.js

@@ -1,15 +0,0 @@
-//#region src/shared/types.ts
-/** Operation type for streaming changes */
-let OperationType = /* @__PURE__ */ function(OperationType$1) {
-	OperationType$1["Delta"] = "delta";
-	OperationType$1["Snapshot"] = "snapshot";
-	return OperationType$1;
-}({});
-const SIZE_MULTIPLIERS = {
-	kb: 1024,
-	mb: 1024 ** 2,
-	gb: 1024 ** 3
-};
-
-//#endregion
-export { OperationType };

package/dist/server/index.d.ts

@@ -1,135 +0,0 @@
-import * as convex_values0 from "convex/values";
-import * as convex_server0 from "convex/server";
-import { GenericDataModel, GenericMutationCtx, GenericQueryCtx } from "convex/server";
-
-//#region src/shared/types.d.ts
-
-type SizeUnit = "kb" | "mb" | "gb";
-type Size = `${number}${SizeUnit}`;
-type DurationUnit = "m" | "h" | "d";
-type Duration = `${number}${DurationUnit}`;
-interface CompactionConfig {
-  sizeThreshold: Size;
-  peerTimeout: Duration;
-}
-//#endregion
-//#region src/server/builder.d.ts
-/**
- * Configuration for replicate handlers (without component - used with factory pattern).
- */
-interface ReplicateConfig<T extends object> {
-  collection: string;
-  compaction?: Partial<CompactionConfig>;
-  hooks?: {
-    evalRead?: (ctx: GenericQueryCtx<GenericDataModel>, collection: string) => void | Promise<void>;
-    evalWrite?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
-    evalRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
-    evalMark?: (ctx: GenericMutationCtx<GenericDataModel>, peerId: string) => void | Promise<void>;
-    evalCompact?: (ctx: GenericMutationCtx<GenericDataModel>, documentId: string) => void | Promise<void>;
-    onStream?: (ctx: GenericQueryCtx<GenericDataModel>, result: any) => void | Promise<void>;
-    onInsert?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
-    onUpdate?: (ctx: GenericMutationCtx<GenericDataModel>, doc: T) => void | Promise<void>;
-    onRemove?: (ctx: GenericMutationCtx<GenericDataModel>, docId: string) => void | Promise<void>;
-    transform?: (docs: T[]) => T[] | Promise<T[]>;
-  };
-}
-/**
- * Create a replicate function bound to your component. Call this once in your
- * convex/replicate.ts file, then use the returned function for all collections.
- *
- * @example
- * ```typescript
- * // convex/replicate.ts (create once)
- * import { replicate } from '@trestleinc/replicate/server';
- * import { components } from './_generated/api';
- *
- * export const tasks = replicate(components.replicate)<Task>({ collection: 'tasks' });
- *
- * // Or bind once and reuse:
- * const r = replicate(components.replicate);
- * export const tasks = r<Task>({ collection: 'tasks' });
- * export const notebooks = r<Notebook>({ collection: 'notebooks' });
- * ```
- */
-declare function replicate(component: any): <T extends object>(config: ReplicateConfig<T>) => {
-  __collection: string;
-  stream: convex_server0.RegisteredQuery<"public", {
-    limit?: number | undefined;
-    sizeThreshold?: number | undefined;
-    cursor: number;
-  }, Promise<any>>;
-  material: convex_server0.RegisteredQuery<"public", {}, Promise<{
-    documents: T[];
-    cursor?: number;
-    count: number;
-    crdtBytes?: ArrayBuffer;
-  }>>;
-  recovery: convex_server0.RegisteredQuery<"public", {
-    clientStateVector: ArrayBuffer;
-  }, Promise<any>>;
-  insert: convex_server0.RegisteredMutation<"public", {
-    documentId: string;
-    crdtBytes: ArrayBuffer;
-    materializedDoc: any;
-  }, Promise<{
-    success: boolean;
-    seq: any;
-  }>>;
-  update: convex_server0.RegisteredMutation<"public", {
-    documentId: string;
-    crdtBytes: ArrayBuffer;
-    materializedDoc: any;
-  }, Promise<{
-    success: boolean;
-    seq: any;
-  }>>;
-  remove: convex_server0.RegisteredMutation<"public", {
-    documentId: string;
-    crdtBytes: ArrayBuffer;
-  }, Promise<{
-    success: boolean;
-    seq: any;
-  }>>;
-  mark: convex_server0.RegisteredMutation<"public", {
-    peerId: string;
-    syncedSeq: number;
-  }, Promise<null>>;
-  compact: convex_server0.RegisteredMutation<"public", {
-    peerTimeout?: number | undefined;
-    documentId: string;
-    snapshotBytes: ArrayBuffer;
-    stateVector: ArrayBuffer;
-  }, Promise<any>>;
-};
-//#endregion
-//#region src/server/schema.d.ts
-/**
- * Define a table with automatic timestamp field for replication.
- * All replicated tables must have an `id` field and define a `by_doc_id` index.
- *
- * @example
- * ```typescript
- * // convex/schema.ts
- * export default defineSchema({
- *   tasks: table(
- *     { id: v.string(), text: v.string(), isCompleted: v.boolean() },
- *     (t) => t.index('by_doc_id', ['id']).index('by_completed', ['isCompleted'])
- *   ),
- * });
- * ```
- */
-declare function table(userFields: Record<string, any>, applyIndexes?: (table: any) => any): any;
-//#endregion
-//#region src/server/index.d.ts
-declare const schema: {
-  readonly table: typeof table;
-  readonly prose: () => convex_values0.VObject<{
-    content?: any[] | undefined;
-    type: "doc";
-  }, {
-    type: convex_values0.VLiteral<"doc", "required">;
-    content: convex_values0.VArray<any[] | undefined, convex_values0.VAny<any, "required", string>, "optional">;
-  }, "required", "type" | "content">;
-};
-//#endregion
-export { type ReplicateConfig, replicate, schema };