@trestleinc/replicate 1.1.2 → 1.2.0-preview.1

package/dist/component/public.d.ts

@@ -1,83 +0,0 @@
-import { OperationType } from "./shared/types.js";
-import * as convex_server0 from "convex/server";
-
-//#region src/component/public.d.ts
-declare namespace public_d_exports {
-  export { OperationType, compact, deleteDocument, getInitialState, insertDocument, mark, recovery, stream, updateDocument };
-}
-declare const insertDocument: convex_server0.RegisteredMutation<"public", {
-  collection: string;
-  documentId: string;
-  crdtBytes: ArrayBuffer;
-}, Promise<{
-  success: boolean;
-  seq: number;
-}>>;
-declare const updateDocument: convex_server0.RegisteredMutation<"public", {
-  collection: string;
-  documentId: string;
-  crdtBytes: ArrayBuffer;
-}, Promise<{
-  success: boolean;
-  seq: number;
-}>>;
-declare const deleteDocument: convex_server0.RegisteredMutation<"public", {
-  collection: string;
-  documentId: string;
-  crdtBytes: ArrayBuffer;
-}, Promise<{
-  success: boolean;
-  seq: number;
-}>>;
-declare const mark: convex_server0.RegisteredMutation<"public", {
-  collection: string;
-  peerId: string;
-  syncedSeq: number;
-}, Promise<null>>;
-declare const compact: convex_server0.RegisteredMutation<"public", {
-  peerTimeout?: number | undefined;
-  collection: string;
-  documentId: string;
-  snapshotBytes: ArrayBuffer;
-  stateVector: ArrayBuffer;
-}, Promise<{
-  success: boolean;
-  removed: number;
-  retained: number;
-}>>;
-declare const stream: convex_server0.RegisteredQuery<"public", {
-  limit?: number | undefined;
-  sizeThreshold?: number | undefined;
-  collection: string;
-  cursor: number;
-}, Promise<{
-  changes: {
-    documentId: any;
-    crdtBytes: any;
-    seq: any;
-    operationType: OperationType;
-  }[];
-  cursor: number;
-  hasMore: boolean;
-  compact: string | undefined;
-}>>;
-declare const getInitialState: convex_server0.RegisteredQuery<"public", {
-  collection: string;
-}, Promise<{
-  crdtBytes: ArrayBuffer;
-  cursor: number;
-} | null>>;
-declare const recovery: convex_server0.RegisteredQuery<"public", {
-  collection: string;
-  clientStateVector: ArrayBuffer;
-}, Promise<{
-  serverStateVector: ArrayBuffer;
-  cursor: number;
-  diff?: undefined;
-} | {
-  diff: ArrayBuffer | undefined;
-  serverStateVector: ArrayBuffer;
-  cursor: number;
-}>>;
-//#endregion
-export { OperationType, compact, deleteDocument, getInitialState, insertDocument, mark, public_d_exports, recovery, stream, updateDocument };

package/dist/component/public.js

@@ -1,325 +0,0 @@
-import { getLogger } from "./logger.js";
-import { mutation, query } from "./_generated/server.js";
-import { OperationType } from "./shared/types.js";
-import * as Y from "yjs";
-import { v } from "convex/values";
-
-//#region src/component/public.ts
-const DEFAULT_SIZE_THRESHOLD = 5e6;
-const DEFAULT_PEER_TIMEOUT = 300 * 1e3;
-async function getNextSeq(ctx, collection) {
-	return ((await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", collection)).order("desc").first())?.seq ?? 0) + 1;
-}
-const insertDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const updateDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const deleteDocument = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		crdtBytes: v.bytes()
-	},
-	returns: v.object({
-		success: v.boolean(),
-		seq: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const seq = await getNextSeq(ctx, args.collection);
-		await ctx.db.insert("documents", {
-			collection: args.collection,
-			documentId: args.documentId,
-			crdtBytes: args.crdtBytes,
-			seq
-		});
-		return {
-			success: true,
-			seq
-		};
-	}
-});
-const mark = mutation({
-	args: {
-		collection: v.string(),
-		peerId: v.string(),
-		syncedSeq: v.number()
-	},
-	returns: v.null(),
-	handler: async (ctx, args) => {
-		const existing = await ctx.db.query("peers").withIndex("by_collection_peer", (q) => q.eq("collection", args.collection).eq("peerId", args.peerId)).first();
-		if (existing) await ctx.db.patch(existing._id, {
-			lastSyncedSeq: Math.max(existing.lastSyncedSeq, args.syncedSeq),
-			lastSeenAt: Date.now()
-		});
-		else await ctx.db.insert("peers", {
-			collection: args.collection,
-			peerId: args.peerId,
-			lastSyncedSeq: args.syncedSeq,
-			lastSeenAt: Date.now()
-		});
-		return null;
-	}
-});
-const compact = mutation({
-	args: {
-		collection: v.string(),
-		documentId: v.string(),
-		snapshotBytes: v.bytes(),
-		stateVector: v.bytes(),
-		peerTimeout: v.optional(v.number())
-	},
-	returns: v.object({
-		success: v.boolean(),
-		removed: v.number(),
-		retained: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["compaction"]);
-		const now = Date.now();
-		const peerCutoff = now - (args.peerTimeout ?? DEFAULT_PEER_TIMEOUT);
-		const deltas = await ctx.db.query("documents").withIndex("by_collection_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).collect();
-		const activePeers = await ctx.db.query("peers").withIndex("by_collection", (q) => q.eq("collection", args.collection)).filter((q) => q.gt(q.field("lastSeenAt"), peerCutoff)).collect();
-		const minSyncedSeq = activePeers.length > 0 ? Math.min(...activePeers.map((p) => p.lastSyncedSeq)) : Infinity;
-		const existingSnapshot = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection).eq("documentId", args.documentId)).first();
-		if (existingSnapshot) await ctx.db.delete(existingSnapshot._id);
-		const snapshotSeq = deltas.length > 0 ? Math.max(...deltas.map((d) => d.seq)) : 0;
-		await ctx.db.insert("snapshots", {
-			collection: args.collection,
-			documentId: args.documentId,
-			snapshotBytes: args.snapshotBytes,
-			stateVector: args.stateVector,
-			snapshotSeq,
-			createdAt: now
-		});
-		let removed = 0;
-		for (const delta of deltas) if (delta.seq < minSyncedSeq) {
-			await ctx.db.delete(delta._id);
-			removed++;
-		}
-		logger.info("Compaction completed", {
-			collection: args.collection,
-			documentId: args.documentId,
-			removed,
-			retained: deltas.length - removed,
-			activePeers: activePeers.length,
-			minSyncedSeq
-		});
-		return {
-			success: true,
-			removed,
-			retained: deltas.length - removed
-		};
-	}
-});
-const stream = query({
-	args: {
-		collection: v.string(),
-		cursor: v.number(),
-		limit: v.optional(v.number()),
-		sizeThreshold: v.optional(v.number())
-	},
-	returns: v.object({
-		changes: v.array(v.object({
-			documentId: v.string(),
-			crdtBytes: v.bytes(),
-			seq: v.number(),
-			operationType: v.string()
-		})),
-		cursor: v.number(),
-		hasMore: v.boolean(),
-		compact: v.optional(v.string())
-	}),
-	handler: async (ctx, args) => {
-		const limit = args.limit ?? 100;
-		const sizeThreshold = args.sizeThreshold ?? DEFAULT_SIZE_THRESHOLD;
-		const documents = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection).gt("seq", args.cursor)).order("asc").take(limit);
-		if (documents.length > 0) {
-			const changes = documents.map((doc) => ({
-				documentId: doc.documentId,
-				crdtBytes: doc.crdtBytes,
-				seq: doc.seq,
-				operationType: OperationType.Delta
-			}));
-			const newCursor = documents[documents.length - 1]?.seq ?? args.cursor;
-			let compactHint;
-			const allDocs = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-			const sizeByDocument = /* @__PURE__ */ new Map();
-			for (const doc of allDocs) {
-				const current = sizeByDocument.get(doc.documentId) ?? 0;
-				sizeByDocument.set(doc.documentId, current + doc.crdtBytes.byteLength);
-			}
-			for (const [docId, size] of sizeByDocument) if (size > sizeThreshold) {
-				compactHint = docId;
-				break;
-			}
-			return {
-				changes,
-				cursor: newCursor,
-				hasMore: documents.length === limit,
-				compact: compactHint
-			};
-		}
-		const oldestDelta = await ctx.db.query("documents").withIndex("by_seq", (q) => q.eq("collection", args.collection)).order("asc").first();
-		if (oldestDelta && args.cursor < oldestDelta.seq) {
-			const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-			if (snapshots.length === 0) throw new Error(`Disparity detected but no snapshots available for collection: ${args.collection}. Client cursor: ${args.cursor}, Oldest delta seq: ${oldestDelta.seq}`);
-			return {
-				changes: snapshots.map((snapshot) => ({
-					documentId: snapshot.documentId,
-					crdtBytes: snapshot.snapshotBytes,
-					seq: snapshot.snapshotSeq,
-					operationType: OperationType.Snapshot
-				})),
-				cursor: Math.max(...snapshots.map((s) => s.snapshotSeq)),
-				hasMore: false,
-				compact: void 0
-			};
-		}
-		return {
-			changes: [],
-			cursor: args.cursor,
-			hasMore: false,
-			compact: void 0
-		};
-	}
-});
-const getInitialState = query({
-	args: { collection: v.string() },
-	returns: v.union(v.object({
-		crdtBytes: v.bytes(),
-		cursor: v.number()
-	}), v.null()),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["ssr"]);
-		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-		if (snapshots.length === 0 && deltas.length === 0) {
-			logger.info("No initial state available - collection is empty", { collection: args.collection });
-			return null;
-		}
-		const updates = [];
-		let latestSeq = 0;
-		for (const snapshot of snapshots) {
-			updates.push(new Uint8Array(snapshot.snapshotBytes));
-			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
-		}
-		const sorted = deltas.sort((a, b) => a.seq - b.seq);
-		for (const delta of sorted) {
-			updates.push(new Uint8Array(delta.crdtBytes));
-			latestSeq = Math.max(latestSeq, delta.seq);
-		}
-		logger.info("Reconstructing initial state", {
-			collection: args.collection,
-			snapshotCount: snapshots.length,
-			deltaCount: deltas.length
-		});
-		const merged = Y.mergeUpdatesV2(updates);
-		logger.info("Initial state reconstructed", {
-			collection: args.collection,
-			originalSize: updates.reduce((sum, u) => sum + u.byteLength, 0),
-			mergedSize: merged.byteLength
-		});
-		return {
-			crdtBytes: merged.buffer,
-			cursor: latestSeq
-		};
-	}
-});
-const recovery = query({
-	args: {
-		collection: v.string(),
-		clientStateVector: v.bytes()
-	},
-	returns: v.object({
-		diff: v.optional(v.bytes()),
-		serverStateVector: v.bytes(),
-		cursor: v.number()
-	}),
-	handler: async (ctx, args) => {
-		const logger = getLogger(["recovery"]);
-		const snapshots = await ctx.db.query("snapshots").withIndex("by_document", (q) => q.eq("collection", args.collection)).collect();
-		const deltas = await ctx.db.query("documents").withIndex("by_collection", (q) => q.eq("collection", args.collection)).collect();
-		if (snapshots.length === 0 && deltas.length === 0) {
-			const emptyDoc = new Y.Doc();
-			const emptyVector = Y.encodeStateVector(emptyDoc);
-			emptyDoc.destroy();
-			return {
-				serverStateVector: emptyVector.buffer,
-				cursor: 0
-			};
-		}
-		const updates = [];
-		let latestSeq = 0;
-		for (const snapshot of snapshots) {
-			updates.push(new Uint8Array(snapshot.snapshotBytes));
-			latestSeq = Math.max(latestSeq, snapshot.snapshotSeq);
-		}
-		for (const delta of deltas) {
-			updates.push(new Uint8Array(delta.crdtBytes));
-			latestSeq = Math.max(latestSeq, delta.seq);
-		}
-		const mergedState = Y.mergeUpdatesV2(updates);
-		const clientVector = new Uint8Array(args.clientStateVector);
-		const diff = Y.diffUpdateV2(mergedState, clientVector);
-		const serverVector = Y.encodeStateVectorFromUpdateV2(mergedState);
-		logger.info("Recovery sync computed", {
-			collection: args.collection,
-			snapshotCount: snapshots.length,
-			deltaCount: deltas.length,
-			diffSize: diff.byteLength,
-			hasDiff: diff.byteLength > 0
-		});
-		return {
-			diff: diff.byteLength > 0 ? diff.buffer : void 0,
-			serverStateVector: serverVector.buffer,
-			cursor: latestSeq
-		};
-	}
-});
-
-//#endregion
-export { OperationType, compact, deleteDocument, getInitialState, insertDocument, mark, recovery, stream, updateDocument };

package/src/client/prose-schema.ts

@@ -1,55 +0,0 @@
-import { z } from "zod";
-import type { ProseValue } from "$/shared/types";
-
-const PROSE_MARKER = Symbol.for("replicate:prose");
-
-function createProseSchema(): z.ZodType<ProseValue> {
-  const schema = z.custom<ProseValue>(
-    (val) => {
-      if (val == null) return true;
-      if (typeof val !== "object") return false;
-      return (val as { type?: string }).type === "doc";
-    },
-    { message: "Expected prose document with type \"doc\"" },
-  );
-
-  Object.defineProperty(schema, PROSE_MARKER, { value: true, writable: false });
-
-  return schema;
-}
-
-function emptyProse(): ProseValue {
-  return { type: "doc", content: [] } as unknown as ProseValue;
-}
-
-export function prose(): z.ZodType<ProseValue> {
-  return createProseSchema();
-}
-
-prose.empty = emptyProse;
-
-export function isProseSchema(schema: unknown): boolean {
-  return (
-    schema != null
-    && typeof schema === "object"
-    && PROSE_MARKER in schema
-    && (schema as Record<symbol, unknown>)[PROSE_MARKER] === true
-  );
-}
-
-export function extractProseFields(schema: z.ZodObject<z.ZodRawShape>): string[] {
-  const fields: string[] = [];
-
-  for (const [key, fieldSchema] of Object.entries(schema.shape)) {
-    let unwrapped = fieldSchema;
-    while (unwrapped instanceof z.ZodOptional || unwrapped instanceof z.ZodNullable) {
-      unwrapped = unwrapped.unwrap();
-    }
-
-    if (isProseSchema(unwrapped)) {
-      fields.push(key);
-    }
-  }
-
-  return fields;
-}

package/src/client/services/cursor.ts

@@ -1,109 +0,0 @@
-import { Effect, Context, Layer } from "effect";
-import { IDBError, IDBWriteError } from "$/client/errors";
-import type { KeyValueStore } from "$/client/persistence/types";
-
-export type Cursor = number;
-
-export class CursorService extends Context.Tag("CursorService")<
-  CursorService,
-  {
-    readonly loadCursor: (collection: string) => Effect.Effect<Cursor, IDBError>;
-    readonly saveCursor: (collection: string, cursor: Cursor) => Effect.Effect<void, IDBWriteError>;
-    readonly clearCursor: (collection: string) => Effect.Effect<void, IDBError>;
-    readonly loadPeerId: (collection: string) => Effect.Effect<string, IDBError | IDBWriteError>;
-  }
->() {}
-
-function generatePeerId(): string {
-  return crypto.randomUUID();
-}
-
-export function createCursorLayer(kv: KeyValueStore) {
-  return Layer.succeed(
-    CursorService,
-    CursorService.of({
-      loadCursor: collection =>
-        Effect.gen(function* (_) {
-          const key = `cursor:${collection}`;
-          const stored = yield* _(
-            Effect.tryPromise({
-              try: () => kv.get<Cursor>(key),
-              catch: cause => new IDBError({ operation: "get", key, cause }),
-            }),
-          );
-
-          if (stored !== undefined) {
-            yield* _(
-              Effect.logDebug("Loaded cursor from storage", {
-                collection,
-                cursor: stored,
-              }),
-            );
-            return stored;
-          }
-
-          yield* _(
-            Effect.logDebug("No stored cursor, using default", {
-              collection,
-            }),
-          );
-          return 0;
-        }),
-
-      saveCursor: (collection, cursor) =>
-        Effect.gen(function* (_) {
-          const key = `cursor:${collection}`;
-          yield* _(
-            Effect.tryPromise({
-              try: () => kv.set(key, cursor),
-              catch: cause => new IDBWriteError({ key, value: cursor, cause }),
-            }),
-          );
-          yield* _(
-            Effect.logDebug("Cursor saved", {
-              collection,
-              cursor,
-            }),
-          );
-        }),
-
-      clearCursor: collection =>
-        Effect.gen(function* (_) {
-          const key = `cursor:${collection}`;
-          yield* _(
-            Effect.tryPromise({
-              try: () => kv.del(key),
-              catch: cause => new IDBError({ operation: "delete", key, cause }),
-            }),
-          );
-          yield* _(Effect.logDebug("Cursor cleared", { collection }));
-        }),
-
-      loadPeerId: collection =>
-        Effect.gen(function* (_) {
-          const key = `peerId:${collection}`;
-          const stored = yield* _(
-            Effect.tryPromise({
-              try: () => kv.get<string>(key),
-              catch: cause => new IDBError({ operation: "get", key, cause }),
-            }),
-          );
-
-          if (stored) {
-            yield* _(Effect.logDebug("Loaded peerId from storage", { collection, peerId: stored }));
-            return stored;
-          }
-
-          const newPeerId = generatePeerId();
-          yield* _(
-            Effect.tryPromise({
-              try: () => kv.set(key, newPeerId),
-              catch: cause => new IDBWriteError({ key, value: newPeerId, cause }),
-            }),
-          );
-          yield* _(Effect.logDebug("Generated new peerId", { collection, peerId: newPeerId }));
-          return newPeerId;
-        }),
-    }),
-  );
-}