@trestleinc/replicate 1.1.0 → 1.1.2-preview.0

package/src/client/prose-schema.ts

@@ -0,0 +1,55 @@
+import { z } from "zod";
+import type { ProseValue } from "$/shared/types";
+
+const PROSE_MARKER = Symbol.for("replicate:prose");
+
+function createProseSchema(): z.ZodType<ProseValue> {
+  const schema = z.custom<ProseValue>(
+    (val) => {
+      if (val == null) return true;
+      if (typeof val !== "object") return false;
+      return (val as { type?: string }).type === "doc";
+    },
+    { message: "Expected prose document with type \"doc\"" },
+  );
+
+  Object.defineProperty(schema, PROSE_MARKER, { value: true, writable: false });
+
+  return schema;
+}
+
+function emptyProse(): ProseValue {
+  return { type: "doc", content: [] } as unknown as ProseValue;
+}
+
+export function prose(): z.ZodType<ProseValue> {
+  return createProseSchema();
+}
+
+prose.empty = emptyProse;
+
+export function isProseSchema(schema: unknown): boolean {
+  return (
+    schema != null
+    && typeof schema === "object"
+    && PROSE_MARKER in schema
+    && (schema as Record<symbol, unknown>)[PROSE_MARKER] === true
+  );
+}
+
+export function extractProseFields(schema: z.ZodObject<z.ZodRawShape>): string[] {
+  const fields: string[] = [];
+
+  for (const [key, fieldSchema] of Object.entries(schema.shape)) {
+    let unwrapped = fieldSchema;
+    while (unwrapped instanceof z.ZodOptional || unwrapped instanceof z.ZodNullable) {
+      unwrapped = unwrapped.unwrap();
+    }
+
+    if (isProseSchema(unwrapped)) {
+      fields.push(key);
+    }
+  }
+
+  return fields;
+}

package/src/client/prose.ts

@@ -5,15 +5,15 @@
  * Uses document-level tracking to prevent race conditions.
  */
 
-import * as Y from 'yjs';
-import type { Collection } from '@tanstack/db';
-import { getLogger } from '$/client/logger.js';
-import { serializeYMapValue } from '$/client/merge.js';
+import * as Y from "yjs";
+import type { Collection } from "@tanstack/db";
+import { getLogger } from "$/client/logger";
+import { serializeYMapValue } from "$/client/merge";
 
 /** Server origin - changes from server should not trigger local sync */
-const SERVER_ORIGIN = 'server';
+const SERVER_ORIGIN = "server";
 
-const logger = getLogger(['replicate', 'prose']);
+const logger = getLogger(["replicate", "prose"]);
 
 // Default debounce time for prose sync
 const DEFAULT_DEBOUNCE_MS = 1000;
@@ -62,12 +62,13 @@ export function isApplyingFromServer(collection: string, documentId: string): bo
 export function setApplyingFromServer(
   collection: string,
   documentId: string,
-  value: boolean
+  value: boolean,
 ): void {
   const key = `${collection}:${documentId}`;
   if (value) {
     applyingFromServer.set(key, true);
-  } else {
+  }
+  else {
     applyingFromServer.delete(key);
   }
 }
@@ -89,8 +90,9 @@ function setPendingInternal(key: string, value: boolean): void {
       for (const cb of listeners) {
         try {
           cb(value);
-        } catch (err) {
-          logger.error('Pending listener error', { key, error: String(err) });
+        }
+        catch (err) {
+          logger.error("Pending listener error", { key, error: String(err) });
         }
       }
     }
@@ -110,7 +112,7 @@ export function isPending(collection: string, documentId: string): boolean {
 export function subscribePending(
   collection: string,
   documentId: string,
-  callback: (pending: boolean) => void
+  callback: (pending: boolean) => void,
 ): () => void {
   const key = `${collection}:${documentId}`;
 
@@ -145,7 +147,7 @@ export function cancelPending(collection: string, documentId: string): void {
     clearTimeout(timer);
     debounceTimers.delete(key);
     setPendingInternal(key, false);
-    logger.debug('Cancelled pending sync due to remote update', { collection, documentId });
+    logger.debug("Cancelled pending sync due to remote update", { collection, documentId });
   }
 }
 
@@ -162,7 +164,7 @@ export function cancelAllPending(collection: string): void {
       setPendingInternal(key, false);
     }
   }
-  logger.debug('Cancelled all pending syncs', { collection });
+  logger.debug("Cancelled all pending syncs", { collection });
 }
 
 // ============================================================================
@@ -201,7 +203,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
   // Skip if already observing this document
   const existingCleanup = fragmentObservers.get(key);
   if (existingCleanup) {
-    logger.debug('Fragment already being observed', { collection, documentId, field });
+    logger.debug("Fragment already being observed", { collection, documentId, field });
     return existingCleanup;
   }
 
@@ -224,7 +226,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
 
       const itemYMap = ymap.get(documentId) as Y.Map<unknown> | undefined;
       if (!itemYMap) {
-        logger.error('Document not found', { collection, documentId });
+        logger.error("Document not found", { collection, documentId });
         setPendingInternal(key, false);
         return;
       }
@@ -237,7 +239,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
           : Y.encodeStateAsUpdateV2(ydoc);
 
         if (delta.length <= 2) {
-          logger.debug('No changes to sync', { collection, documentId });
+          logger.debug("No changes to sync", { collection, documentId });
           setPendingInternal(key, false);
           return;
         }
@@ -245,7 +247,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
         const crdtBytes = delta.buffer as ArrayBuffer;
         const currentVector = Y.encodeStateVector(ydoc);
 
-        logger.debug('Syncing prose delta', {
+        logger.debug("Syncing prose delta", {
           collection,
           documentId,
           deltaSize: delta.byteLength,
@@ -259,7 +261,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
           { metadata: { contentSync: { crdtBytes, materializedDoc } } },
           (draft: any) => {
             draft.updatedAt = Date.now();
-          }
+          },
         );
         await result.isPersisted.promise;
 
@@ -267,9 +269,10 @@ export function observeFragment(config: ProseObserverConfig): () => void {
         lastSyncedVectors.set(key, currentVector);
         failedSyncQueue.delete(key);
         setPendingInternal(key, false);
-        logger.debug('Prose sync completed', { collection, documentId });
-      } catch (err) {
-        logger.error('Prose sync failed, queued for retry', {
+        logger.debug("Prose sync completed", { collection, documentId });
+      }
+      catch (err) {
+        logger.error("Prose sync failed, queued for retry", {
           collection,
           documentId,
           error: String(err),
@@ -284,7 +287,7 @@ export function observeFragment(config: ProseObserverConfig): () => void {
     // Also retry any failed syncs for this document
     if (failedSyncQueue.has(key)) {
       failedSyncQueue.delete(key);
-      logger.debug('Retrying failed sync', { collection, documentId });
+      logger.debug("Retrying failed sync", { collection, documentId });
     }
   };
 
@@ -296,11 +299,11 @@ export function observeFragment(config: ProseObserverConfig): () => void {
     cancelPending(collection, documentId);
     fragmentObservers.delete(key);
     lastSyncedVectors.delete(key);
-    logger.debug('Fragment observer cleaned up', { collection, documentId, field });
+    logger.debug("Fragment observer cleaned up", { collection, documentId, field });
   };
 
   fragmentObservers.set(key, cleanup);
-  logger.debug('Fragment observer registered', { collection, documentId, field });
+  logger.debug("Fragment observer registered", { collection, documentId, field });
 
   return cleanup;
 }
@@ -365,5 +368,5 @@ export function cleanup(collection: string): void {
     }
   }
 
-  logger.debug('Prose cleanup complete', { collection });
+  logger.debug("Prose cleanup complete", { collection });
 }

package/src/client/replicate.ts

@@ -7,7 +7,7 @@
 
 export interface ReplicateParams {
   readonly begin: () => void;
-  readonly write: (message: { type: 'insert' | 'update' | 'delete'; value: unknown }) => void;
+  readonly write: (message: { type: "insert" | "update" | "delete"; value: unknown }) => void;
   readonly commit: () => void;
   readonly truncate: () => void;
 }
@@ -41,7 +41,7 @@ export function createReplicateOps<T>(params: ReplicateParams): BoundReplicateOp
     insert(items: T[]): void {
       params.begin();
       for (const item of items) {
-        params.write({ type: 'insert', value: item });
+        params.write({ type: "insert", value: item });
       }
       params.commit();
     },
@@ -49,7 +49,7 @@ export function createReplicateOps<T>(params: ReplicateParams): BoundReplicateOp
     delete(items: T[]): void {
       params.begin();
       for (const item of items) {
-        params.write({ type: 'delete', value: item });
+        params.write({ type: "delete", value: item });
       }
       params.commit();
     },
@@ -58,7 +58,7 @@ export function createReplicateOps<T>(params: ReplicateParams): BoundReplicateOp
     upsert(items: T[]): void {
       params.begin();
       for (const item of items) {
-        params.write({ type: 'update', value: item });
+        params.write({ type: "update", value: item });
       }
       params.commit();
     },
@@ -67,7 +67,7 @@ export function createReplicateOps<T>(params: ReplicateParams): BoundReplicateOp
       params.begin();
       params.truncate();
       for (const item of items) {
-        params.write({ type: 'insert', value: item });
+        params.write({ type: "insert", value: item });
       }
       params.commit();
     },

package/src/client/services/cursor.ts

@@ -0,0 +1,109 @@
+import { Effect, Context, Layer } from "effect";
+import { IDBError, IDBWriteError } from "$/client/errors";
+import type { KeyValueStore } from "$/client/persistence/types";
+
+export type Cursor = number;
+
+export class CursorService extends Context.Tag("CursorService")<
+  CursorService,
+  {
+    readonly loadCursor: (collection: string) => Effect.Effect<Cursor, IDBError>;
+    readonly saveCursor: (collection: string, cursor: Cursor) => Effect.Effect<void, IDBWriteError>;
+    readonly clearCursor: (collection: string) => Effect.Effect<void, IDBError>;
+    readonly loadPeerId: (collection: string) => Effect.Effect<string, IDBError | IDBWriteError>;
+  }
+>() {}
+
+function generatePeerId(): string {
+  return crypto.randomUUID();
+}
+
+export function createCursorLayer(kv: KeyValueStore) {
+  return Layer.succeed(
+    CursorService,
+    CursorService.of({
+      loadCursor: collection =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          const stored = yield* _(
+            Effect.tryPromise({
+              try: () => kv.get<Cursor>(key),
+              catch: cause => new IDBError({ operation: "get", key, cause }),
+            }),
+          );
+
+          if (stored !== undefined) {
+            yield* _(
+              Effect.logDebug("Loaded cursor from storage", {
+                collection,
+                cursor: stored,
+              }),
+            );
+            return stored;
+          }
+
+          yield* _(
+            Effect.logDebug("No stored cursor, using default", {
+              collection,
+            }),
+          );
+          return 0;
+        }),
+
+      saveCursor: (collection, cursor) =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          yield* _(
+            Effect.tryPromise({
+              try: () => kv.set(key, cursor),
+              catch: cause => new IDBWriteError({ key, value: cursor, cause }),
+            }),
+          );
+          yield* _(
+            Effect.logDebug("Cursor saved", {
+              collection,
+              cursor,
+            }),
+          );
+        }),
+
+      clearCursor: collection =>
+        Effect.gen(function* (_) {
+          const key = `cursor:${collection}`;
+          yield* _(
+            Effect.tryPromise({
+              try: () => kv.del(key),
+              catch: cause => new IDBError({ operation: "delete", key, cause }),
+            }),
+          );
+          yield* _(Effect.logDebug("Cursor cleared", { collection }));
+        }),
+
+      loadPeerId: collection =>
+        Effect.gen(function* (_) {
+          const key = `peerId:${collection}`;
+          const stored = yield* _(
+            Effect.tryPromise({
+              try: () => kv.get<string>(key),
+              catch: cause => new IDBError({ operation: "get", key, cause }),
+            }),
+          );
+
+          if (stored) {
+            yield* _(Effect.logDebug("Loaded peerId from storage", { collection, peerId: stored }));
+            return stored;
+          }
+
+          const newPeerId = generatePeerId();
+          yield* _(
+            Effect.tryPromise({
+              try: () => kv.set(key, newPeerId),
+              catch: cause => new IDBWriteError({ key, value: newPeerId, cause }),
+            }),
+          );
+          yield* _(Effect.logDebug("Generated new peerId", { collection, peerId: newPeerId }));
+          return newPeerId;
+        }),
+    }),
+  );
+}

package/src/component/_generated/component.ts

@@ -24,64 +24,72 @@ import type { FunctionReference } from "convex/server";
 export type ComponentApi<Name extends string | undefined = string | undefined> =
   {
     public: {
-      deleteDocument: FunctionReference<
+      compact: FunctionReference<
         "mutation",
         "internal",
         {
           collection: string;
-          crdtBytes: ArrayBuffer;
           documentId: string;
-          threshold?: number;
-          version: number;
+          peerTimeout?: number;
+          snapshotBytes: ArrayBuffer;
+          stateVector: ArrayBuffer;
         },
-        { compacted?: boolean; success: boolean },
+        { removed: number; retained: number; success: boolean },
+        Name
+      >;
+      deleteDocument: FunctionReference<
+        "mutation",
+        "internal",
+        { collection: string; crdtBytes: ArrayBuffer; documentId: string },
+        { seq: number; success: boolean },
         Name
       >;
       getInitialState: FunctionReference<
         "query",
         "internal",
         { collection: string },
-        { checkpoint: { lastModified: number }; crdtBytes: ArrayBuffer } | null,
+        { crdtBytes: ArrayBuffer; cursor: number } | null,
         Name
       >;
       insertDocument: FunctionReference<
         "mutation",
         "internal",
-        {
-          collection: string;
-          crdtBytes: ArrayBuffer;
-          documentId: string;
-          threshold?: number;
-          version: number;
-        },
-        { compacted?: boolean; success: boolean },
+        { collection: string; crdtBytes: ArrayBuffer; documentId: string },
+        { seq: number; success: boolean },
+        Name
+      >;
+      mark: FunctionReference<
+        "mutation",
+        "internal",
+        { collection: string; peerId: string; syncedSeq: number },
+        null,
         Name
       >;
       recovery: FunctionReference<
         "query",
         "internal",
         { clientStateVector: ArrayBuffer; collection: string },
-        { diff?: ArrayBuffer; serverStateVector: ArrayBuffer },
+        { cursor: number; diff?: ArrayBuffer; serverStateVector: ArrayBuffer },
         Name
       >;
       stream: FunctionReference<
         "query",
         "internal",
         {
-          checkpoint: { lastModified: number };
           collection: string;
+          cursor: number;
           limit?: number;
-          vector?: ArrayBuffer;
+          sizeThreshold?: number;
         },
         {
           changes: Array<{
             crdtBytes: ArrayBuffer;
-            documentId?: string;
+            documentId: string;
             operationType: string;
-            timestamp: number;
-            version: number;
+            seq: number;
           }>;
-          checkpoint: { lastModified: number };
+          compact?: string;
+          cursor: number;
           hasMore: boolean;
         },
         Name
@@ -89,14 +97,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
       updateDocument: FunctionReference<
         "mutation",
         "internal",
-        {
-          collection: string;
-          crdtBytes: ArrayBuffer;
-          documentId: string;
-          threshold?: number;
-          version: number;
-        },
-        { compacted?: boolean; success: boolean },
+        { collection: string; crdtBytes: ArrayBuffer; documentId: string },
+        { seq: number; success: boolean },
         Name
       >;
     };

package/src/component/convex.config.ts

@@ -1,5 +1,5 @@
-import { defineComponent } from 'convex/server';
+import { defineComponent } from "convex/server";
 
-const component = defineComponent('replicate');
+const component = defineComponent("replicate");
 
 export default component;

package/src/component/logger.ts

@@ -9,28 +9,28 @@ class ComponentLogger implements Logger {
   constructor(private category: string[]) {}
 
   private format(level: string, message: string, context?: Record<string, any>): string {
-    const prefix = `[${this.category.join(':')}]`;
-    const contextStr = context ? ` ${JSON.stringify(context)}` : '';
+    const prefix = `[${this.category.join(":")}]`;
+    const contextStr = context ? ` ${JSON.stringify(context)}` : "";
     return `${prefix} ${level}: ${message}${contextStr}`;
   }
 
   debug(message: string, context?: Record<string, any>): void {
-    console.log(this.format('DEBUG', message, context));
+    console.log(this.format("DEBUG", message, context));
   }
 
   info(message: string, context?: Record<string, any>): void {
-    console.log(this.format('INFO', message, context));
+    console.log(this.format("INFO", message, context));
   }
 
   warn(message: string, context?: Record<string, any>): void {
-    console.warn(this.format('WARN', message, context));
+    console.warn(this.format("WARN", message, context));
   }
 
   error(message: string, context?: Record<string, any>): void {
-    console.error(this.format('ERROR', message, context));
+    console.error(this.format("ERROR", message, context));
   }
 }
 
 export function getLogger(category: string[]): Logger {
-  return new ComponentLogger(['component', ...category]);
+  return new ComponentLogger(["component", ...category]);
 }