@trestleinc/replicate 1.1.0 → 1.1.2-preview.0

package/src/client/collection.ts

@@ -1,18 +1,22 @@
-import * as Y from 'yjs';
-import { createMutex } from 'lib0/mutex';
-import type { Persistence, PersistenceProvider } from '$/client/persistence/types.js';
-import type { ConvexClient } from 'convex/browser';
-import type { FunctionReference } from 'convex/server';
-import type { CollectionConfig, Collection } from '@tanstack/db';
-import { Effect, Layer } from 'effect';
-import { getLogger } from '$/client/logger.js';
-import { ProseError, NonRetriableError } from '$/client/errors.js';
-import { Checkpoint, createCheckpointLayer } from '$/client/services/checkpoint.js';
-import { Reconciliation, ReconciliationLive } from '$/client/services/reconciliation.js';
-import { createReplicateOps, type BoundReplicateOps } from '$/client/replicate.js';
+import * as Y from "yjs";
+import { createMutex } from "lib0/mutex";
+import type { Persistence, PersistenceProvider } from "$/client/persistence/types";
+import type { ConvexClient } from "convex/browser";
+import { getFunctionName, type FunctionReference } from "convex/server";
+import {
+  createCollection,
+  type CollectionConfig,
+  type Collection,
+  type NonSingleResult,
+  type BaseCollectionConfig,
+} from "@tanstack/db";
+import type { StandardSchemaV1 } from "@standard-schema/spec";
+import { Effect } from "effect";
+import { getLogger } from "$/client/logger";
+import { ProseError, NonRetriableError } from "$/client/errors";
+import { CursorService, createCursorLayer, type Cursor } from "$/client/services/cursor";
+import { createReplicateOps, type BoundReplicateOps } from "$/client/replicate";
 import {
-  createYjsDocument,
-  getYMap,
   transactWithDelta,
   applyUpdate,
   extractItems,
@@ -21,18 +25,20 @@ import {
   fragmentFromJSON,
   serializeYMapValue,
   getFragmentFromYMap,
-} from '$/client/merge.js';
-import * as prose from '$/client/prose.js';
+} from "$/client/merge";
+import * as prose from "$/client/prose";
+import { extractProseFields } from "$/client/prose-schema";
+import { z } from "zod";
 
 /** Origin markers for Yjs transactions */
 enum YjsOrigin {
-  Local = 'local',
-  Fragment = 'fragment',
-  Server = 'server',
+  Local = "local",
+  Fragment = "fragment",
+  Server = "server",
 }
-import type { ProseFields, XmlFragmentJSON } from '$/shared/types.js';
+import type { ProseFields } from "$/shared/types";
 
-const logger = getLogger(['replicate', 'collection']);
+const logger = getLogger(["replicate", "collection"]);
 
 interface HttpError extends Error {
   status?: number;
@@ -62,8 +68,8 @@ interface CollectionTransaction<T> {
 
 function handleMutationError(
   error: unknown,
-  operation: 'Insert' | 'Update' | 'Delete',
-  collection: string
+  operation: "Insert" | "Update" | "Delete",
+  collection: string,
 ): never {
   const httpError = error as HttpError;
   logger.error(`${operation} failed`, {
@@ -73,10 +79,10 @@ function handleMutationError(
   });
 
   if (httpError?.status === 401 || httpError?.status === 403) {
-    throw new NonRetriableError('Authentication failed');
+    throw new NonRetriableError("Authentication failed");
   }
   if (httpError?.status === 422) {
-    throw new NonRetriableError('Validation error');
+    throw new NonRetriableError("Validation error");
   }
   throw error;
 }
@@ -84,34 +90,36 @@ function handleMutationError(
 const cleanupFunctions = new Map<string, () => void>();
 
 /** Server-rendered material data for SSR hydration */
-export type Materialized<T> = {
-  documents: ReadonlyArray<T>;
-  checkpoint?: { lastModified: number };
+export interface Materialized<T> {
+  documents: readonly T[];
+  cursor?: Cursor;
   count?: number;
   crdtBytes?: ArrayBuffer;
-};
+}
 
-/** Configuration for creating a Convex-backed collection */
-export interface ConvexCollectionOptionsConfig<T extends object> {
-  getKey: (item: T) => string | number;
-  material?: Materialized<T>;
+/** API object from replicate() */
+interface ConvexCollectionApi {
+  stream: FunctionReference<"query">;
+  insert: FunctionReference<"mutation">;
+  update: FunctionReference<"mutation">;
+  remove: FunctionReference<"mutation">;
+  recovery: FunctionReference<"query">;
+  mark: FunctionReference<"mutation">;
+  compact: FunctionReference<"mutation">;
+  material?: FunctionReference<"query">;
+}
+
+export interface ConvexCollectionConfig<
+  T extends object = object,
+  TSchema extends StandardSchemaV1 = never,
+  TKey extends string | number = string | number,
+> extends BaseCollectionConfig<T, TKey, TSchema> {
+  schema: TSchema;
   convexClient: ConvexClient;
-  api: {
-    stream: FunctionReference<'query'>;
-    insert: FunctionReference<'mutation'>;
-    update: FunctionReference<'mutation'>;
-    remove: FunctionReference<'mutation'>;
-    recovery: FunctionReference<'query'>;
-    material?: FunctionReference<'query'>;
-    [key: string]: any;
-  };
-  collection: string;
-  /** Fields that contain prose (rich text) content stored as Y.XmlFragment */
-  prose: Array<ProseFields<T>>;
-  /** Undo capture timeout in ms. Changes within this window merge into one undo. Default: 500 */
-  undoCaptureTimeout?: number;
-  /** Persistence provider for Y.Doc and key-value storage */
+  api: ConvexCollectionApi;
   persistence: Persistence;
+  material?: Materialized<T>;
+  undoCaptureTimeout?: number;
 }
 
 /** Editor binding for BlockNote/TipTap collaboration */
@@ -153,12 +161,6 @@ interface ConvexCollectionUtils<T extends object> {
   prose(documentId: string, field: ProseFields<T>): Promise<EditorBinding>;
 }
 
-/** Extended collection with prose field utilities */
-export interface ConvexCollection<T extends object> extends Collection<T> {
-  /** Utilities for prose field operations */
-  utils: ConvexCollectionUtils<T>;
-}
-
 // Module-level storage for Y.Doc and Y.Map instances
 const collectionDocs = new Map<string, { ydoc: Y.Doc; ymap: Y.Map<unknown> }>();
 
@@ -217,7 +219,7 @@ function getOrCreateFragmentUndoManager(
   collection: string,
   documentId: string,
   field: string,
-  fragment: Y.XmlFragment
+  fragment: Y.XmlFragment,
 ): Y.UndoManager {
   const key = `${collection}:${documentId}:${field}`;
 
@@ -236,43 +238,53 @@ function getOrCreateFragmentUndoManager(
   return um;
 }
 
-/**
- * Create TanStack DB collection options with Convex + Yjs replication.
- *
- * @example
- * ```typescript
- * const options = convexCollectionOptions<Task>({
- *   getKey: (t) => t.id,
- *   convexClient,
- *   api: { stream: api.tasks.stream, insert: api.tasks.insert, ... },
- *   collection: 'tasks',
- * });
- * const collection = createCollection(options);
- * ```
- */
-export function convexCollectionOptions<T extends object>({
-  getKey,
-  material,
-  convexClient,
-  api,
-  collection,
-  prose: proseFields,
-  undoCaptureTimeout = 500,
-  persistence,
-}: ConvexCollectionOptionsConfig<T>): CollectionConfig<T> & {
-  _convexClient: ConvexClient;
-  _collection: string;
-  _proseFields: Array<ProseFields<T>>;
-  _persistence: Persistence;
-  utils: ConvexCollectionUtils<T>;
+export function convexCollectionOptions<
+  TSchema extends z.ZodObject<z.ZodRawShape>,
+  TKey extends string | number = string | number,
+>(
+  config: ConvexCollectionConfig<z.infer<TSchema>, TSchema, TKey>,
+): CollectionConfig<z.infer<TSchema>, TKey, TSchema, ConvexCollectionUtils<z.infer<TSchema>>> & {
+  id: string;
+  utils: ConvexCollectionUtils<z.infer<TSchema>>;
+  schema: TSchema;
+};
+
+export function convexCollectionOptions(
+  config: ConvexCollectionConfig<any, any, any>,
+): CollectionConfig<any, any, any, ConvexCollectionUtils<any>> & {
+  id: string;
+  utils: ConvexCollectionUtils<any>;
+  schema: any;
 } {
+  const {
+    schema,
+    getKey,
+    material,
+    convexClient,
+    api,
+    undoCaptureTimeout = 500,
+    persistence,
+  } = config;
+
+  // Extract collection name from function reference path (e.g., "intervals:stream" -> "intervals")
+  const functionPath = getFunctionName(api.stream);
+  const collection = functionPath.split(":")[0];
+  if (!collection) {
+    throw new Error("Could not extract collection name from api.stream function reference");
+  }
+
+  const proseFields: string[]
+    = schema && schema instanceof z.ZodObject ? extractProseFields(schema) : [];
+
+  // DataType is 'any' in implementation - type safety comes from overload signatures
+  type DataType = any;
   // Create a Set for O(1) lookup of prose fields
-  const proseFieldSet = new Set<string>(proseFields as string[]);
+  const proseFieldSet = new Set<string>(proseFields);
 
   // Create utils object - prose() waits for Y.Doc to be ready via collectionDocs
-  const utils: ConvexCollectionUtils<T> = {
-    async prose(documentId: string, field: ProseFields<T>): Promise<EditorBinding> {
-      const fieldStr = field as string;
+  const utils: ConvexCollectionUtils<DataType> = {
+    async prose(documentId: string, field: ProseFields<DataType>): Promise<EditorBinding> {
+      const fieldStr = field;
 
       // Validate field is in prose config
       if (!proseFieldSet.has(fieldStr)) {
@@ -295,14 +307,15 @@ export function convexCollectionOptions<T extends object>({
             if (collectionDocs.has(collection)) {
               clearInterval(check);
               resolve();
-            } else if (Date.now() - startTime > maxWait) {
+            }
+            else if (Date.now() - startTime > maxWait) {
               clearInterval(check);
               reject(
                 new ProseError({
                   documentId,
                   field: fieldStr,
                   collection,
-                })
+                }),
               );
             }
           }, 10);
@@ -346,7 +359,7 @@ export function convexCollectionOptions<T extends object>({
         collection,
         documentId,
         fieldStr,
-        fragment
+        fragment,
       );
 
       // Return EditorBinding with reactive pending state from prose module
@@ -381,17 +394,21 @@ export function convexCollectionOptions<T extends object>({
     },
   };
 
-  let ydoc: Y.Doc = null as any;
-  let ymap: Y.Map<unknown> = null as any;
+  // Create ydoc/ymap synchronously for immediate local-first operations
+  // Persistence sync will load state into this doc later
+  const ydoc: Y.Doc = new Y.Doc({ guid: collection } as any);
+  const ymap: Y.Map<unknown> = ydoc.getMap(collection);
   let docPersistence: PersistenceProvider = null as any;
 
+  // Register ydoc immediately so utils.prose() can access it
+  collectionDocs.set(collection, { ydoc, ymap });
+
   // Bound replicate operations - set during sync initialization
   // Used by onDelete and other handlers that need to sync with TanStack DB
-  let ops: BoundReplicateOps<T> = null as any;
+  let ops: BoundReplicateOps<DataType> = null as any;
 
   // Create services layer with the persistence KV store
-  const checkpointLayer = createCheckpointLayer(persistence.kv);
-  const servicesLayer = Layer.mergeAll(checkpointLayer, ReconciliationLive);
+  const cursorLayer = createCursorLayer(persistence.kv);
 
   let resolvePersistenceReady: (() => void) | undefined;
   const persistenceReadyPromise = new Promise<void>((resolve) => {
@@ -403,94 +420,47 @@ export function convexCollectionOptions<T extends object>({
     resolveOptimisticReady = resolve;
   });
 
-  const reconcile = (ops: BoundReplicateOps<T>) =>
-    Effect.gen(function* () {
-      if (!api.material) return;
-
-      const materialApi = api.material;
-      const reconciliation = yield* Reconciliation;
-
-      const serverResponse = yield* Effect.tryPromise({
-        try: () => convexClient.query(materialApi, {}),
-        catch: (error) => new Error(`Reconciliation query failed: ${error}`),
-      });
-
-      const serverDocs = Array.isArray(serverResponse)
-        ? serverResponse
-        : ((serverResponse as any).documents as T[] | undefined) || [];
-
-      const removedItems = yield* reconciliation.reconcile(
-        ydoc,
-        ymap,
-        collection,
-        serverDocs,
-        (doc: T) => String(getKey(doc))
-      );
-
-      if (removedItems.length > 0) {
-        ops.delete(removedItems);
-      }
-    }).pipe(
-      Effect.catchAll((error) =>
-        Effect.gen(function* () {
-          yield* Effect.logError('Reconciliation failed', { collection, error });
-        })
-      )
-    );
-
-  /**
-   * Recovery sync using state vectors.
-   * Fetches missing data from server based on local state vector.
-   */
-  const recoverSync = async (): Promise<void> => {
+  const recover = async (): Promise<Cursor> => {
     if (!api.recovery) {
-      logger.debug('No recovery API configured, skipping recovery sync', { collection });
-      return;
+      logger.debug("No recovery API configured", { collection });
+      return 0;
     }
 
     try {
-      // Encode local state vector
       const localStateVector = Y.encodeStateVector(ydoc);
-
-      logger.debug('Starting recovery sync', {
+      logger.debug("Starting recovery", {
         collection,
         localVectorSize: localStateVector.byteLength,
       });
 
-      // Query server for diff
       const response = await convexClient.query(api.recovery, {
         clientStateVector: localStateVector.buffer as ArrayBuffer,
       });
 
-      // Apply diff if any
       if (response.diff) {
         const mux = getOrCreateMutex(collection);
         mux(() => {
           applyUpdate(ydoc, new Uint8Array(response.diff), YjsOrigin.Server);
         });
-
-        logger.info('Recovery sync applied diff', {
-          collection,
-          diffSize: response.diff.byteLength,
-        });
-      } else {
-        logger.debug('Recovery sync - no diff needed', { collection });
+        logger.info("Recovery applied diff", { collection, diffSize: response.diff.byteLength });
       }
 
-      // Store server state vector for future reference
       if (response.serverStateVector) {
         serverStateVectors.set(collection, new Uint8Array(response.serverStateVector));
       }
-    } catch (error) {
-      logger.error('Recovery sync failed', {
-        collection,
-        error: String(error),
-      });
-      // Don't throw - recovery is best-effort, subscription will catch up
+
+      const cursor = response.cursor ?? 0;
+      await persistence.kv.set(`cursor:${collection}`, cursor);
+      logger.info("Recovery complete", { collection, cursor });
+      return cursor;
+    }
+    catch (error) {
+      logger.error("Recovery failed", { collection, error: String(error) });
+      return 0;
     }
   };
 
-  const applyYjsInsert = (mutations: CollectionMutation<T>[]): Uint8Array => {
+  const applyYjsInsert = (mutations: CollectionMutation<DataType>[]): Uint8Array => {
     const { delta } = transactWithDelta(
       ydoc,
       () => {
@@ -505,19 +475,20 @@ export function convexCollectionOptions<T extends object>({
               // Add fragment to map FIRST (binds it to the Y.Doc)
               itemYMap.set(k, fragment);
               // THEN populate content (now it's part of the document)
-              fragmentFromJSON(fragment, v as XmlFragmentJSON);
-            } else {
+              fragmentFromJSON(fragment, v);
+            }
+            else {
               itemYMap.set(k, v);
             }
           });
         });
       },
-      YjsOrigin.Local
+      YjsOrigin.Local,
     );
     return delta;
   };
 
-  const applyYjsUpdate = (mutations: CollectionMutation<T>[]): Uint8Array => {
+  const applyYjsUpdate = (mutations: CollectionMutation<DataType>[]): Uint8Array => {
     const { delta } = transactWithDelta(
       ydoc,
       () => {
@@ -526,7 +497,7 @@ export function convexCollectionOptions<T extends object>({
           if (itemYMap) {
             const modifiedFields = mut.modified as Record<string, unknown>;
             if (!modifiedFields) {
-              logger.warn('mut.modified is null/undefined', { collection, key: String(mut.key) });
+              logger.warn("mut.modified is null/undefined", { collection, key: String(mut.key) });
               return;
             }
             Object.entries(modifiedFields).forEach(([k, v]) => {
@@ -537,33 +508,34 @@ export function convexCollectionOptions<T extends object>({
               // Server sync goes: subscription → applyUpdate(ydoc) → CRDT merge
               // Writing serialized JSON back would corrupt the CRDT state
               if (proseFieldSet.has(k)) {
-                logger.debug('Skipping prose field in applyYjsUpdate', { field: k });
+                logger.debug("Skipping prose field in applyYjsUpdate", { field: k });
                 return;
               }
 
               // Also skip if existing value is a Y.XmlFragment (defensive check)
               if (existingValue instanceof Y.XmlFragment) {
-                logger.debug('Preserving live fragment field', { field: k });
+                logger.debug("Preserving live fragment field", { field: k });
                 return;
               }
 
               // Regular field update
               itemYMap.set(k, v);
             });
-          } else {
-            logger.error('Update attempted on non-existent item', {
+          }
+          else {
+            logger.error("Update attempted on non-existent item", {
               collection,
               key: String(mut.key),
             });
           }
         });
       },
-      YjsOrigin.Local
+      YjsOrigin.Local,
     );
     return delta;
   };
 
-  const applyYjsDelete = (mutations: CollectionMutation<T>[]): Uint8Array => {
+  const applyYjsDelete = (mutations: CollectionMutation<DataType>[]): Uint8Array => {
     const { delta } = transactWithDelta(
       ydoc,
       () => {
@@ -571,7 +543,7 @@ export function convexCollectionOptions<T extends object>({
           ymap.delete(String(mut.key));
         });
       },
-      YjsOrigin.Local
+      YjsOrigin.Local,
     );
     return delta;
   };
@@ -579,20 +551,17 @@ export function convexCollectionOptions<T extends object>({
   return {
     id: collection,
     getKey,
-    _convexClient: convexClient,
-    _collection: collection,
-    _proseFields: proseFields,
-    _persistence: persistence,
+    schema: schema,
     utils,
 
-    onInsert: async ({ transaction }: CollectionTransaction<T>) => {
+    onInsert: async ({ transaction }: CollectionTransaction<DataType>) => {
+      const delta = applyYjsInsert(transaction.mutations);
+
       try {
         await Promise.all([persistenceReadyPromise, optimisticReadyPromise]);
-        const delta = applyYjsInsert(transaction.mutations);
         if (delta.length > 0) {
           const documentKey = String(transaction.mutations[0].key);
           const itemYMap = ymap.get(documentKey) as Y.Map<unknown>;
-          // Use serializeYMapValue to convert Y.XmlFragment → XmlFragmentJSON (same as onUpdate)
           const materializedDoc = itemYMap
             ? serializeYMapValue(itemYMap)
             : transaction.mutations[0].modified;
@@ -602,32 +571,33 @@ export function convexCollectionOptions<T extends object>({
             materializedDoc,
           });
         }
-      } catch (error) {
-        handleMutationError(error, 'Insert', collection);
+      }
+      catch (error) {
+        handleMutationError(error, "Insert", collection);
       }
     },
 
-    onUpdate: async ({ transaction }: CollectionTransaction<T>) => {
-      try {
-        const mutation = transaction.mutations[0];
-        const documentKey = String(mutation.key);
+    onUpdate: async ({ transaction }: CollectionTransaction<DataType>) => {
+      const mutation = transaction.mutations[0];
+      const documentKey = String(mutation.key);
 
-        // Skip if this update originated from server (prevents echo loops)
-        // Now checks DOCUMENT-level flag, not collection-level
-        if (prose.isApplyingFromServer(collection, documentKey)) {
-          logger.debug('Skipping onUpdate - data from server', { collection, documentKey });
-          return;
-        }
+      if (prose.isApplyingFromServer(collection, documentKey)) {
+        logger.debug("Skipping onUpdate - data from server", { collection, documentKey });
+        return;
+      }
 
-        await Promise.all([persistenceReadyPromise, optimisticReadyPromise]);
+      const metadata = mutation.metadata as { contentSync?: ContentSyncMetadata } | undefined;
+      const isContentSync = !!metadata?.contentSync;
 
-        // Metadata is on mutation, not transaction (TanStack DB API)
-        const metadata = mutation.metadata as { contentSync?: ContentSyncMetadata } | undefined;
+      // Apply regular updates to Yjs immediately (local-first)
+      // ContentSync updates already have Yjs changes applied via fragment observer
+      const delta = isContentSync ? null : applyYjsUpdate(transaction.mutations);
 
-        // Check if this is a content sync from utils.prose()
-        if (metadata?.contentSync) {
-          const { crdtBytes, materializedDoc } = metadata.contentSync;
+      try {
+        await Promise.all([persistenceReadyPromise, optimisticReadyPromise]);
 
+        if (isContentSync && metadata?.contentSync) {
+          const { crdtBytes, materializedDoc } = metadata.contentSync;
           await convexClient.mutation(api.update, {
             documentId: documentKey,
             crdtBytes,
@@ -636,11 +606,8 @@ export function convexCollectionOptions<T extends object>({
           return;
         }
 
-        // Regular update - apply to Y.Doc and generate delta
-        const delta = applyYjsUpdate(transaction.mutations);
-        if (delta.length > 0) {
+        if (delta && delta.length > 0) {
           const itemYMap = ymap.get(documentKey) as Y.Map<unknown>;
-          // Use serializeYMapValue to properly handle XmlFragment fields
           const fullDoc = itemYMap ? serializeYMapValue(itemYMap) : mutation.modified;
           await convexClient.mutation(api.update, {
             documentId: documentKey,
@@ -648,18 +615,20 @@ export function convexCollectionOptions<T extends object>({
             materializedDoc: fullDoc,
           });
         }
-      } catch (error) {
-        handleMutationError(error, 'Update', collection);
+      }
+      catch (error) {
+        handleMutationError(error, "Update", collection);
       }
     },
 
-    onDelete: async ({ transaction }: CollectionTransaction<T>) => {
+    onDelete: async ({ transaction }: CollectionTransaction<DataType>) => {
+      const delta = applyYjsDelete(transaction.mutations);
+
       try {
         await Promise.all([persistenceReadyPromise, optimisticReadyPromise]);
-        const delta = applyYjsDelete(transaction.mutations);
         const itemsToDelete = transaction.mutations
-          .map((mut) => mut.original)
-          .filter((item): item is T => item !== undefined && Object.keys(item).length > 0);
+          .map(mut => mut.original)
+          .filter((item): item is DataType => item !== undefined && Object.keys(item).length > 0);
         ops.delete(itemsToDelete);
         if (delta.length > 0) {
           const documentKey = String(transaction.mutations[0].key);
@@ -668,13 +637,14 @@ export function convexCollectionOptions<T extends object>({
             crdtBytes: delta.slice().buffer,
           });
         }
-      } catch (error) {
-        handleMutationError(error, 'Delete', collection);
+      }
+      catch (error) {
+        handleMutationError(error, "Delete", collection);
       }
     },
 
     sync: {
-      rowUpdateMode: 'partial',
+      rowUpdateMode: "partial",
       sync: (params: any) => {
         const { markReady, collection: collectionInstance } = params;
 
@@ -689,35 +659,31 @@ export function convexCollectionOptions<T extends object>({
 
         let subscription: (() => void) | null = null;
         const ssrDocuments = material?.documents;
-        const ssrCheckpoint = material?.checkpoint;
+        const ssrCursor = material?.cursor;
         const ssrCRDTBytes = material?.crdtBytes;
-        const docs: T[] = ssrDocuments ? [...ssrDocuments] : [];
+        const docs: DataType[] = ssrDocuments ? [...ssrDocuments] : [];
 
         (async () => {
           try {
-            ydoc = await createYjsDocument(collection, persistence.kv);
-            ymap = getYMap<unknown>(ydoc, collection);
-
-            collectionDocs.set(collection, { ydoc, ymap });
-
-            // Store undo config for per-document undo managers
+            // ydoc/ymap already created synchronously - just set up undo config
             const trackedOrigins = new Set([YjsOrigin.Local]);
             collectionUndoConfig.set(collection, {
               captureTimeout: undoCaptureTimeout,
               trackedOrigins,
             });
 
+            // Load persisted state into existing ydoc
             docPersistence = persistence.createDocPersistence(collection, ydoc);
             docPersistence.whenSynced.then(() => {
-              logger.debug('Persistence synced', { collection });
+              logger.debug("Persistence synced", { collection });
               resolvePersistenceReady?.();
             });
             await persistenceReadyPromise;
-            logger.info('Persistence ready', { collection, ymapSize: ymap.size });
+            logger.info("Persistence ready", { collection, ymapSize: ymap.size });
 
             // Create bound replicate operations for this collection
             // These are tied to this collection's TanStack DB params
-            ops = createReplicateOps<T>(params);
+            ops = createReplicateOps<DataType>(params);
             resolveOptimisticReady?.();
 
             // Note: Fragment sync is handled by utils.prose() debounce handler
@@ -727,55 +693,37 @@ export function convexCollectionOptions<T extends object>({
               applyUpdate(ydoc, new Uint8Array(ssrCRDTBytes), YjsOrigin.Server);
             }
 
-            // === LOCAL-FIRST FLOW WITH RECOVERY ===
-            // 1. Local data (IndexedDB/Yjs) is the source of truth
-            // 2. Recovery sync - get any missing data from server using state vectors
-            // 3. Push local+recovered data to TanStack DB with ops.replace
-            // 4. Reconcile phantom documents (hidden in loading state)
-            // 5. markReady() - UI renders DATA immediately
-            // 6. Subscription starts in background (replication)
-
-            // Step 1: Recovery sync - fetch missing server data
-            await recoverSync();
+            const recoveryCursor = await recover();
 
-            // Step 2: Push local+recovered data to TanStack DB
             if (ymap.size > 0) {
-              const items = extractItems<T>(ymap);
-              ops.replace(items); // Atomic replace, not accumulative insert
-              logger.info('Data loaded to TanStack DB', {
+              const items = extractItems<DataType>(ymap);
+              ops.replace(items);
+              logger.info("Data loaded to TanStack DB", {
                 collection,
                 itemCount: items.length,
               });
-            } else {
-              // No data - clear TanStack DB to avoid stale state
+            }
+            else {
               ops.replace([]);
-              logger.info('No data, cleared TanStack DB', { collection });
+              logger.info("No data, cleared TanStack DB", { collection });
             }
 
-            // Step 3: Reconcile phantom documents (still in loading state)
-            logger.debug('Running reconciliation', { collection, ymapSize: ymap.size });
-            await Effect.runPromise(reconcile(ops).pipe(Effect.provide(servicesLayer)));
-            logger.debug('Reconciliation complete', { collection });
-
-            // Step 4: Mark ready - UI shows data immediately
             markReady();
-            logger.info('Collection ready', { collection, ymapSize: ymap.size });
-
-            // Step 4: Load checkpoint for subscription (background replication)
-            const checkpoint =
-              ssrCheckpoint ||
-              (await Effect.runPromise(
-                Effect.gen(function* () {
-                  const checkpointSvc = yield* Checkpoint;
-                  return yield* checkpointSvc.loadCheckpoint(collection);
-                }).pipe(Effect.provide(checkpointLayer))
-              ));
-
-            logger.info('Checkpoint loaded', {
+            logger.info("Collection ready", { collection, ymapSize: ymap.size });
+
+            const peerId = await Effect.runPromise(
+              Effect.gen(function* () {
+                const cursorSvc = yield* CursorService;
+                return yield* cursorSvc.loadPeerId(collection);
+              }).pipe(Effect.provide(cursorLayer)),
+            );
+            const cursor = ssrCursor ?? recoveryCursor;
+
+            logger.info("Starting subscription", {
               collection,
-              checkpoint,
-              source: ssrCheckpoint ? 'SSR' : 'IndexedDB',
-              ymapSize: ymap.size,
+              cursor,
+              peerId,
+              source: ssrCursor !== undefined ? "SSR" : "recovery",
             });
 
             // Get mutex for thread-safe updates
@@ -787,16 +735,17 @@ export function convexCollectionOptions<T extends object>({
 
               mux(() => {
                 try {
-                  logger.debug('Applying snapshot', {
+                  logger.debug("Applying snapshot", {
                     collection,
                     bytesLength: crdtBytes.byteLength,
                   });
                   applyUpdate(ydoc, new Uint8Array(crdtBytes), YjsOrigin.Server);
-                  const items = extractItems<T>(ymap);
-                  logger.debug('Snapshot applied', { collection, itemCount: items.length });
+                  const items = extractItems<DataType>(ymap);
+                  logger.debug("Snapshot applied", { collection, itemCount: items.length });
                   ops.replace(items);
-                } catch (error) {
-                  logger.error('Error applying snapshot', { collection, error: String(error) });
+                }
+                catch (error) {
+                  logger.error("Error applying snapshot", { collection, error: String(error) });
                   throw new Error(`Snapshot application failed: ${error}`);
                 }
               });
@@ -812,38 +761,42 @@ export function convexCollectionOptions<T extends object>({
 
               mux(() => {
                 try {
-                  logger.debug('Applying delta', {
+                  logger.debug("Applying delta", {
                     collection,
                     documentId,
                     bytesLength: crdtBytes.byteLength,
                   });
 
-                  const itemBefore = documentId ? extractItem<T>(ymap, documentId) : null;
+                  const itemBefore = documentId ? extractItem<DataType>(ymap, documentId) : null;
                   applyUpdate(ydoc, new Uint8Array(crdtBytes), YjsOrigin.Server);
 
                   if (!documentId) {
-                    logger.debug('Delta applied (no documentId)', { collection });
+                    logger.debug("Delta applied (no documentId)", { collection });
                     return;
                   }
 
-                  const itemAfter = extractItem<T>(ymap, documentId);
+                  const itemAfter = extractItem<DataType>(ymap, documentId);
                   if (itemAfter) {
-                    logger.debug('Upserting item after delta', { collection, documentId });
+                    logger.debug("Upserting item after delta", { collection, documentId });
                     ops.upsert([itemAfter]);
-                  } else if (itemBefore) {
-                    logger.debug('Deleting item after delta', { collection, documentId });
+                  }
+                  else if (itemBefore) {
+                    logger.debug("Deleting item after delta", { collection, documentId });
                     ops.delete([itemBefore]);
-                  } else {
-                    logger.debug('No change detected after delta', { collection, documentId });
                   }
-                } catch (error) {
-                  logger.error('Error applying delta', {
+                  else {
+                    logger.debug("No change detected after delta", { collection, documentId });
+                  }
+                }
+                catch (error) {
+                  logger.error("Error applying delta", {
                     collection,
                     documentId,
                     error: String(error),
                   });
                   throw new Error(`Delta application failed for ${documentId}: ${error}`);
-                } finally {
+                }
+                finally {
                   // Clear document-level flag after delta processing
                   if (documentId) {
                     prose.setApplyingFromServer(collection, documentId, false);
@@ -852,86 +805,111 @@ export function convexCollectionOptions<T extends object>({
               });
             };
 
-            // Simple async subscription handler - bypasses Effect for reliability
             const handleSubscriptionUpdate = async (response: any) => {
               try {
-                // Validate response shape
                 if (!response || !Array.isArray(response.changes)) {
-                  logger.error('Invalid subscription response', { response });
+                  logger.error("Invalid subscription response", { response });
                   return;
                 }
 
-                const { changes, checkpoint: newCheckpoint } = response;
+                const { changes, cursor: newCursor, compact: compactHint } = response;
 
-                // Process each change
                 for (const change of changes) {
                   const { operationType, crdtBytes, documentId } = change;
                   if (!crdtBytes) {
-                    logger.warn('Skipping change with missing crdtBytes', { change });
+                    logger.warn("Skipping change with missing crdtBytes", { change });
                     continue;
                   }
 
                   try {
-                    if (operationType === 'snapshot') {
+                    if (operationType === "snapshot") {
                       handleSnapshotChange(crdtBytes);
-                    } else {
+                    }
+                    else {
                       handleDeltaChange(crdtBytes, documentId);
                     }
-                  } catch (changeError) {
-                    logger.error('Failed to apply change', {
+                  }
+                  catch (changeError) {
+                    logger.error("Failed to apply change", {
                       operationType,
                       documentId,
                       error: String(changeError),
                     });
-                    // Continue processing other changes
                   }
                 }
 
-                // Save checkpoint using persistence KV store
-                if (newCheckpoint) {
+                if (newCursor !== undefined) {
+                  try {
+                    const key = `cursor:${collection}`;
+                    await persistence.kv.set(key, newCursor);
+                    logger.debug("Cursor saved", { collection, cursor: newCursor });
+
+                    await convexClient.mutation(api.mark, {
+                      peerId,
+                      syncedSeq: newCursor,
+                    });
+                    logger.debug("Ack sent", { collection, peerId, syncedSeq: newCursor });
+                  }
+                  catch (ackError) {
+                    logger.error("Failed to save cursor or ack", {
+                      collection,
+                      error: String(ackError),
+                    });
+                  }
+                }
+
+                if (compactHint) {
                   try {
-                    const key = `checkpoint:${collection}`;
-                    await persistence.kv.set(key, newCheckpoint);
-                    logger.debug('Checkpoint saved', { collection, checkpoint: newCheckpoint });
-                  } catch (checkpointError) {
-                    logger.error('Failed to save checkpoint', {
+                    const snapshot = Y.encodeStateAsUpdate(ydoc);
+                    const stateVector = Y.encodeStateVector(ydoc);
+                    await convexClient.mutation(api.compact, {
+                      documentId: compactHint,
+                      snapshotBytes: snapshot.buffer,
+                      stateVector: stateVector.buffer,
+                    });
+                    logger.info("Compaction triggered", { collection, documentId: compactHint });
+                  }
+                  catch (compactError) {
+                    logger.error("Compaction failed", {
                       collection,
-                      error: String(checkpointError),
+                      documentId: compactHint,
+                      error: String(compactError),
                     });
                   }
                 }
-              } catch (error) {
-                logger.error('Subscription handler error', { collection, error: String(error) });
+              }
+              catch (error) {
+                logger.error("Subscription handler error", { collection, error: String(error) });
               }
             };
 
-            logger.info('Establishing subscription', {
+            logger.info("Establishing subscription", {
               collection,
-              checkpoint,
+              cursor,
               limit: 1000,
             });
 
             subscription = convexClient.onUpdate(
               api.stream,
-              { checkpoint, limit: 1000 },
+              { cursor, limit: 1000 },
               (response: any) => {
-                logger.debug('Subscription received update', {
+                logger.debug("Subscription received update", {
                   collection,
                   changesCount: response.changes?.length ?? 0,
-                  checkpoint: response.checkpoint,
+                  cursor: response.cursor,
                   hasMore: response.hasMore,
                 });
 
-                // Call async handler directly - no Effect wrapper
                 handleSubscriptionUpdate(response);
-              }
+              },
             );
 
             // Note: markReady() was already called above (local-first)
             // Subscription is background replication, not blocking
-            logger.info('Subscription established', { collection });
-          } catch (error) {
-            logger.error('Failed to set up collection', { error, collection });
+            logger.info("Subscription established", { collection });
+          }
+          catch (error) {
+            logger.error("Failed to set up collection", { error, collection });
             // Still mark ready on error so UI isn't stuck loading
             markReady();
           }
@@ -975,3 +953,58 @@ export function convexCollectionOptions<T extends object>({
     },
   };
 }
+
+type LazyCollectionConfig<TSchema extends z.ZodObject<z.ZodRawShape>> = Omit<
+  ConvexCollectionConfig<z.infer<TSchema>, TSchema, string>,
+  "persistence" | "material"
+>;
+
+interface LazyCollection<T extends object> {
+  init(material?: Materialized<T>): Promise<void>;
+  get(): Collection<T, string, ConvexCollectionUtils<T>, any, T> & NonSingleResult;
+}
+
+export type ConvexCollection<T extends object>
+  = Collection<T, any, ConvexCollectionUtils<T>, any, T> & NonSingleResult;
+
+interface CreateCollectionOptions<TSchema extends z.ZodObject<z.ZodRawShape>> {
+  persistence: () => Promise<Persistence>;
+  config: () => Omit<LazyCollectionConfig<TSchema>, "material">;
+}
+
+export const collection = {
+  create<TSchema extends z.ZodObject<z.ZodRawShape>>(
+    options: CreateCollectionOptions<TSchema>,
+  ): LazyCollection<z.infer<TSchema>> {
+    let persistence: Persistence | null = null;
+    let resolvedConfig: LazyCollectionConfig<TSchema> | null = null;
+    let material: Materialized<z.infer<TSchema>> | undefined;
+    type Instance = LazyCollection<z.infer<TSchema>>["get"] extends () => infer R ? R : never;
+    let instance: Instance | null = null;
+
+    return {
+      async init(mat?: Materialized<z.infer<TSchema>>) {
+        if (!persistence) {
+          persistence = await options.persistence();
+          resolvedConfig = options.config();
+          material = mat;
+        }
+      },
+
+      get() {
+        if (!persistence || !resolvedConfig) {
+          throw new Error("Call init() before get()");
+        }
+        if (!instance) {
+          const opts = convexCollectionOptions({
+            ...resolvedConfig,
+            persistence,
+            material,
+          } as any);
+          instance = createCollection(opts) as any;
+        }
+        return instance!;
+      },
+    };
+  },
+};