@trestleinc/replicate 1.1.2 → 1.2.0-preview.1

package/src/client/errors.ts

@@ -26,7 +26,7 @@ export class ReconciliationError extends Data.TaggedError("ReconciliationError")
 }> {}
 
 export class ProseError extends Data.TaggedError("ProseError")<{
-  readonly documentId: string;
+  readonly document: string;
   readonly field: string;
   readonly collection: string;
 }> {}

package/src/client/index.ts

@@ -5,6 +5,8 @@ export {
   type Materialized,
 } from "$/client/collection";
 
+export { type Seq } from "$/client/services/seq";
+
 import {
   NetworkError,
   IDBError,
@@ -25,14 +27,9 @@ export const errors = {
   NonRetriable: NonRetriableError,
 } as const;
 
-import type { ProseValue } from "$/shared/types";
 import { extract } from "$/client/merge";
-import { prose as proseSchema } from "$/client/prose-schema";
-
-function empty(): ProseValue {
-  return { type: "doc", content: [] } as unknown as ProseValue;
-}
+import { prose as proseSchema } from "$/client/prose";
 
-export const prose = Object.assign(proseSchema, { extract, empty });
+export const prose = Object.assign(proseSchema, { extract });
 
 export { persistence, type StorageAdapter, type Persistence } from "$/client/persistence/index";

package/src/client/merge.ts

@@ -360,10 +360,10 @@ export function serializeYMapValue(value: unknown): unknown {
  */
 export function getFragmentFromYMap(
   ymap: Y.Map<unknown>,
-  documentId: string,
+  document: string,
   field: string,
 ): Y.XmlFragment | null {
-  const doc = ymap.get(documentId);
+  const doc = ymap.get(document);
   if (!isYMap(doc)) {
     return null;
   }

package/src/client/persistence/indexeddb.ts

@@ -12,15 +12,10 @@ function openDatabase(dbName: string): Promise<IDBDatabase> {
     request.onsuccess = () => resolve(request.result);
     request.onupgradeneeded = () => {
       const db = request.result;
-      if (!db.objectStoreNames.contains(UPDATES_STORE)) {
-        db.createObjectStore(UPDATES_STORE, { autoIncrement: true });
-      }
-      if (!db.objectStoreNames.contains(SNAPSHOTS_STORE)) {
-        db.createObjectStore(SNAPSHOTS_STORE);
-      }
-      if (!db.objectStoreNames.contains(KV_STORE)) {
-        db.createObjectStore(KV_STORE);
-      }
+      db.createObjectStore(SNAPSHOTS_STORE);
+      db.createObjectStore(KV_STORE);
+      const updatesStore = db.createObjectStore(UPDATES_STORE, { autoIncrement: true });
+      updatesStore.createIndex("by_collection", "collection", { unique: false });
     };
   });
 }
@@ -91,12 +86,13 @@ class IDBPersistenceProvider implements PersistenceProvider {
         }
 
         const updatesStore = tx.objectStore(UPDATES_STORE);
-        const updatesRequest = updatesStore.getAll();
+        const index = updatesStore.index("by_collection");
+        const updatesRequest = index.getAll(this.collection);
 
         updatesRequest.onsuccess = () => {
-          const updates = updatesRequest.result as Uint8Array[];
-          for (const update of updates) {
-            Y.applyUpdate(this.ydoc, update, "idb");
+          const records = updatesRequest.result as { collection: string; data: Uint8Array }[];
+          for (const record of records) {
+            Y.applyUpdate(this.ydoc, record.data, "idb");
           }
           resolve();
         };
@@ -114,7 +110,7 @@ class IDBPersistenceProvider implements PersistenceProvider {
     return new Promise((resolve, reject) => {
       const tx = this.db.transaction(UPDATES_STORE, "readwrite");
       const store = tx.objectStore(UPDATES_STORE);
-      const request = store.add(update);
+      const request = store.add({ collection: this.collection, data: update });
       request.onerror = () => reject(request.error ?? new Error("IndexedDB save update failed"));
       request.onsuccess = () => resolve();
     });

package/src/client/prose.ts

@@ -2,179 +2,128 @@
  * Prose Field Helpers - Document-level state management for rich text sync
  *
  * Manages Y.XmlFragment observation, debounced sync, and pending state.
- * Uses document-level tracking to prevent race conditions.
+ * Uses CollectionContext for state storage.
  */
 
 import * as Y from "yjs";
+import { z } from "zod";
 import type { Collection } from "@tanstack/db";
 import { getLogger } from "$/client/logger";
 import { serializeYMapValue } from "$/client/merge";
+import { getContext, hasContext, type ProseState } from "$/client/services/context";
+import type { ProseValue } from "$/shared/types";
 
-/** Server origin - changes from server should not trigger local sync */
 const SERVER_ORIGIN = "server";
+const noop = (): void => undefined;
 
 const logger = getLogger(["replicate", "prose"]);
 
-// Default debounce time for prose sync
 const DEFAULT_DEBOUNCE_MS = 1000;
 
-// ============================================================================
-// Document-Level State (keyed by "collection:documentId")
-// ============================================================================
-
-// Track when applying server data to prevent echo loops - DOCUMENT-LEVEL
-const applyingFromServer = new Map<string, boolean>();
-
-// Debounce timers for prose sync
-const debounceTimers = new Map<string, ReturnType<typeof setTimeout>>();
-
-// Last synced state vectors for computing deltas
-const lastSyncedVectors = new Map<string, Uint8Array>();
-
-// Pending sync state
-const pendingState = new Map<string, boolean>();
-
-// Pending state change listeners
-const pendingListeners = new Map<string, Set<(pending: boolean) => void>>();
-
-// Fragment observer cleanup functions
-const fragmentObservers = new Map<string, () => void>();
-
-// Failed sync queue for retry
-const failedSyncQueue = new Map<string, boolean>();
-
-// ============================================================================
-// Applying From Server (Document-Level)
-// ============================================================================
+function getProseState(collection: string): ProseState | null {
+  if (!hasContext(collection)) return null;
+  return getContext(collection).prose;
+}
 
-/**
- * Check if a document is currently applying server data.
- * Used to prevent echo loops in onUpdate handlers.
- */
-export function isApplyingFromServer(collection: string, documentId: string): boolean {
-  const key = `${collection}:${documentId}`;
-  return applyingFromServer.get(key) ?? false;
+export function isApplyingFromServer(collection: string, document: string): boolean {
+  const state = getProseState(collection);
+  if (!state) return false;
+  return state.applyingFromServer.get(document) ?? false;
 }
 
-/**
- * Set whether a document is currently applying server data.
- */
 export function setApplyingFromServer(
   collection: string,
-  documentId: string,
+  document: string,
   value: boolean,
 ): void {
-  const key = `${collection}:${documentId}`;
+  const state = getProseState(collection);
+  if (!state) return;
   if (value) {
-    applyingFromServer.set(key, true);
+    state.applyingFromServer.set(document, true);
   }
   else {
-    applyingFromServer.delete(key);
+    state.applyingFromServer.delete(document);
   }
 }
 
-// ============================================================================
-// Pending State Management
-// ============================================================================
-
-/**
- * Set pending state and notify listeners.
- */
-function setPendingInternal(key: string, value: boolean): void {
-  const current = pendingState.get(key) ?? false;
+function setPendingInternal(collection: string, document: string, value: boolean): void {
+  const state = getProseState(collection);
+  if (!state) return;
 
+  const current = state.pendingState.get(document) ?? false;
   if (current !== value) {
-    pendingState.set(key, value);
-    const listeners = pendingListeners.get(key);
+    state.pendingState.set(document, value);
+    const listeners = state.pendingListeners.get(document);
     if (listeners) {
       for (const cb of listeners) {
         try {
           cb(value);
         }
         catch (err) {
-          logger.error("Pending listener error", { key, error: String(err) });
+          logger.error("Pending listener error", { collection, document, error: String(err) });
         }
       }
     }
   }
 }
 
-/**
- * Get current pending state for a document.
- */
-export function isPending(collection: string, documentId: string): boolean {
-  return pendingState.get(`${collection}:${documentId}`) ?? false;
+export function isPending(collection: string, document: string): boolean {
+  const state = getProseState(collection);
+  if (!state) return false;
+  return state.pendingState.get(document) ?? false;
 }
 
-/**
- * Subscribe to pending state changes for a document.
- */
 export function subscribePending(
   collection: string,
-  documentId: string,
+  document: string,
   callback: (pending: boolean) => void,
 ): () => void {
-  const key = `${collection}:${documentId}`;
+  const state = getProseState(collection);
+  if (!state) return noop;
 
-  let listeners = pendingListeners.get(key);
+  let listeners = state.pendingListeners.get(document);
   if (!listeners) {
     listeners = new Set();
-    pendingListeners.set(key, listeners);
+    state.pendingListeners.set(document, listeners);
   }
 
   listeners.add(callback);
   return () => {
     listeners?.delete(callback);
     if (listeners?.size === 0) {
-      pendingListeners.delete(key);
+      state.pendingListeners.delete(document);
     }
   };
 }
 
-// ============================================================================
-// Cancel Pending Sync
-// ============================================================================
-
-/**
- * Cancel any pending debounced sync for a document.
- * Called when receiving remote updates to avoid conflicts.
- */
-export function cancelPending(collection: string, documentId: string): void {
-  const key = `${collection}:${documentId}`;
-  const timer = debounceTimers.get(key);
+export function cancelPending(collection: string, document: string): void {
+  const state = getProseState(collection);
+  if (!state) return;
 
+  const timer = state.debounceTimers.get(document);
   if (timer) {
     clearTimeout(timer);
-    debounceTimers.delete(key);
-    setPendingInternal(key, false);
-    logger.debug("Cancelled pending sync due to remote update", { collection, documentId });
+    state.debounceTimers.delete(document);
+    setPendingInternal(collection, document, false);
+    logger.debug("Cancelled pending sync due to remote update", { collection, document });
   }
 }
 
-/**
- * Cancel all pending syncs for a collection.
- * Called when receiving a snapshot that replaces all state.
- */
 export function cancelAllPending(collection: string): void {
-  const prefix = `${collection}:`;
-  for (const [key, timer] of debounceTimers) {
-    if (key.startsWith(prefix)) {
-      clearTimeout(timer);
-      debounceTimers.delete(key);
-      setPendingInternal(key, false);
-    }
+  const state = getProseState(collection);
+  if (!state) return;
+
+  for (const [doc, timer] of state.debounceTimers) {
+    clearTimeout(timer);
+    state.debounceTimers.delete(doc);
+    setPendingInternal(collection, doc, false);
   }
   logger.debug("Cancelled all pending syncs", { collection });
 }
 
-// ============================================================================
-// Fragment Observation
-// ============================================================================
-
-/** Configuration for fragment observation */
 export interface ProseObserverConfig {
   collection: string;
-  documentId: string;
+  document: string;
   field: string;
   fragment: Y.XmlFragment;
   ydoc: Y.Doc;
@@ -183,14 +132,10 @@ export interface ProseObserverConfig {
   debounceMs?: number;
 }
 
-/**
- * Set up observation for a prose field's Y.XmlFragment.
- * Returns a cleanup function.
- */
 export function observeFragment(config: ProseObserverConfig): () => void {
   const {
     collection,
-    documentId,
+    document,
     field,
     fragment,
     ydoc,
@@ -198,175 +143,174 @@ export function observeFragment(config: ProseObserverConfig): () => void {
     collectionRef,
     debounceMs = DEFAULT_DEBOUNCE_MS,
   } = config;
-  const key = `${collection}:${documentId}`;
 
-  // Skip if already observing this document
-  const existingCleanup = fragmentObservers.get(key);
+  const state = getProseState(collection);
+  if (!state) {
+    logger.warn("Cannot observe fragment - collection not initialized", { collection, document });
+    return noop;
+  }
+
+  const existingCleanup = state.fragmentObservers.get(document);
   if (existingCleanup) {
-    logger.debug("Fragment already being observed", { collection, documentId, field });
+    logger.debug("Fragment already being observed", { collection, document, field });
     return existingCleanup;
   }
 
   const observerHandler = (_events: Y.YEvent<any>[], transaction: Y.Transaction) => {
-    // Skip server-originated changes (echo prevention via transaction origin)
     if (transaction.origin === SERVER_ORIGIN) {
       return;
     }
 
-    // Clear existing timer
-    const existing = debounceTimers.get(key);
+    const existing = state.debounceTimers.get(document);
     if (existing) clearTimeout(existing);
 
-    // Mark as pending
-    setPendingInternal(key, true);
+    setPendingInternal(collection, document, true);
 
-    // Schedule sync
     const timer = setTimeout(async () => {
-      debounceTimers.delete(key);
-
-      const itemYMap = ymap.get(documentId) as Y.Map<unknown> | undefined;
-      if (!itemYMap) {
-        logger.error("Document not found", { collection, documentId });
-        setPendingInternal(key, false);
-        return;
-      }
+      state.debounceTimers.delete(document);
 
       try {
-        // Compute delta since last sync
-        const lastVector = lastSyncedVectors.get(key);
+        const lastVector = state.lastSyncedVectors.get(document);
         const delta = lastVector
           ? Y.encodeStateAsUpdateV2(ydoc, lastVector)
           : Y.encodeStateAsUpdateV2(ydoc);
 
         if (delta.length <= 2) {
-          logger.debug("No changes to sync", { collection, documentId });
-          setPendingInternal(key, false);
+          logger.debug("No changes to sync", { collection, document });
+          setPendingInternal(collection, document, false);
           return;
         }
 
-        const crdtBytes = delta.buffer as ArrayBuffer;
+        const bytes = delta.buffer as ArrayBuffer;
         const currentVector = Y.encodeStateVector(ydoc);
 
         logger.debug("Syncing prose delta", {
           collection,
-          documentId,
+          document,
           deltaSize: delta.byteLength,
         });
 
-        const materializedDoc = serializeYMapValue(itemYMap);
+        const material = serializeYMapValue(ymap);
 
-        // Send via collection.update with contentSync metadata
         const result = collectionRef.update(
-          documentId,
-          { metadata: { contentSync: { crdtBytes, materializedDoc } } },
+          document,
+          { metadata: { contentSync: { bytes, material } } },
           (draft: any) => {
             draft.updatedAt = Date.now();
           },
         );
         await result.isPersisted.promise;
 
-        // Update last synced vector
-        lastSyncedVectors.set(key, currentVector);
-        failedSyncQueue.delete(key);
-        setPendingInternal(key, false);
-        logger.debug("Prose sync completed", { collection, documentId });
+        state.lastSyncedVectors.set(document, currentVector);
+        state.failedSyncQueue.delete(document);
+        setPendingInternal(collection, document, false);
+        logger.debug("Prose sync completed", { collection, document });
       }
       catch (err) {
         logger.error("Prose sync failed, queued for retry", {
           collection,
-          documentId,
+          document,
           error: String(err),
         });
-        failedSyncQueue.set(key, true);
-        // Keep pending=true for retry indication
+        state.failedSyncQueue.set(document, true);
       }
     }, debounceMs);
 
-    debounceTimers.set(key, timer);
+    state.debounceTimers.set(document, timer);
 
-    // Also retry any failed syncs for this document
-    if (failedSyncQueue.has(key)) {
-      failedSyncQueue.delete(key);
-      logger.debug("Retrying failed sync", { collection, documentId });
+    if (state.failedSyncQueue.has(document)) {
+      state.failedSyncQueue.delete(document);
+      logger.debug("Retrying failed sync", { collection, document });
     }
   };
 
-  // Set up deep observation on the fragment
   fragment.observeDeep(observerHandler);
 
   const cleanup = () => {
     fragment.unobserveDeep(observerHandler);
-    cancelPending(collection, documentId);
-    fragmentObservers.delete(key);
-    lastSyncedVectors.delete(key);
-    logger.debug("Fragment observer cleaned up", { collection, documentId, field });
+    cancelPending(collection, document);
+    state.fragmentObservers.delete(document);
+    state.lastSyncedVectors.delete(document);
+    logger.debug("Fragment observer cleaned up", { collection, document, field });
   };
 
-  fragmentObservers.set(key, cleanup);
-  logger.debug("Fragment observer registered", { collection, documentId, field });
+  state.fragmentObservers.set(document, cleanup);
+  logger.debug("Fragment observer registered", { collection, document, field });
 
   return cleanup;
 }
 
-// ============================================================================
-// Cleanup
-// ============================================================================
-
-/**
- * Clean up all prose state for a collection.
- * Called when collection is destroyed.
- */
 export function cleanup(collection: string): void {
-  const prefix = `${collection}:`;
-
-  // Cancel all pending syncs
-  for (const [key, timer] of debounceTimers) {
-    if (key.startsWith(prefix)) {
-      clearTimeout(timer);
-      debounceTimers.delete(key);
-    }
-  }
+  const state = getProseState(collection);
+  if (!state) return;
 
-  // Clear pending state and listeners
-  for (const key of pendingState.keys()) {
-    if (key.startsWith(prefix)) {
-      pendingState.delete(key);
-    }
+  for (const [, timer] of state.debounceTimers) {
+    clearTimeout(timer);
   }
-  for (const key of pendingListeners.keys()) {
-    if (key.startsWith(prefix)) {
-      pendingListeners.delete(key);
-    }
+  state.debounceTimers.clear();
+  state.pendingState.clear();
+  state.pendingListeners.clear();
+  state.applyingFromServer.clear();
+  state.lastSyncedVectors.clear();
+
+  for (const [, cleanupFn] of state.fragmentObservers) {
+    cleanupFn();
   }
+  state.fragmentObservers.clear();
+  state.failedSyncQueue.clear();
 
-  // Clear applying from server flags
-  for (const key of applyingFromServer.keys()) {
-    if (key.startsWith(prefix)) {
-      applyingFromServer.delete(key);
-    }
-  }
+  logger.debug("Prose cleanup complete", { collection });
+}
 
-  // Clear last synced vectors
-  for (const key of lastSyncedVectors.keys()) {
-    if (key.startsWith(prefix)) {
-      lastSyncedVectors.delete(key);
-    }
-  }
+const PROSE_MARKER = Symbol.for("replicate:prose");
+
+function createProseSchema(): z.ZodType<ProseValue> {
+  const schema = z.custom<ProseValue>(
+    (val) => {
+      if (val == null) return true;
+      if (typeof val !== "object") return false;
+      return (val as { type?: string }).type === "doc";
+    },
+    { message: "Expected prose document with type \"doc\"" },
+  );
+
+  Object.defineProperty(schema, PROSE_MARKER, { value: true, writable: false });
+
+  return schema;
+}
+
+function emptyProse(): ProseValue {
+  return { type: "doc", content: [] } as unknown as ProseValue;
+}
 
-  // Clean up fragment observers
-  for (const [key, cleanupFn] of fragmentObservers) {
-    if (key.startsWith(prefix)) {
-      cleanupFn();
-      fragmentObservers.delete(key);
+export function prose(): z.ZodType<ProseValue> {
+  return createProseSchema();
+}
+
+prose.empty = emptyProse;
+
+export function isProseSchema(schema: unknown): boolean {
+  return (
+    schema != null
+    && typeof schema === "object"
+    && PROSE_MARKER in schema
+    && (schema as Record<symbol, unknown>)[PROSE_MARKER] === true
+  );
+}
+
+export function extractProseFields(schema: z.ZodObject<z.ZodRawShape>): string[] {
+  const fields: string[] = [];
+
+  for (const [key, fieldSchema] of Object.entries(schema.shape)) {
+    let unwrapped = fieldSchema;
+    while (unwrapped instanceof z.ZodOptional || unwrapped instanceof z.ZodNullable) {
+      unwrapped = unwrapped.unwrap();
     }
-  }
 
-  // Clear failed sync queue
-  for (const key of failedSyncQueue.keys()) {
-    if (key.startsWith(prefix)) {
-      failedSyncQueue.delete(key);
+    if (isProseSchema(unwrapped)) {
+      fields.push(key);
     }
   }
 
-  logger.debug("Prose cleanup complete", { collection });
+  return fields;
 }