@multiplatform.one/frappe 6.0.0

package/src/sync/index.ts

@@ -0,0 +1,412 @@
+import type { FrappeDoc, FrappeDocData } from "../common/types";
+import type { DbModule } from "../db";
+import type { RealtimeModule } from "../realtime";
+import { onAppFocus, onOnline } from "@multiplatform.one/platform";
+import { MutationEngine } from "./mutationEngine";
+import { MutationJournal } from "./mutationJournal";
+import { NormalizedCache } from "./normalizedCache";
+import { PersistenceManager } from "./persistence";
+import { RealtimeHandler } from "./realtime";
+import { RehydrationManager } from "./rehydration";
+import { SubscriptionManager } from "./subscriptionManager";
+import { BackfillScope, Status } from "./types";
+import type {
+  BackfillArgs,
+  DebugMutationEntry,
+  DebugSubscription,
+  SyncOptions,
+  FrappeDebugState,
+  Mutation,
+  SubscriptionQuery,
+} from "./types";
+
+/**
+ * Synchronization engine that provides Zero-style UX on top of Frappe's REST API.
+ *
+ * Core capabilities:
+ * - Maintains live subscriptions with realtime hints (Socket.IO events)
+ * - Local normalized cache for instant reads and optimistic updates
+ * - Mutation journal for idempotency, offline queuing, and replay
+ * - Cursors and backfill to avoid misses after disconnects or tab sleeps
+ * - Optional persistence (IndexedDB/MMKV/localStorage) and multi-tab coordination
+ * - Cross-page coordination to prevent duplicate optimistic items
+ * - Page locking for stable pagination during edits
+ * - Intelligent rehydration strategies based on operation type
+ * - Sophisticated conflict resolution with field-level merge strategies
+ *
+ * Design constraints:
+ * - All reads/writes flow through Frappe REST to respect middleware and permissions
+ * - Realtime is advisory: correctness relies on HTTP backfill, not sockets
+ * - No assumptions about framework: minimal store subscription surface
+ *
+ * @example
+ * ```typescript
+ * const sync = new SyncModule(db, realtime, {
+ *   pageSize: 50,
+ *   persistKey: 'my-app',
+ *   onConflict: (info) => console.log('Conflict:', info),
+ *   fieldMergeConfig: {
+ *     'Task': {
+ *       'tags': 'union-set',   // FieldMergeStrategy enum value
+ *       'priority': 'max',     // pick whichever side is higher
+ *     }
+ *   }
+ * });
+ *
+ * // Subscribe to documents
+ * const { subscriptionId } = await sync.subscribe({
+ *   doctype: 'Task',
+ *   fields: ['name', 'subject', 'status'],
+ *   filters: { project: 'PROJ-001' },
+ *   orderBy: { field: 'priority', order: 'desc' },
+ *   limit: 20
+ * });
+ *
+ * // Get documents from cache
+ * const tasks = sync.getDocsForSubscription(subscriptionId);
+ * ```
+ */
+export class SyncModule {
+  private normalizedCache: NormalizedCache;
+  private mutationJournal: MutationJournal;
+  private storeListeners = new Set<() => void>();
+  private subscriptionManager: SubscriptionManager;
+  /** Mutation engine for optimistic updates and conflict resolution. */
+  private mutationEngine: MutationEngine;
+  /** Realtime event handler for Socket.IO integration. */
+  private realtimeHandler: RealtimeHandler;
+  /** Persistence manager for state snapshots. */
+  private persistenceManager: PersistenceManager;
+  /** Rehydration manager for intelligent subscription updates. */
+  private rehydrationManager: RehydrationManager;
+  /** Cleanup functions returned by lifecycle subscriptions. */
+  private cleanupFocus?: () => void;
+  private cleanupOnlineBackfill?: () => void;
+  private cleanupOnlineReplay?: () => void;
+  /** Timer handle for periodic CDC polling. */
+  private cdcIntervalHandle?: ReturnType<typeof setInterval>;
+
+  constructor(
+    private readonly db: DbModule,
+    private readonly realtime?: RealtimeModule,
+    private readonly options: SyncOptions = {},
+  ) {
+    // Initialize normalized cache with LRU eviction if maxCacheSize is set
+    this.normalizedCache = new NormalizedCache({
+      maxSize: this.options.maxCacheSize,
+    });
+    this.mutationJournal = new MutationJournal({
+      ttl: this.options.mutationTTL,
+      enableAutoCleanup: true,
+      storage: typeof localStorage !== "undefined" ? localStorage : undefined,
+    });
+    this.subscriptionManager = new SubscriptionManager(this.db, this.normalizedCache, this.options);
+
+    // Initialize rehydration and page lock managers
+    this.rehydrationManager = new RehydrationManager(
+      this.db,
+      this.normalizedCache,
+      this.subscriptionManager,
+      this.mutationJournal,
+    );
+
+    this.mutationEngine = new MutationEngine(
+      this.db,
+      this.realtime,
+      this.normalizedCache,
+      this.mutationJournal,
+      this.subscriptionManager,
+      this.rehydrationManager,
+      this.options,
+    );
+
+    // Setup realtime handlers with callbacks to main module
+    const realtimeOptions: SyncOptions = {
+      ...this.options,
+      onResetCache: () => this.resetCache(),
+      onBackfillAll: () => this.backfill({ scope: BackfillScope.All }),
+    };
+
+    this.realtimeHandler = new RealtimeHandler(
+      this.db,
+      this.realtime,
+      this.normalizedCache,
+      this.subscriptionManager,
+      this.rehydrationManager,
+      realtimeOptions,
+    );
+
+    this.persistenceManager = new PersistenceManager(
+      this.normalizedCache,
+      this.mutationJournal,
+      this.subscriptionManager,
+      this.mutationEngine,
+      realtimeOptions,
+    );
+
+    this.setupWindowEvents();
+  }
+
+  /**
+   * Sets up lifecycle event listeners for sync triggers.
+   * Stores cleanup functions returned by each subscription.
+   */
+  private setupWindowEvents() {
+    this.cleanupFocus = onAppFocus(() => this.backfill({ scope: BackfillScope.All }));
+    this.cleanupOnlineBackfill = onOnline(() => this.backfill({ scope: BackfillScope.All }));
+    this.cleanupOnlineReplay = onOnline(() => this.replayPending());
+
+    if (this.options.cdcInterval && this.options.cdcInterval > 0) {
+      this.cdcIntervalHandle = setInterval(
+        () => this.backfill({ scope: BackfillScope.All }),
+        this.options.cdcInterval,
+      );
+    }
+  }
+
+  /**
+   * Creates a subscription, seeds cache with initial rows, and stores a backfill checkpoint.
+   */
+  async subscribe<Data extends FrappeDocData>(query: SubscriptionQuery) {
+    const result = await this.subscriptionManager.subscribe<Data>(query);
+    this.emit();
+    return result;
+  }
+
+  /**
+   * Removes a subscription by id.
+   */
+  unsubscribe(subscriptionId: string) {
+    this.subscriptionManager.unsubscribe(subscriptionId);
+    this.emit();
+  }
+
+  async mutate(mutation: Mutation<FrappeDocData>) {
+    await this.mutationEngine.mutate(mutation);
+    // Broadcast mutation to other tabs for cross-tab sync
+    this.persistenceManager.broadcastMutation(mutation);
+    this.emit();
+  }
+
+  /**
+   * Reconciles local state by fetching changes since stored cursors.
+   */
+  async backfill(args: BackfillArgs) {
+    await this.subscriptionManager.backfill(args);
+    this.emit();
+  }
+
+  /**
+   * Replays pending mutations with error handling.
+   */
+  async replayPending() {
+    await this.mutationEngine.replayPending();
+    this.emit();
+  }
+
+  /** Reads a single document from the local cache. */
+  getDocFromCache<T extends FrappeDocData = FrappeDocData>(
+    doctype: string,
+    name: string,
+  ): T | undefined {
+    return this.normalizedCache.get({ doctype, name }) as T | undefined;
+  }
+
+  /** Returns subscription metadata by id. */
+  getSubscription(id: string) {
+    return this.subscriptionManager.getSubscription(id);
+  }
+
+  /**
+   * Returns current rows for a subscription from the cache.
+   */
+  getDocsForSubscription<Data extends FrappeDocData>(id: string): FrappeDoc<Data>[] {
+    return this.subscriptionManager.getDocsForSubscription<FrappeDoc<Data>>(id);
+  }
+
+  /**
+   * Get a specific page of documents using virtual pagination.
+   * Requires the subscription to have been created with `progressive: true`.
+   */
+  async getPage<Data extends FrappeDocData>(
+    subscriptionId: string,
+    page: number,
+  ): Promise<FrappeDoc<Data>[]> {
+    const result = await this.subscriptionManager.getPage<FrappeDoc<Data>>(subscriptionId, page);
+    this.emit();
+    return result;
+  }
+
+  /**
+   * Get the next batch of documents after a cursor for infinite scroll.
+   * Requires the subscription to have been created with `progressive: true`.
+   */
+  async getNextBatch<Data extends FrappeDocData>(
+    subscriptionId: string,
+    cursorDocId: string,
+    limit: number,
+  ): Promise<FrappeDoc<Data>[]> {
+    const result = await this.subscriptionManager.getNextBatch<FrappeDoc<Data>>(
+      subscriptionId,
+      cursorDocId,
+      limit,
+    );
+    this.emit();
+    return result;
+  }
+
+  /**
+   * Get page information for a subscription.
+   */
+  getPageInfo(subscriptionId: string) {
+    return this.subscriptionManager.getPageInfo(subscriptionId);
+  }
+
+  /** Refetches authoritative copies for specific documents and updates cache. */
+  async refetchDocs(docs: { doctype: string; name: string }[]) {
+    await this.subscriptionManager.backfill({
+      scope: BackfillScope.Docs,
+      docs,
+    });
+    this.emit();
+  }
+
+  /** Adds a store listener; returns an unsubscribe function. */
+  subscribeStore(listener: () => void): () => void {
+    this.storeListeners.add(listener);
+    return () => {
+      this.storeListeners.delete(listener);
+    };
+  }
+
+  /** Notifies listeners, schedules persistence, and broadcasts cross-tab changes. */
+  private emit() {
+    // Fan out local state changes to all subscribers
+    for (const listener of this.storeListeners) listener();
+    // Debounce a persistence snapshot if configured
+    this.persistenceManager.scheduleSave();
+    // Notify other tabs so they can converge local state
+    this.persistenceManager.broadcastChange();
+  }
+
+  /** Clears subscriptions, cache, journal, and temp id mappings. */
+  resetCache() {
+    // Drop all live subs, in-memory cache, journal, and temp mappings
+    this.subscriptionManager.clear();
+    this.normalizedCache.clear();
+    this.mutationJournal.clear();
+    this.mutationEngine.clearTemporaryMappings();
+    // Notify listeners of the global reset so UI can re-subscribe as needed
+    this.emit();
+  }
+
+  /** Cleanup and dispose of resources */
+  async dispose() {
+    // Clear all listeners
+    this.storeListeners.clear();
+
+    // Clear subscriptions and cache
+    this.resetCache();
+
+    // Dispose of all managers
+    this.mutationJournal.dispose();
+    this.mutationEngine.dispose();
+    this.realtimeHandler.dispose();
+    this.persistenceManager.destroy();
+    this.rehydrationManager.clear();
+
+    // Remove lifecycle event listeners
+    this.cleanupFocus?.();
+    this.cleanupOnlineBackfill?.();
+    this.cleanupOnlineReplay?.();
+    this.cleanupFocus = undefined;
+    this.cleanupOnlineBackfill = undefined;
+    this.cleanupOnlineReplay = undefined;
+
+    if (this.cdcIntervalHandle) {
+      clearInterval(this.cdcIntervalHandle);
+      this.cdcIntervalHandle = undefined;
+    }
+  }
+
+  /**
+   * Returns a snapshot of internal state for devtools inspection.
+   * Prefixed with __ to signal internal/debug use only.
+   */
+  __debug(): FrappeDebugState {
+    const allSubs = this.subscriptionManager.getAllSubscriptions();
+    const vpm = this.subscriptionManager.getVirtualPageManager();
+
+    const subscriptions: DebugSubscription[] = [];
+    for (const [, sub] of allSubs) {
+      const virtualSub = vpm.getSubscription(sub.id);
+      const pageInfo = vpm.getPageInfo(sub.id);
+      const islands = vpm.getPageIslands(sub.id);
+
+      const debugSub: DebugSubscription = {
+        id: sub.id,
+        query: sub.query,
+        docCount: sub.docIds.length,
+        tombstoneCount: sub.tombstones?.size ?? 0,
+        checkpoint: sub.checkpoint,
+        cursor: sub.cursor,
+        pageInfo,
+      };
+
+      if (virtualSub) {
+        const pc = virtualSub.getProgressiveCache();
+        debugSub.progressiveCache = {
+          docIds: pc.docIds,
+          nextCursor: pc.nextCursor,
+          hasMore: pc.hasMore,
+          totalCount: pc.totalCount,
+          lastFetchedAt: pc.lastFetchedAt?.toISOString(),
+          islands,
+        };
+      }
+
+      subscriptions.push(debugSub);
+    }
+
+    const cacheStats = this.normalizedCache.getStats();
+
+    const allEntries = this.mutationJournal.getAllEntries();
+    const pending: DebugMutationEntry[] = [];
+    const confirmed: DebugMutationEntry[] = [];
+    const failed: DebugMutationEntry[] = [];
+
+    for (const { id, entry } of allEntries) {
+      const debugEntry: DebugMutationEntry = {
+        id,
+        operation: entry.operation,
+        doctype: entry.doctype,
+        docname: entry.docname,
+        status: entry.status,
+        createdAt: entry.createdAt ?? 0,
+        retryCount: entry.retryCount ?? 0,
+        error: entry.error ? ((entry.error as Error).message ?? String(entry.error)) : undefined,
+      };
+      if (entry.status === Status.Pending) pending.push(debugEntry);
+      else if (entry.status === Status.Confirmed) confirmed.push(debugEntry);
+      else if (entry.status === Status.Failed) failed.push(debugEntry);
+    }
+
+    return {
+      subscriptions,
+      normalizedCache: cacheStats,
+      mutationJournal: { pending, confirmed, failed },
+      connection: {
+        socketConnected: this.realtimeHandler.getIsConnected(),
+        isLeader: this.persistenceManager.getIsLeader(),
+        isOnline: typeof navigator !== "undefined" ? navigator.onLine : true,
+      },
+    };
+  }
+
+  /**
+   * Checks if this tab is the leader for multi-tab coordination.
+   */
+  isLeader(): boolean {
+    return this.persistenceManager.getIsLeader();
+  }
+}