@multiplatform.one/frappe 6.0.0

package/src/sync/backfill.ts

@@ -0,0 +1,378 @@
+import type { FrappeDoc, FrappeDocBase, FrappeDocData } from "../common/types";
+import type { DbModule } from "../db";
+import { matchesFilters } from "./filters";
+
+/**
+ * Calculate checkpoint from documents (find max modified timestamp)
+ * Uses proper Date parsing for accurate comparison across different timestamp formats
+ */
+function calculateCheckpointFromDocs(docs: FrappeDocBase[]): string | undefined {
+  if (docs.length === 0) return undefined;
+
+  let maxTimestamp = "";
+  let maxDate: Date | null = null;
+
+  for (const doc of docs) {
+    const modified = doc.modified as string | undefined;
+    if (!modified) continue;
+
+    try {
+      const date = new Date(modified);
+      if (Number.isNaN(date.getTime())) continue;
+
+      if (!maxDate || date > maxDate) {
+        maxDate = date;
+        maxTimestamp = modified;
+      }
+    } catch {
+      // Fallback to string comparison if Date parsing fails
+      if (!maxTimestamp || modified > maxTimestamp) {
+        maxTimestamp = modified;
+      }
+    }
+  }
+
+  return maxTimestamp || undefined;
+}
+import type { NormalizedCache } from "./normalizedCache";
+import type { VirtualSubscription } from "./virtualPageManager";
+
+const log = console;
+
+export enum BackfillReason {
+  DeleteGap = "delete-gap",
+  Initial = "initial",
+  Pagination = "pagination",
+  Periodic = "periodic",
+  Recovery = "recovery",
+  Rehydration = "rehydration",
+  RemoteCreate = "remote-create",
+  RemoteUpdate = "remote-update",
+}
+
+export enum BackfillStrategy {
+  CursorFetch = "cursor-fetch",
+  FullRefresh = "full-refresh",
+  GapFill = "gap-fill",
+  Incremental = "incremental",
+  PageOnly = "page-only",
+}
+
+export interface BackfillOptions {
+  maxRetries?: number;
+  pageSize?: number;
+  retryDelay?: number;
+  batchSize?: number;
+}
+
+export class BackfillManager {
+  private activeBackfills = new Map<string, Promise<void>>();
+  private backfillQueue = new Map<string, BackfillReason[]>();
+
+  constructor(
+    private db: DbModule,
+    private normalizedCache: NormalizedCache,
+    private options: BackfillOptions = {},
+  ) {}
+
+  async backfill(subscription: VirtualSubscription, reason: BackfillReason): Promise<void> {
+    const subscriptionId = subscription.id;
+    if (this.activeBackfills.has(subscriptionId)) {
+      this.queueBackfill(subscriptionId, reason);
+      return this.activeBackfills.get(subscriptionId)!;
+    }
+    const backfillPromise = this.performBackfill(subscription, reason);
+    this.activeBackfills.set(subscriptionId, backfillPromise);
+    let backfillSucceeded = false;
+    try {
+      await backfillPromise;
+      backfillSucceeded = true;
+    } catch (err) {
+      log.error(err);
+    } finally {
+      this.activeBackfills.delete(subscriptionId);
+      if (backfillSucceeded) {
+        const queued = this.backfillQueue.get(subscriptionId);
+        if (queued && queued.length > 0) {
+          const reason = queued.shift()!;
+          if (queued.length === 0) this.backfillQueue.delete(subscriptionId);
+          if (reason) this.backfill(subscription, reason).catch(log.error);
+        }
+      } else {
+        this.backfillQueue.delete(subscriptionId);
+      }
+    }
+  }
+
+  /**
+   * Backfill one item to fill a gap
+   */
+  async backfillOne(subscription: VirtualSubscription): Promise<FrappeDoc<FrappeDocData> | null> {
+    if (!subscription.getProgressiveCache().hasMore) {
+      return null;
+    }
+
+    try {
+      // Fetch one document using cursor if available
+      const docs = await this.fetchWithCursor(subscription, 1);
+
+      if (docs.length > 0) {
+        const doc = docs[0];
+
+        // Update caches using subscription methods
+        subscription.processDocuments([doc], this.normalizedCache, {
+          calculateCheckpoint: false, // Single doc doesn't need checkpoint update
+        });
+
+        return doc;
+      }
+
+      // No more documents
+      subscription.updateProgressiveCache({ hasMore: false });
+      return null;
+    } catch (error) {
+      console.error("Failed to backfill one item:", error);
+      return null;
+    }
+  }
+
+  /**
+   * Rehydrate current page
+   */
+  async rehydrateCurrentPage(subscription: VirtualSubscription): Promise<void> {
+    const pageSize = subscription.viewWindow.pageSize;
+    const currentPage = subscription.viewWindow.currentPage;
+    const start = (currentPage - 1) * pageSize;
+
+    try {
+      // Fetch current page from server
+      const response = await this.db.getDocList(subscription.query.doctype, {
+        fields: subscription.query.fields,
+        filters: subscription.query.filters,
+        orderBy: subscription.query.orderBy,
+        limitStart: start,
+        limit: pageSize,
+      });
+
+      // Replace page in cache
+      const newDocIds = response.map((doc) => doc.name);
+      const currentDocIds = subscription.getProgressiveCache().docIds;
+      const updatedDocIds = [...currentDocIds];
+      updatedDocIds.splice(start, pageSize, ...newDocIds);
+      subscription.updateProgressiveCache({ docIds: updatedDocIds });
+
+      // Update normalized cache
+      for (const doc of response) {
+        this.normalizedCache.upsert(doc);
+      }
+
+      // Clear rehydration flag
+      subscription.needsRehydration = false;
+    } catch (error) {
+      console.error("Failed to rehydrate current page:", error);
+      throw error;
+    }
+  }
+
+  private async performBackfill(
+    subscription: VirtualSubscription,
+    reason: BackfillReason,
+  ): Promise<void> {
+    const strategy = this.determineStrategy(subscription, reason);
+    switch (strategy) {
+      case BackfillStrategy.FullRefresh:
+        await this.fullRefresh(subscription);
+        break;
+      case BackfillStrategy.Incremental:
+        await this.incrementalBackfill(subscription);
+        break;
+      case BackfillStrategy.PageOnly:
+        await this.pageBackfill(subscription);
+        break;
+      case BackfillStrategy.GapFill:
+        await this.gapBackfill(subscription);
+        break;
+      case BackfillStrategy.CursorFetch:
+        await this.cursorFetch(subscription);
+        break;
+    }
+  }
+
+  private determineStrategy(
+    subscription: VirtualSubscription,
+    reason: BackfillReason,
+  ): BackfillStrategy {
+    switch (reason) {
+      case BackfillReason.Initial:
+        return BackfillStrategy.FullRefresh;
+      case BackfillReason.Pagination:
+        return subscription.getProgressiveCache().nextCursor
+          ? BackfillStrategy.CursorFetch
+          : BackfillStrategy.Incremental;
+      case BackfillReason.DeleteGap:
+        return BackfillStrategy.GapFill;
+      case BackfillReason.RemoteCreate:
+      case BackfillReason.RemoteUpdate:
+      case BackfillReason.Rehydration:
+        return BackfillStrategy.PageOnly;
+      case BackfillReason.Periodic:
+        return subscription.checkpoint
+          ? BackfillStrategy.Incremental
+          : BackfillStrategy.FullRefresh;
+      case BackfillReason.Recovery:
+        return subscription.checkpoint
+          ? BackfillStrategy.Incremental
+          : BackfillStrategy.FullRefresh;
+      default:
+        return BackfillStrategy.Incremental;
+    }
+  }
+
+  private async fullRefresh(subscription: VirtualSubscription): Promise<void> {
+    const batchSize = this.options.batchSize || this.options.pageSize || 100;
+    const response = await this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      limit: batchSize,
+      limitStart: 0,
+    });
+    // Update caches using subscription methods
+    subscription.processDocuments(response, this.normalizedCache);
+    subscription.updateProgressiveCache({
+      hasMore: response.length === batchSize,
+    });
+  }
+
+  private async incrementalBackfill(subscription: VirtualSubscription): Promise<void> {
+    if (!subscription.checkpoint) {
+      return this.fullRefresh(subscription);
+    }
+
+    try {
+      // Use CDC backfill API
+      const response = await this.db.backfill({
+        doctype: subscription.query.doctype,
+        fields: subscription.query.fields,
+        filters: subscription.query.filters,
+        modifiedAfter: subscription.checkpoint,
+        limit: this.options.pageSize || 200,
+      });
+
+      // Process deletions using shared utility
+      if ("deleted" in response && Array.isArray(response.deleted)) {
+        const deletedDocIds = (response.deleted as any[])
+          .filter((deletion) => deletion?.name)
+          .map((deletion) => deletion.name);
+
+        if (deletedDocIds.length > 0) {
+          subscription.removeDocuments(deletedDocIds, this.normalizedCache);
+        }
+      }
+
+      // Process updates and creates
+      for (const doc of response.rows) {
+        const exists = subscription.getProgressiveCache().docIds.includes(doc.name);
+        const matches = matchesFilters(doc as FrappeDoc<FrappeDocData>, subscription.query.filters);
+
+        if (exists && !matches) {
+          // Remove - no longer matches
+          subscription.removeDocuments([doc.name], this.normalizedCache);
+        } else if (!exists && matches) {
+          // Add - now matches (but we don't know where)
+          subscription.needsRehydration = true;
+        } else if (exists) {
+          // Update existing
+          this.normalizedCache.upsert(doc);
+        }
+      }
+
+      // Update checkpoint
+      subscription.checkpoint = calculateCheckpointFromDocs(response.rows);
+    } catch (error) {
+      console.error("Incremental backfill failed:", error);
+      // Fall back to full refresh
+      await this.fullRefresh(subscription);
+    }
+  }
+
+  private async pageBackfill(subscription: VirtualSubscription): Promise<void> {
+    const pageSize = subscription.viewWindow.pageSize;
+    const currentPage = subscription.viewWindow.currentPage;
+    const start = (currentPage - 1) * pageSize;
+
+    // Fetch current page from server
+    const response = await this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      limitStart: start,
+      limit: pageSize,
+    });
+
+    // Replace page in cache
+    const newDocIds = response.map((doc) => doc.name);
+    const currentDocIds = subscription.getProgressiveCache().docIds;
+    const updatedDocIds = [...currentDocIds];
+    updatedDocIds.splice(start, pageSize, ...newDocIds);
+    subscription.updateProgressiveCache({ docIds: updatedDocIds });
+
+    // Update normalized cache
+    for (const doc of response) {
+      this.normalizedCache.upsert(doc);
+    }
+
+    // Clear flags
+    subscription.needsRehydration = false;
+  }
+
+  private async gapBackfill(subscription: VirtualSubscription): Promise<void> {
+    // Fill one gap
+    await this.backfillOne(subscription);
+
+    // Clear flag
+    subscription.pendingBackfill = false;
+  }
+
+  private async cursorFetch(subscription: VirtualSubscription): Promise<void> {
+    if (!subscription.getProgressiveCache().hasMore) return;
+    const batchSize = this.options.batchSize || this.options.pageSize || 100;
+    const docs = await this.fetchWithCursor(subscription, batchSize);
+
+    // Update caches using subscription methods
+    subscription.updateProgressiveCache({
+      docIds: [...subscription.getProgressiveCache().docIds, ...docs.map((doc) => doc.name)],
+      nextCursor:
+        docs.length > 0
+          ? docs[docs.length - 1].name
+          : subscription.getProgressiveCache().nextCursor,
+      hasMore: docs.length === batchSize,
+    });
+
+    // Update normalized cache only (skip checkpoint calculation for batch updates)
+    for (const doc of docs) {
+      this.normalizedCache.upsert(doc);
+    }
+  }
+
+  private async fetchWithCursor(
+    subscription: VirtualSubscription,
+    limit: number,
+  ): Promise<FrappeDoc<FrappeDocData>[]> {
+    return this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      cursor: subscription.getProgressiveCache().nextCursor,
+      limit,
+    });
+  }
+
+  private queueBackfill(subscriptionId: string, reason: BackfillReason): void {
+    if (!this.backfillQueue.has(subscriptionId)) {
+      this.backfillQueue.set(subscriptionId, []);
+    }
+    const queue = this.backfillQueue.get(subscriptionId)!;
+    if (!queue.includes(reason)) queue.push(reason);
+  }
+}

package/src/sync/debug.spec.ts

@@ -0,0 +1,110 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { DbModule } from "../db";
+import * as filters from "./filters";
+import { SyncModule } from "./index";
+
+function createMockDb() {
+  return {
+    getDocList: vi.fn().mockResolvedValue([
+      {
+        name: "todo-1",
+        doctype: "ToDo",
+        modified: "2026-01-01 00:00:00",
+        status: "Open",
+      },
+      {
+        name: "todo-2",
+        doctype: "ToDo",
+        modified: "2026-01-01 00:00:00",
+        status: "Open",
+      },
+    ]),
+    getDoc: vi.fn().mockResolvedValue({
+      name: "todo-1",
+      doctype: "ToDo",
+      modified: "2026-01-01",
+      status: "Open",
+    }),
+    backfill: vi.fn().mockResolvedValue({ rows: [], checkpoint: null }),
+    getCount: vi.fn().mockResolvedValue(2),
+    createDoc: vi.fn().mockResolvedValue({ name: "todo-new", doctype: "ToDo" }),
+    updateDoc: vi.fn().mockResolvedValue(undefined),
+    deleteDoc: vi.fn().mockResolvedValue(undefined),
+  } as unknown as DbModule;
+}
+
+describe("SyncModule.__debug()", () => {
+  let syncModule: SyncModule;
+  let mockDb: ReturnType<typeof createMockDb>;
+
+  beforeEach(() => {
+    vi.spyOn(filters, "matchesFilters").mockReturnValue(true);
+    vi.spyOn(filters, "extractSingleName").mockReturnValue(undefined);
+    mockDb = createMockDb();
+    Object.defineProperty(global, "crypto", {
+      value: { randomUUID: vi.fn().mockReturnValue("debug-test-uuid") },
+      writable: true,
+    });
+    syncModule = new SyncModule(mockDb, undefined, {});
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it("returns subscriptions with correct doc counts", async () => {
+    const { subscriptionId } = await syncModule.subscribe({
+      doctype: "ToDo",
+      fields: ["name", "status"],
+      limit: 20,
+    });
+
+    const debug = syncModule.__debug();
+
+    expect(debug.subscriptions).toHaveLength(1);
+    expect(debug.subscriptions[0].id).toBe(subscriptionId);
+    expect(debug.subscriptions[0].docCount).toBe(2);
+    expect(debug.subscriptions[0].tombstoneCount).toBe(0);
+    expect(debug.subscriptions[0].query.doctype).toBe("ToDo");
+  });
+
+  it("normalized cache stats match after subscribing", async () => {
+    await syncModule.subscribe({
+      doctype: "ToDo",
+      fields: ["name", "status"],
+      limit: 20,
+    });
+
+    const debug = syncModule.__debug();
+
+    expect(debug.normalizedCache.size).toBe(2);
+    expect(debug.normalizedCache.doctypes).toContainEqual({
+      doctype: "ToDo",
+      count: 2,
+    });
+  });
+
+  it("connection state reports correctly", () => {
+    const debug = syncModule.__debug();
+
+    expect(debug.connection).toBeDefined();
+    expect(typeof debug.connection.isOnline).toBe("boolean");
+    expect(typeof debug.connection.isLeader).toBe("boolean");
+    expect(typeof debug.connection.socketConnected).toBe("boolean");
+  });
+
+  it("mutation journal has correct structure", async () => {
+    await syncModule.subscribe({
+      doctype: "ToDo",
+      fields: ["name", "status"],
+      limit: 20,
+    });
+
+    const debug = syncModule.__debug();
+
+    expect(debug.mutationJournal).toBeDefined();
+    expect(Array.isArray(debug.mutationJournal.pending)).toBe(true);
+    expect(Array.isArray(debug.mutationJournal.confirmed)).toBe(true);
+    expect(Array.isArray(debug.mutationJournal.failed)).toBe(true);
+  });
+});