@multiplatform.one/frappe 6.0.0

package/src/sync/subscriptionManager.ts

@@ -0,0 +1,577 @@
+import type { FrappeDoc, FrappeDocData } from "../common/types";
+import type { ListOptions } from "../core/types";
+import type { DbModule } from "../db";
+import { BackfillManager } from "./backfill";
+import { extractSingleName, matchesFilters } from "./filters";
+import type { NormalizedCache } from "./normalizedCache";
+import { BackfillScope, Order } from "./types";
+import type {
+  BackfillArgs,
+  OrderByField,
+  Subscription,
+  SubscriptionQuery,
+  SyncOptions,
+} from "./types";
+import { VirtualPageManager } from "./virtualPageManager";
+
+export class SubscriptionManager {
+  private subscriptions = new Map<string, Subscription>();
+  private virtualPageManager: VirtualPageManager;
+  private backfillManager: BackfillManager;
+  /** Maps a stable query key to an existing subscriptionId for dedup */
+  private queryKeyToSubId = new Map<string, string>();
+  /** Reference counts per subscriptionId for shared subscription lifecycle */
+  private subRefCounts = new Map<string, number>();
+
+  constructor(
+    private readonly db: DbModule,
+    private readonly normalizedCache: NormalizedCache,
+    private readonly options: SyncOptions = {},
+  ) {
+    this.virtualPageManager = new VirtualPageManager(this.normalizedCache, this.db, {
+      batchSize: this.options.batchSize,
+    });
+    this.backfillManager = new BackfillManager(this.db, this.normalizedCache, {
+      pageSize: this.options.pageSize,
+    });
+  }
+
+  async subscribe<Data extends FrappeDocData>(query: SubscriptionQuery) {
+    // Dedup: if an identical query is already active, reuse it
+    const queryKey = JSON.stringify(query);
+    const existingId = this.queryKeyToSubId.get(queryKey);
+    if (existingId && this.subscriptions.has(existingId)) {
+      this.subRefCounts.set(existingId, (this.subRefCounts.get(existingId) || 1) + 1);
+      const rows = this.getDocsForSubscription<Data>(existingId);
+      const checkpoint = this.subscriptions.get(existingId)!.checkpoint;
+      return { subscriptionId: existingId, rows, checkpoint };
+    }
+
+    const subscriptionId = cryptoRandomId();
+    // Only restrict fields when the caller explicitly specifies them.
+    // When fields is undefined/empty, pass undefined so all fields are returned.
+    const queryFields = query.fields?.length
+      ? Array.from(new Set([...query.fields, "modified", "name"]))
+      : undefined;
+    let rows: FrappeDoc<Data>[] = [];
+    let cursor: string | undefined;
+    const singleName = extractSingleName(query.filters);
+    if (singleName) {
+      try {
+        const document = await this.db.getDoc<Data>(query.doctype, singleName);
+        rows = document ? [document] : [];
+      } catch (error) {
+        // Check if this is a 404 (document not found) vs other errors
+        const errorWithStatus = error as { status?: number };
+        if (errorWithStatus?.status === 404) {
+          // Document not found - return empty rows
+          rows = [];
+        } else {
+          // Re-throw other errors (network issues, permission errors, etc.)
+          throw error;
+        }
+      }
+    } else {
+      const listOptions: ListOptions = {
+        fields: queryFields,
+        filters: query.filters,
+        orderBy: query.orderBy,
+      };
+      if (this.options.useCursorPagination) listOptions.cursor = undefined;
+      const response = await this.db.getDocList<Data>(query.doctype, listOptions);
+
+      // Handle response based on whether it includes cursor
+      if (typeof response === "object" && response !== null && "data" in response) {
+        rows = (response as any).data as FrappeDoc<Data>[];
+        cursor = (response as any).cursor;
+      } else {
+        rows = response;
+      }
+    }
+    // Seed the normalized cache with every returned row for instant local reads.
+    // Frappe's standard get_list API omits `doctype` from each row, so inject
+    // it from the subscription query to ensure the cache keys correctly.
+    for (const row of rows) {
+      if (!row.doctype) row.doctype = query.doctype;
+      this.normalizedCache.upsert(row);
+    }
+    // Compute checkpoint for CDC as the max `modified` across returned rows (robust to server ordering)
+    let checkpoint: string | undefined = undefined;
+    for (const row of rows) {
+      if (!row.modified) continue;
+      if (!checkpoint || row.modified > checkpoint) checkpoint = row.modified;
+    }
+    // Materialize the in-memory subscription record with initial membership
+    const subscription: Subscription = {
+      id: subscriptionId,
+      query,
+      checkpoint, // For CDC backfill
+      cursor, // For stable pagination
+      docIds: rows.map((row) => row.name),
+      tombstones: new Map<string, number>(),
+    };
+
+    // Track page boundaries for optimistic placement
+    if (rows.length > 0) {
+      subscription.boundaries = {
+        firstItem: rows[0] as unknown as FrappeDoc<Data>,
+        lastItem: rows[rows.length - 1] as unknown as FrappeDoc<Data>,
+        offset: query.limitStart || 0,
+        pageSize: query.limit || 20,
+      };
+    }
+
+    // Initialize metadata for tracking items outside current page
+    subscription.metadata = {
+      itemsBeforePage: 0,
+      itemsAfterPage: 0,
+    };
+
+    // Activate it
+    this.subscriptions.set(subscriptionId, subscription);
+    this.queryKeyToSubId.set(queryKey, subscriptionId);
+    this.subRefCounts.set(subscriptionId, 1);
+
+    // Register with VirtualPageManager for progressive pagination if requested
+    if (query.progressive) {
+      this.virtualPageManager.registerSubscription(
+        subscriptionId,
+        subscription,
+        subscription.docIds,
+        cursor,
+      );
+    }
+
+    // Provide the caller with the seeded rows and computed checkpoint
+    return { subscriptionId: subscriptionId, rows, checkpoint };
+  }
+
+  /**
+   * Removes a subscription by id. Cached docs remain until evicted by
+   * realtime/backfill or explicit resets. This keeps other subs fast if they
+   * share the same doctypes.
+   */
+  unsubscribe(subscriptionId: string) {
+    const refCount = (this.subRefCounts.get(subscriptionId) || 1) - 1;
+    if (refCount > 0) {
+      this.subRefCounts.set(subscriptionId, refCount);
+      return;
+    }
+    // Last consumer released — clean up
+    this.subRefCounts.delete(subscriptionId);
+    for (const [key, id] of this.queryKeyToSubId) {
+      if (id === subscriptionId) {
+        this.queryKeyToSubId.delete(key);
+        break;
+      }
+    }
+    this.subscriptions.delete(subscriptionId);
+  }
+
+  /** Returns subscription metadata by id. */
+  getSubscription(id: string) {
+    // Retrieve an active subscription record by its identifier
+    return this.subscriptions.get(id);
+  }
+
+  /**
+   * Returns current rows for a subscription from the cache.
+   * Note: This is local-only; authoritative refresh happens via backfill/realtime.
+   */
+  getDocsForSubscription<Data extends FrappeDocData>(id: string): FrappeDoc<Data>[] {
+    // Try to get from virtual page system first
+    const virtualSub = this.virtualPageManager.getSubscription(id);
+    if (virtualSub) {
+      // Use virtual page system for progressive cache
+      const docIds = virtualSub.getProgressiveCache().docIds.filter((id: string) => id !== "");
+      const out: FrappeDoc<Data>[] = [];
+      for (const name of docIds) {
+        const document = this.normalizedCache.get({
+          doctype: virtualSub.query.doctype,
+          name,
+        });
+        if (document) out.push(document as FrappeDoc<Data>);
+      }
+      return out;
+    }
+
+    // Fall back to standard subscription lookup
+    const subscription = this.subscriptions.get(id);
+    if (!subscription) return [];
+    const out: FrappeDoc<Data>[] = [];
+    for (const name of subscription.docIds) {
+      const document = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name,
+      });
+      if (document) out.push(document as FrappeDoc<Data>);
+    }
+    return out;
+  }
+
+  /**
+   * Get a specific page of documents using virtual pagination
+   */
+  async getPage<Data extends FrappeDocData>(
+    subscriptionId: string,
+    page: number,
+  ): Promise<FrappeDoc<Data>[]> {
+    return this.virtualPageManager.getPage(subscriptionId, page) as Promise<FrappeDoc<Data>[]>;
+  }
+
+  /**
+   * Get documents after a cursor (for infinite scroll)
+   */
+  async getNextBatch<Data extends FrappeDocData>(
+    subscriptionId: string,
+    cursorDocId: string,
+    limit: number,
+  ): Promise<FrappeDoc<Data>[]> {
+    return this.virtualPageManager.getNextBatch(subscriptionId, cursorDocId, limit) as Promise<
+      FrappeDoc<Data>[]
+    >;
+  }
+
+  /**
+   * Get page information
+   */
+  getPageInfo(subscriptionId: string) {
+    return this.virtualPageManager.getPageInfo(subscriptionId);
+  }
+
+  /**
+   * Reconciles local state by fetching changes since stored cursors.
+   *
+   * Scopes
+   * - all: iterate all subscriptions
+   * - subscriptions: selected subscription ids
+   * - doctypes: all subscriptions matching any of these doctypes
+   * - docs: refetch specific documents (authority refresh)
+   */
+  async backfill<Data extends FrappeDocData>(args: BackfillArgs) {
+    if (args.scope === BackfillScope.All) {
+      for (const subscription of this.subscriptions.values())
+        await this.backfillSub(subscription, args.pageSize);
+      return;
+    }
+    if (args.scope === BackfillScope.Subscriptions) {
+      for (const subscriptionId of args.subscriptions || []) {
+        const subscription = this.subscriptions.get(subscriptionId);
+        if (subscription) await this.backfillSub(subscription, args.pageSize);
+      }
+      return;
+    }
+    if (args.scope === BackfillScope.Doctypes) {
+      const set = new Set(args.doctypes || []);
+      for (const subscription of this.subscriptions.values())
+        if (set.has(subscription.query.doctype))
+          await this.backfillSub(subscription, args.pageSize);
+      return;
+    }
+    if (args.scope === BackfillScope.Docs) {
+      for (const docTarget of args.docs || []) {
+        const document = await this.db.getDoc<Data>(docTarget.doctype, docTarget.name);
+        this.normalizedCache.upsert(document);
+      }
+    }
+  }
+
+  /**
+   * Backfills one subscription using CDC checkpoint and updates membership.
+   *
+   * Algorithm
+   * - Query backfill using checkpoint for CDC (Change Data Capture)
+   * - Upsert each row to cache
+   * - For each row: test client-side filters; add/remove membership accordingly
+   * - Track tombstones for deleted documents
+   * - Advance checkpoint to response checkpoint
+   * - If no filters were specified, derive membership directly from returned rows
+   */
+  async backfillSub<Data extends FrappeDocData>(
+    subscription: Subscription,
+    pageSize = this.options.pageSize ?? 200,
+  ) {
+    // Request changes since the last known checkpoint using CDC backfill API
+    const response = await this.db.backfill<Data>({
+      doctype: subscription.query.doctype,
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      modifiedAfter: subscription.checkpoint, // CDC checkpoint
+      limit: pageSize,
+    });
+    // Apply authoritative deletes first if backend returns tombstones.
+    // The backfill endpoint returns {name: string} objects (no doctype),
+    // so we fall back to the subscription's doctype when not provided.
+    if ((response as any).deleted && Array.isArray((response as any).deleted)) {
+      for (const deleted of (response as any).deleted as {
+        doctype?: string;
+        name: string;
+      }[]) {
+        if (!deleted?.name) continue;
+        const deletedDoctype = deleted.doctype || subscription.query.doctype;
+
+        // Track in tombstones map for this subscription with timestamp
+        if (deletedDoctype === subscription.query.doctype && subscription.tombstones) {
+          subscription.tombstones.set(deleted.name, Date.now());
+        }
+        // Evict from cache and any list memberships for this doctype
+        this.normalizedCache.remove({
+          doctype: deletedDoctype,
+          name: deleted.name,
+        });
+        if (deletedDoctype === subscription.query.doctype)
+          subscription.docIds = subscription.docIds.filter(
+            (currentName) => currentName !== deleted.name,
+          );
+      }
+    }
+    // Upsert each changed/inserted row, then reconcile membership using client filters
+    for (const row of response.rows) {
+      if (!row.doctype) row.doctype = subscription.query.doctype;
+      this.normalizedCache.upsert(row);
+      const name = row.name;
+      if (!name) continue;
+      const match = matchesFilters(row as FrappeDoc<Data>, subscription.query.filters);
+      if (match && !subscription.docIds.includes(name)) subscription.docIds.push(name);
+      if (!match)
+        subscription.docIds = subscription.docIds.filter((currentName) => currentName !== name);
+    }
+    // Advance the checkpoint to the server-provided checkpoint for next cycle
+    if (response.checkpoint) subscription.checkpoint = response.checkpoint;
+    // If no filters, the rows represent the full page; maintain membership directly
+    if (response.rows.length && !subscription.query.filters)
+      subscription.docIds = response.rows.map((row: any) => row.name);
+
+    // Update page boundaries after backfill
+    if (subscription.docIds.length > 0) {
+      const firstDoc = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name: subscription.docIds[0],
+      });
+      const lastDoc = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name: subscription.docIds[subscription.docIds.length - 1],
+      });
+
+      if (firstDoc && lastDoc) {
+        subscription.boundaries = {
+          firstItem: firstDoc,
+          lastItem: lastDoc,
+          offset: subscription.boundaries?.offset || 0,
+          pageSize: subscription.boundaries?.pageSize || subscription.query.limit || 20,
+        };
+      }
+    }
+
+    // Maintain ordering when an orderBy is present for stable UX
+    this.resortSubscriptionDocIds(subscription);
+
+    // Cleanup expired tombstones to prevent memory leaks
+    this.cleanupTombstones(subscription.id);
+
+    if (this.options.onBackfill)
+      this.options.onBackfill({
+        doctype: subscription.query.doctype,
+        fetched: response.rows.length,
+        checkpoint: subscription.checkpoint,
+      });
+  }
+
+  resortSubscriptionDocIds(subscription: Subscription) {
+    const { orderBy, doctype } = subscription.query;
+    if (!orderBy) return;
+    const orderByArray = Array.isArray(orderBy) ? orderBy : [orderBy];
+    subscription.docIds.sort((firstId, secondId) => {
+      const firstDoc = this.normalizedCache.get({ doctype, name: firstId });
+      const secondDoc = this.normalizedCache.get({ doctype, name: secondId });
+      for (const orderByField of orderByArray) {
+        const field = orderByField.field;
+        const ascending = orderByField.order !== Order.Desc;
+        const firstValue = firstDoc?.[field];
+        const secondValue = secondDoc?.[field];
+        const firstIsNullish = firstValue === null || firstValue === undefined;
+        const secondIsNullish = secondValue === null || secondValue === undefined;
+        if (firstIsNullish && !secondIsNullish) return ascending ? -1 : 1;
+        if (secondIsNullish && !firstIsNullish) return ascending ? 1 : -1;
+        if (firstIsNullish && secondIsNullish) continue;
+        if (firstIsNullish || secondIsNullish) continue;
+        if (firstValue < secondValue) return ascending ? -1 : 1;
+        if (firstValue > secondValue) return ascending ? 1 : -1;
+      }
+      return firstId < secondId ? -1 : firstId > secondId ? 1 : 0;
+    });
+  }
+
+  updateSubscriptionMembership<Data extends FrappeDocData>(doc: FrappeDoc<Data>): Subscription[] {
+    const subscriptionsNeedingRehydration: Subscription[] = [];
+    for (const subscription of this.subscriptions.values()) {
+      if (subscription.query.doctype !== doc.doctype) continue;
+      const match = matchesFilters(doc, subscription.query.filters);
+      const isExisting = subscription.docIds.includes(doc.name);
+      if (match && !isExisting) {
+        subscriptionsNeedingRehydration.push(subscription);
+        // Also update VPM progressive cache for remote creates
+        this.virtualPageManager.handleRemoteCreate(
+          subscription.id,
+          doc as FrappeDoc<FrappeDocData>,
+        );
+      } else if (isExisting) {
+        if (!match) {
+          subscription.docIds = subscription.docIds.filter((docId) => docId !== doc.name);
+          this.virtualPageManager.handleRemoteDelete(subscription.id, doc.name);
+        } else {
+          this.virtualPageManager.handleRemoteUpdate(
+            subscription.id,
+            doc as FrappeDoc<FrappeDocData>,
+          );
+        }
+        this.resortSubscriptionDocIds(subscription);
+      }
+    }
+    return subscriptionsNeedingRehydration;
+  }
+
+  /**
+   * Removes a document from all subscription memberships.
+   * Returns subscriptions that had the document removed (for potential backfill).
+   */
+  removeFromAllSubscriptions<_Data extends FrappeDocData>(
+    doctype: string,
+    name: string,
+  ): Subscription[] {
+    const affectedSubscriptions: Subscription[] = [];
+
+    for (const subscription of this.subscriptions.values()) {
+      if (subscription.query.doctype !== doctype) continue;
+
+      const hadDocument = subscription.docIds.includes(name as string);
+      if (hadDocument) {
+        subscription.docIds = subscription.docIds.filter((docId) => docId !== name);
+        this.virtualPageManager.handleRemoteDelete(subscription.id, name);
+        affectedSubscriptions.push(subscription);
+      }
+    }
+
+    return affectedSubscriptions;
+  }
+
+  /** Gets all active subscriptions. */
+  getAllSubscriptions(): Map<string, Subscription> {
+    return this.subscriptions;
+  }
+
+  getVirtualPageManager(): VirtualPageManager {
+    return this.virtualPageManager;
+  }
+
+  insertDocumentIntoSubscription<Data extends FrappeDocData>(
+    subscription: Subscription,
+    doc: FrappeDoc<Data>,
+  ) {
+    if (subscription.docIds.includes(doc.name)) return;
+    if (!subscription.query.orderBy || subscription.docIds.length === 0) {
+      return subscription.docIds.push(doc.name);
+    }
+    const insertIndex = this.findInsertionIndex(subscription, doc);
+    subscription.docIds.splice(insertIndex, 0, doc.name);
+  }
+
+  private findInsertionIndex<Data extends FrappeDocData>(
+    subscription: Subscription,
+    doc: FrappeDoc<Data>,
+  ): number {
+    const { docIds, query } = subscription;
+    const orderByArray = Array.isArray(query.orderBy) ? query.orderBy : [query.orderBy!];
+    let left = 0;
+    let right = docIds.length - 1;
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      const midDoc = this.normalizedCache.get({
+        doctype: query.doctype,
+        name: docIds[mid],
+      });
+      if (!midDoc) {
+        return docIds.length;
+      }
+      const comparison = this.compareDocuments(doc, midDoc, orderByArray);
+      if (comparison <= 0) {
+        right = mid - 1;
+      } else {
+        left = mid + 1;
+      }
+    }
+    return left;
+  }
+
+  private compareDocuments<Data extends FrappeDocData>(
+    doc1: FrappeDoc<Data>,
+    doc2: FrappeDoc<Data>,
+    orderByArray: OrderByField[],
+  ): number {
+    for (const { field, order } of orderByArray) {
+      const value1 = doc1[field];
+      const value2 = doc2[field];
+      const isAsc = order !== Order.Desc;
+      const isNull1 = value1 == null;
+      const isNull2 = value2 == null;
+      if (isNull1 && !isNull2) return isAsc ? -1 : 1;
+      if (!isNull1 && isNull2) return isAsc ? 1 : -1;
+      if (isNull1 && isNull2) continue;
+      if (value1! < value2!) return isAsc ? -1 : 1;
+      if (value1! > value2!) return isAsc ? 1 : -1;
+    }
+    const name1 = doc1.name || "";
+    const name2 = doc2.name || "";
+    return name1 < name2 ? -1 : name1 > name2 ? 1 : 0;
+  }
+
+  clear() {
+    this.subscriptions.clear();
+    this.queryKeyToSubId.clear();
+    this.subRefCounts.clear();
+  }
+
+  /**
+   * Cleans up expired tombstones based on age and count limits.
+   * Should be called periodically or after backfill operations.
+   */
+  cleanupTombstones(subscriptionId?: string) {
+    const maxAge = this.options.tombstoneMaxAge ?? 30 * 60 * 1000; // Default 30 minutes
+    const maxCount = this.options.tombstoneMaxCount ?? 1000; // Default 1000 per subscription
+    const now = Date.now();
+
+    const subscriptionsToClean = subscriptionId
+      ? [this.subscriptions.get(subscriptionId)].filter(Boolean)
+      : [...this.subscriptions.values()];
+
+    for (const subscription of subscriptionsToClean) {
+      if (!subscription?.tombstones || subscription.tombstones.size === 0) continue;
+
+      // First pass: remove expired tombstones by age
+      for (const [name, timestamp] of subscription.tombstones) {
+        if (now - timestamp > maxAge) {
+          subscription.tombstones.delete(name);
+        }
+      }
+
+      // Second pass: if still over max count, remove oldest tombstones
+      if (subscription.tombstones.size > maxCount) {
+        const entries = [...subscription.tombstones.entries()];
+        // Sort by timestamp ascending (oldest first)
+        entries.sort((a, b) => a[1] - b[1]);
+        // Remove oldest entries to get under limit
+        const toRemove = entries.slice(0, subscription.tombstones.size - maxCount);
+        for (const [name] of toRemove) {
+          subscription.tombstones.delete(name);
+        }
+      }
+    }
+  }
+}
+
+function cryptoRandomId() {
+  if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+    return crypto.randomUUID();
+  }
+  return Math.random().toString(36).slice(2);
+}