@multiplatform.one/frappe 6.0.0

package/dist/cjs/sync/subscriptionManager.cjs

@@ -0,0 +1,487 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all) __defProp(target, name, {
+    get: all[name],
+    enumerable: true
+  });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from)) if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    });
+  }
+  return to;
+};
+var __toCommonJS = mod => __copyProps(__defProp({}, "__esModule", {
+  value: true
+}), mod);
+var subscriptionManager_exports = {};
+__export(subscriptionManager_exports, {
+  SubscriptionManager: () => SubscriptionManager
+});
+module.exports = __toCommonJS(subscriptionManager_exports);
+var import_backfill = require("./backfill.cjs");
+var import_filters = require("./filters.cjs");
+var import_types = require("./types.cjs");
+var import_virtualPageManager = require("./virtualPageManager.cjs");
+class SubscriptionManager {
+  constructor(db, normalizedCache, options = {}) {
+    this.db = db;
+    this.normalizedCache = normalizedCache;
+    this.options = options;
+    this.virtualPageManager = new import_virtualPageManager.VirtualPageManager(this.normalizedCache, this.db, {
+      batchSize: this.options.batchSize
+    });
+    this.backfillManager = new import_backfill.BackfillManager(this.db, this.normalizedCache, {
+      pageSize: this.options.pageSize
+    });
+  }
+  db;
+  normalizedCache;
+  options;
+  subscriptions = /* @__PURE__ */new Map();
+  virtualPageManager;
+  backfillManager;
+  /** Maps a stable query key to an existing subscriptionId for dedup */
+  queryKeyToSubId = /* @__PURE__ */new Map();
+  /** Reference counts per subscriptionId for shared subscription lifecycle */
+  subRefCounts = /* @__PURE__ */new Map();
+  async subscribe(query) {
+    const queryKey = JSON.stringify(query);
+    const existingId = this.queryKeyToSubId.get(queryKey);
+    if (existingId && this.subscriptions.has(existingId)) {
+      this.subRefCounts.set(existingId, (this.subRefCounts.get(existingId) || 1) + 1);
+      const rows2 = this.getDocsForSubscription(existingId);
+      const checkpoint2 = this.subscriptions.get(existingId).checkpoint;
+      return {
+        subscriptionId: existingId,
+        rows: rows2,
+        checkpoint: checkpoint2
+      };
+    }
+    const subscriptionId = cryptoRandomId();
+    const queryFields = query.fields?.length ? Array.from(/* @__PURE__ */new Set([...query.fields, "modified", "name"])) : void 0;
+    let rows = [];
+    let cursor;
+    const singleName = (0, import_filters.extractSingleName)(query.filters);
+    if (singleName) {
+      try {
+        const document = await this.db.getDoc(query.doctype, singleName);
+        rows = document ? [document] : [];
+      } catch (error) {
+        const errorWithStatus = error;
+        if (errorWithStatus?.status === 404) {
+          rows = [];
+        } else {
+          throw error;
+        }
+      }
+    } else {
+      const listOptions = {
+        fields: queryFields,
+        filters: query.filters,
+        orderBy: query.orderBy
+      };
+      if (this.options.useCursorPagination) listOptions.cursor = void 0;
+      const response = await this.db.getDocList(query.doctype, listOptions);
+      if (typeof response === "object" && response !== null && "data" in response) {
+        rows = response.data;
+        cursor = response.cursor;
+      } else {
+        rows = response;
+      }
+    }
+    for (const row of rows) {
+      if (!row.doctype) row.doctype = query.doctype;
+      this.normalizedCache.upsert(row);
+    }
+    let checkpoint = void 0;
+    for (const row of rows) {
+      if (!row.modified) continue;
+      if (!checkpoint || row.modified > checkpoint) checkpoint = row.modified;
+    }
+    const subscription = {
+      id: subscriptionId,
+      query,
+      checkpoint,
+      // For CDC backfill
+      cursor,
+      // For stable pagination
+      docIds: rows.map(row => row.name),
+      tombstones: /* @__PURE__ */new Map()
+    };
+    if (rows.length > 0) {
+      subscription.boundaries = {
+        firstItem: rows[0],
+        lastItem: rows[rows.length - 1],
+        offset: query.limitStart || 0,
+        pageSize: query.limit || 20
+      };
+    }
+    subscription.metadata = {
+      itemsBeforePage: 0,
+      itemsAfterPage: 0
+    };
+    this.subscriptions.set(subscriptionId, subscription);
+    this.queryKeyToSubId.set(queryKey, subscriptionId);
+    this.subRefCounts.set(subscriptionId, 1);
+    if (query.progressive) {
+      this.virtualPageManager.registerSubscription(subscriptionId, subscription, subscription.docIds, cursor);
+    }
+    return {
+      subscriptionId,
+      rows,
+      checkpoint
+    };
+  }
+  /**
+   * Removes a subscription by id. Cached docs remain until evicted by
+   * realtime/backfill or explicit resets. This keeps other subs fast if they
+   * share the same doctypes.
+   */
+  unsubscribe(subscriptionId) {
+    const refCount = (this.subRefCounts.get(subscriptionId) || 1) - 1;
+    if (refCount > 0) {
+      this.subRefCounts.set(subscriptionId, refCount);
+      return;
+    }
+    this.subRefCounts.delete(subscriptionId);
+    for (const [key, id] of this.queryKeyToSubId) {
+      if (id === subscriptionId) {
+        this.queryKeyToSubId.delete(key);
+        break;
+      }
+    }
+    this.subscriptions.delete(subscriptionId);
+  }
+  /** Returns subscription metadata by id. */
+  getSubscription(id) {
+    return this.subscriptions.get(id);
+  }
+  /**
+   * Returns current rows for a subscription from the cache.
+   * Note: This is local-only; authoritative refresh happens via backfill/realtime.
+   */
+  getDocsForSubscription(id) {
+    const virtualSub = this.virtualPageManager.getSubscription(id);
+    if (virtualSub) {
+      const docIds = virtualSub.getProgressiveCache().docIds.filter(id2 => id2 !== "");
+      const out2 = [];
+      for (const name of docIds) {
+        const document = this.normalizedCache.get({
+          doctype: virtualSub.query.doctype,
+          name
+        });
+        if (document) out2.push(document);
+      }
+      return out2;
+    }
+    const subscription = this.subscriptions.get(id);
+    if (!subscription) return [];
+    const out = [];
+    for (const name of subscription.docIds) {
+      const document = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name
+      });
+      if (document) out.push(document);
+    }
+    return out;
+  }
+  /**
+   * Get a specific page of documents using virtual pagination
+   */
+  async getPage(subscriptionId, page) {
+    return this.virtualPageManager.getPage(subscriptionId, page);
+  }
+  /**
+   * Get documents after a cursor (for infinite scroll)
+   */
+  async getNextBatch(subscriptionId, cursorDocId, limit) {
+    return this.virtualPageManager.getNextBatch(subscriptionId, cursorDocId, limit);
+  }
+  /**
+   * Get page information
+   */
+  getPageInfo(subscriptionId) {
+    return this.virtualPageManager.getPageInfo(subscriptionId);
+  }
+  /**
+   * Reconciles local state by fetching changes since stored cursors.
+   *
+   * Scopes
+   * - all: iterate all subscriptions
+   * - subscriptions: selected subscription ids
+   * - doctypes: all subscriptions matching any of these doctypes
+   * - docs: refetch specific documents (authority refresh)
+   */
+  async backfill(args) {
+    if (args.scope === import_types.BackfillScope.All) {
+      for (const subscription of this.subscriptions.values()) await this.backfillSub(subscription, args.pageSize);
+      return;
+    }
+    if (args.scope === import_types.BackfillScope.Subscriptions) {
+      for (const subscriptionId of args.subscriptions || []) {
+        const subscription = this.subscriptions.get(subscriptionId);
+        if (subscription) await this.backfillSub(subscription, args.pageSize);
+      }
+      return;
+    }
+    if (args.scope === import_types.BackfillScope.Doctypes) {
+      const set = new Set(args.doctypes || []);
+      for (const subscription of this.subscriptions.values()) if (set.has(subscription.query.doctype)) await this.backfillSub(subscription, args.pageSize);
+      return;
+    }
+    if (args.scope === import_types.BackfillScope.Docs) {
+      for (const docTarget of args.docs || []) {
+        const document = await this.db.getDoc(docTarget.doctype, docTarget.name);
+        this.normalizedCache.upsert(document);
+      }
+    }
+  }
+  /**
+   * Backfills one subscription using CDC checkpoint and updates membership.
+   *
+   * Algorithm
+   * - Query backfill using checkpoint for CDC (Change Data Capture)
+   * - Upsert each row to cache
+   * - For each row: test client-side filters; add/remove membership accordingly
+   * - Track tombstones for deleted documents
+   * - Advance checkpoint to response checkpoint
+   * - If no filters were specified, derive membership directly from returned rows
+   */
+  async backfillSub(subscription, pageSize = this.options.pageSize ?? 200) {
+    const response = await this.db.backfill({
+      doctype: subscription.query.doctype,
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      modifiedAfter: subscription.checkpoint,
+      // CDC checkpoint
+      limit: pageSize
+    });
+    if (response.deleted && Array.isArray(response.deleted)) {
+      for (const deleted of response.deleted) {
+        if (!deleted?.name) continue;
+        const deletedDoctype = deleted.doctype || subscription.query.doctype;
+        if (deletedDoctype === subscription.query.doctype && subscription.tombstones) {
+          subscription.tombstones.set(deleted.name, Date.now());
+        }
+        this.normalizedCache.remove({
+          doctype: deletedDoctype,
+          name: deleted.name
+        });
+        if (deletedDoctype === subscription.query.doctype) subscription.docIds = subscription.docIds.filter(currentName => currentName !== deleted.name);
+      }
+    }
+    for (const row of response.rows) {
+      if (!row.doctype) row.doctype = subscription.query.doctype;
+      this.normalizedCache.upsert(row);
+      const name = row.name;
+      if (!name) continue;
+      const match = (0, import_filters.matchesFilters)(row, subscription.query.filters);
+      if (match && !subscription.docIds.includes(name)) subscription.docIds.push(name);
+      if (!match) subscription.docIds = subscription.docIds.filter(currentName => currentName !== name);
+    }
+    if (response.checkpoint) subscription.checkpoint = response.checkpoint;
+    if (response.rows.length && !subscription.query.filters) subscription.docIds = response.rows.map(row => row.name);
+    if (subscription.docIds.length > 0) {
+      const firstDoc = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name: subscription.docIds[0]
+      });
+      const lastDoc = this.normalizedCache.get({
+        doctype: subscription.query.doctype,
+        name: subscription.docIds[subscription.docIds.length - 1]
+      });
+      if (firstDoc && lastDoc) {
+        subscription.boundaries = {
+          firstItem: firstDoc,
+          lastItem: lastDoc,
+          offset: subscription.boundaries?.offset || 0,
+          pageSize: subscription.boundaries?.pageSize || subscription.query.limit || 20
+        };
+      }
+    }
+    this.resortSubscriptionDocIds(subscription);
+    this.cleanupTombstones(subscription.id);
+    if (this.options.onBackfill) this.options.onBackfill({
+      doctype: subscription.query.doctype,
+      fetched: response.rows.length,
+      checkpoint: subscription.checkpoint
+    });
+  }
+  resortSubscriptionDocIds(subscription) {
+    const {
+      orderBy,
+      doctype
+    } = subscription.query;
+    if (!orderBy) return;
+    const orderByArray = Array.isArray(orderBy) ? orderBy : [orderBy];
+    subscription.docIds.sort((firstId, secondId) => {
+      const firstDoc = this.normalizedCache.get({
+        doctype,
+        name: firstId
+      });
+      const secondDoc = this.normalizedCache.get({
+        doctype,
+        name: secondId
+      });
+      for (const orderByField of orderByArray) {
+        const field = orderByField.field;
+        const ascending = orderByField.order !== import_types.Order.Desc;
+        const firstValue = firstDoc?.[field];
+        const secondValue = secondDoc?.[field];
+        const firstIsNullish = firstValue === null || firstValue === void 0;
+        const secondIsNullish = secondValue === null || secondValue === void 0;
+        if (firstIsNullish && !secondIsNullish) return ascending ? -1 : 1;
+        if (secondIsNullish && !firstIsNullish) return ascending ? 1 : -1;
+        if (firstIsNullish && secondIsNullish) continue;
+        if (firstIsNullish || secondIsNullish) continue;
+        if (firstValue < secondValue) return ascending ? -1 : 1;
+        if (firstValue > secondValue) return ascending ? 1 : -1;
+      }
+      return firstId < secondId ? -1 : firstId > secondId ? 1 : 0;
+    });
+  }
+  updateSubscriptionMembership(doc) {
+    const subscriptionsNeedingRehydration = [];
+    for (const subscription of this.subscriptions.values()) {
+      if (subscription.query.doctype !== doc.doctype) continue;
+      const match = (0, import_filters.matchesFilters)(doc, subscription.query.filters);
+      const isExisting = subscription.docIds.includes(doc.name);
+      if (match && !isExisting) {
+        subscriptionsNeedingRehydration.push(subscription);
+        this.virtualPageManager.handleRemoteCreate(subscription.id, doc);
+      } else if (isExisting) {
+        if (!match) {
+          subscription.docIds = subscription.docIds.filter(docId => docId !== doc.name);
+          this.virtualPageManager.handleRemoteDelete(subscription.id, doc.name);
+        } else {
+          this.virtualPageManager.handleRemoteUpdate(subscription.id, doc);
+        }
+        this.resortSubscriptionDocIds(subscription);
+      }
+    }
+    return subscriptionsNeedingRehydration;
+  }
+  /**
+   * Removes a document from all subscription memberships.
+   * Returns subscriptions that had the document removed (for potential backfill).
+   */
+  removeFromAllSubscriptions(doctype, name) {
+    const affectedSubscriptions = [];
+    for (const subscription of this.subscriptions.values()) {
+      if (subscription.query.doctype !== doctype) continue;
+      const hadDocument = subscription.docIds.includes(name);
+      if (hadDocument) {
+        subscription.docIds = subscription.docIds.filter(docId => docId !== name);
+        this.virtualPageManager.handleRemoteDelete(subscription.id, name);
+        affectedSubscriptions.push(subscription);
+      }
+    }
+    return affectedSubscriptions;
+  }
+  /** Gets all active subscriptions. */
+  getAllSubscriptions() {
+    return this.subscriptions;
+  }
+  getVirtualPageManager() {
+    return this.virtualPageManager;
+  }
+  insertDocumentIntoSubscription(subscription, doc) {
+    if (subscription.docIds.includes(doc.name)) return;
+    if (!subscription.query.orderBy || subscription.docIds.length === 0) {
+      return subscription.docIds.push(doc.name);
+    }
+    const insertIndex = this.findInsertionIndex(subscription, doc);
+    subscription.docIds.splice(insertIndex, 0, doc.name);
+  }
+  findInsertionIndex(subscription, doc) {
+    const {
+      docIds,
+      query
+    } = subscription;
+    const orderByArray = Array.isArray(query.orderBy) ? query.orderBy : [query.orderBy];
+    let left = 0;
+    let right = docIds.length - 1;
+    while (left <= right) {
+      const mid = Math.floor((left + right) / 2);
+      const midDoc = this.normalizedCache.get({
+        doctype: query.doctype,
+        name: docIds[mid]
+      });
+      if (!midDoc) {
+        return docIds.length;
+      }
+      const comparison = this.compareDocuments(doc, midDoc, orderByArray);
+      if (comparison <= 0) {
+        right = mid - 1;
+      } else {
+        left = mid + 1;
+      }
+    }
+    return left;
+  }
+  compareDocuments(doc1, doc2, orderByArray) {
+    for (const {
+      field,
+      order
+    } of orderByArray) {
+      const value1 = doc1[field];
+      const value2 = doc2[field];
+      const isAsc = order !== import_types.Order.Desc;
+      const isNull1 = value1 == null;
+      const isNull2 = value2 == null;
+      if (isNull1 && !isNull2) return isAsc ? -1 : 1;
+      if (!isNull1 && isNull2) return isAsc ? 1 : -1;
+      if (isNull1 && isNull2) continue;
+      if (value1 < value2) return isAsc ? -1 : 1;
+      if (value1 > value2) return isAsc ? 1 : -1;
+    }
+    const name1 = doc1.name || "";
+    const name2 = doc2.name || "";
+    return name1 < name2 ? -1 : name1 > name2 ? 1 : 0;
+  }
+  clear() {
+    this.subscriptions.clear();
+    this.queryKeyToSubId.clear();
+    this.subRefCounts.clear();
+  }
+  /**
+   * Cleans up expired tombstones based on age and count limits.
+   * Should be called periodically or after backfill operations.
+   */
+  cleanupTombstones(subscriptionId) {
+    const maxAge = this.options.tombstoneMaxAge ?? 30 * 60 * 1e3;
+    const maxCount = this.options.tombstoneMaxCount ?? 1e3;
+    const now = Date.now();
+    const subscriptionsToClean = subscriptionId ? [this.subscriptions.get(subscriptionId)].filter(Boolean) : [...this.subscriptions.values()];
+    for (const subscription of subscriptionsToClean) {
+      if (!subscription?.tombstones || subscription.tombstones.size === 0) continue;
+      for (const [name, timestamp] of subscription.tombstones) {
+        if (now - timestamp > maxAge) {
+          subscription.tombstones.delete(name);
+        }
+      }
+      if (subscription.tombstones.size > maxCount) {
+        const entries = [...subscription.tombstones.entries()];
+        entries.sort((a, b) => a[1] - b[1]);
+        const toRemove = entries.slice(0, subscription.tombstones.size - maxCount);
+        for (const [name] of toRemove) {
+          subscription.tombstones.delete(name);
+        }
+      }
+    }
+  }
+}
+function cryptoRandomId() {
+  if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+    return crypto.randomUUID();
+  }
+  return Math.random().toString(36).slice(2);
+}