@multiplatform.one/frappe 6.0.0

package/dist/cjs/sync/backfill.cjs

@@ -0,0 +1,314 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all) __defProp(target, name, {
+    get: all[name],
+    enumerable: true
+  });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from)) if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
+      get: () => from[key],
+      enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
+    });
+  }
+  return to;
+};
+var __toCommonJS = mod => __copyProps(__defProp({}, "__esModule", {
+  value: true
+}), mod);
+var backfill_exports = {};
+__export(backfill_exports, {
+  BackfillManager: () => BackfillManager,
+  BackfillReason: () => BackfillReason,
+  BackfillStrategy: () => BackfillStrategy
+});
+module.exports = __toCommonJS(backfill_exports);
+var import_filters = require("./filters.cjs");
+function calculateCheckpointFromDocs(docs) {
+  if (docs.length === 0) return void 0;
+  let maxTimestamp = "";
+  let maxDate = null;
+  for (const doc of docs) {
+    const modified = doc.modified;
+    if (!modified) continue;
+    try {
+      const date = new Date(modified);
+      if (Number.isNaN(date.getTime())) continue;
+      if (!maxDate || date > maxDate) {
+        maxDate = date;
+        maxTimestamp = modified;
+      }
+    } catch {
+      if (!maxTimestamp || modified > maxTimestamp) {
+        maxTimestamp = modified;
+      }
+    }
+  }
+  return maxTimestamp || void 0;
+}
+const log = console;
+var BackfillReason = /* @__PURE__ */(BackfillReason2 => {
+  BackfillReason2["DeleteGap"] = "delete-gap";
+  BackfillReason2["Initial"] = "initial";
+  BackfillReason2["Pagination"] = "pagination";
+  BackfillReason2["Periodic"] = "periodic";
+  BackfillReason2["Recovery"] = "recovery";
+  BackfillReason2["Rehydration"] = "rehydration";
+  BackfillReason2["RemoteCreate"] = "remote-create";
+  BackfillReason2["RemoteUpdate"] = "remote-update";
+  return BackfillReason2;
+})(BackfillReason || {});
+var BackfillStrategy = /* @__PURE__ */(BackfillStrategy2 => {
+  BackfillStrategy2["CursorFetch"] = "cursor-fetch";
+  BackfillStrategy2["FullRefresh"] = "full-refresh";
+  BackfillStrategy2["GapFill"] = "gap-fill";
+  BackfillStrategy2["Incremental"] = "incremental";
+  BackfillStrategy2["PageOnly"] = "page-only";
+  return BackfillStrategy2;
+})(BackfillStrategy || {});
+class BackfillManager {
+  constructor(db, normalizedCache, options = {}) {
+    this.db = db;
+    this.normalizedCache = normalizedCache;
+    this.options = options;
+  }
+  db;
+  normalizedCache;
+  options;
+  activeBackfills = /* @__PURE__ */new Map();
+  backfillQueue = /* @__PURE__ */new Map();
+  async backfill(subscription, reason) {
+    const subscriptionId = subscription.id;
+    if (this.activeBackfills.has(subscriptionId)) {
+      this.queueBackfill(subscriptionId, reason);
+      return this.activeBackfills.get(subscriptionId);
+    }
+    const backfillPromise = this.performBackfill(subscription, reason);
+    this.activeBackfills.set(subscriptionId, backfillPromise);
+    let backfillSucceeded = false;
+    try {
+      await backfillPromise;
+      backfillSucceeded = true;
+    } catch (err) {
+      log.error(err);
+    } finally {
+      this.activeBackfills.delete(subscriptionId);
+      if (backfillSucceeded) {
+        const queued = this.backfillQueue.get(subscriptionId);
+        if (queued && queued.length > 0) {
+          const reason2 = queued.shift();
+          if (queued.length === 0) this.backfillQueue.delete(subscriptionId);
+          if (reason2) this.backfill(subscription, reason2).catch(log.error);
+        }
+      } else {
+        this.backfillQueue.delete(subscriptionId);
+      }
+    }
+  }
+  /**
+   * Backfill one item to fill a gap
+   */
+  async backfillOne(subscription) {
+    if (!subscription.getProgressiveCache().hasMore) {
+      return null;
+    }
+    try {
+      const docs = await this.fetchWithCursor(subscription, 1);
+      if (docs.length > 0) {
+        const doc = docs[0];
+        subscription.processDocuments([doc], this.normalizedCache, {
+          calculateCheckpoint: false
+          // Single doc doesn't need checkpoint update
+        });
+        return doc;
+      }
+      subscription.updateProgressiveCache({
+        hasMore: false
+      });
+      return null;
+    } catch (error) {
+      console.error("Failed to backfill one item:", error);
+      return null;
+    }
+  }
+  /**
+   * Rehydrate current page
+   */
+  async rehydrateCurrentPage(subscription) {
+    const pageSize = subscription.viewWindow.pageSize;
+    const currentPage = subscription.viewWindow.currentPage;
+    const start = (currentPage - 1) * pageSize;
+    try {
+      const response = await this.db.getDocList(subscription.query.doctype, {
+        fields: subscription.query.fields,
+        filters: subscription.query.filters,
+        orderBy: subscription.query.orderBy,
+        limitStart: start,
+        limit: pageSize
+      });
+      const newDocIds = response.map(doc => doc.name);
+      const currentDocIds = subscription.getProgressiveCache().docIds;
+      const updatedDocIds = [...currentDocIds];
+      updatedDocIds.splice(start, pageSize, ...newDocIds);
+      subscription.updateProgressiveCache({
+        docIds: updatedDocIds
+      });
+      for (const doc of response) {
+        this.normalizedCache.upsert(doc);
+      }
+      subscription.needsRehydration = false;
+    } catch (error) {
+      console.error("Failed to rehydrate current page:", error);
+      throw error;
+    }
+  }
+  async performBackfill(subscription, reason) {
+    const strategy = this.determineStrategy(subscription, reason);
+    switch (strategy) {
+      case "full-refresh" /* FullRefresh */:
+        await this.fullRefresh(subscription);
+        break;
+      case "incremental" /* Incremental */:
+        await this.incrementalBackfill(subscription);
+        break;
+      case "page-only" /* PageOnly */:
+        await this.pageBackfill(subscription);
+        break;
+      case "gap-fill" /* GapFill */:
+        await this.gapBackfill(subscription);
+        break;
+      case "cursor-fetch" /* CursorFetch */:
+        await this.cursorFetch(subscription);
+        break;
+    }
+  }
+  determineStrategy(subscription, reason) {
+    switch (reason) {
+      case "initial" /* Initial */:
+        return "full-refresh" /* FullRefresh */;
+      case "pagination" /* Pagination */:
+        return subscription.getProgressiveCache().nextCursor ? "cursor-fetch" /* CursorFetch */ : "incremental" /* Incremental */;
+      case "delete-gap" /* DeleteGap */:
+        return "gap-fill" /* GapFill */;
+      case "remote-create" /* RemoteCreate */:
+      case "remote-update" /* RemoteUpdate */:
+      case "rehydration" /* Rehydration */:
+        return "page-only" /* PageOnly */;
+      case "periodic" /* Periodic */:
+        return subscription.checkpoint ? "incremental" /* Incremental */ : "full-refresh" /* FullRefresh */;
+      case "recovery" /* Recovery */:
+        return subscription.checkpoint ? "incremental" /* Incremental */ : "full-refresh" /* FullRefresh */;
+      default:
+        return "incremental" /* Incremental */;
+    }
+  }
+  async fullRefresh(subscription) {
+    const batchSize = this.options.batchSize || this.options.pageSize || 100;
+    const response = await this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      limit: batchSize,
+      limitStart: 0
+    });
+    subscription.processDocuments(response, this.normalizedCache);
+    subscription.updateProgressiveCache({
+      hasMore: response.length === batchSize
+    });
+  }
+  async incrementalBackfill(subscription) {
+    if (!subscription.checkpoint) {
+      return this.fullRefresh(subscription);
+    }
+    try {
+      const response = await this.db.backfill({
+        doctype: subscription.query.doctype,
+        fields: subscription.query.fields,
+        filters: subscription.query.filters,
+        modifiedAfter: subscription.checkpoint,
+        limit: this.options.pageSize || 200
+      });
+      if ("deleted" in response && Array.isArray(response.deleted)) {
+        const deletedDocIds = response.deleted.filter(deletion => deletion?.name).map(deletion => deletion.name);
+        if (deletedDocIds.length > 0) {
+          subscription.removeDocuments(deletedDocIds, this.normalizedCache);
+        }
+      }
+      for (const doc of response.rows) {
+        const exists = subscription.getProgressiveCache().docIds.includes(doc.name);
+        const matches = (0, import_filters.matchesFilters)(doc, subscription.query.filters);
+        if (exists && !matches) {
+          subscription.removeDocuments([doc.name], this.normalizedCache);
+        } else if (!exists && matches) {
+          subscription.needsRehydration = true;
+        } else if (exists) {
+          this.normalizedCache.upsert(doc);
+        }
+      }
+      subscription.checkpoint = calculateCheckpointFromDocs(response.rows);
+    } catch (error) {
+      console.error("Incremental backfill failed:", error);
+      await this.fullRefresh(subscription);
+    }
+  }
+  async pageBackfill(subscription) {
+    const pageSize = subscription.viewWindow.pageSize;
+    const currentPage = subscription.viewWindow.currentPage;
+    const start = (currentPage - 1) * pageSize;
+    const response = await this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      limitStart: start,
+      limit: pageSize
+    });
+    const newDocIds = response.map(doc => doc.name);
+    const currentDocIds = subscription.getProgressiveCache().docIds;
+    const updatedDocIds = [...currentDocIds];
+    updatedDocIds.splice(start, pageSize, ...newDocIds);
+    subscription.updateProgressiveCache({
+      docIds: updatedDocIds
+    });
+    for (const doc of response) {
+      this.normalizedCache.upsert(doc);
+    }
+    subscription.needsRehydration = false;
+  }
+  async gapBackfill(subscription) {
+    await this.backfillOne(subscription);
+    subscription.pendingBackfill = false;
+  }
+  async cursorFetch(subscription) {
+    if (!subscription.getProgressiveCache().hasMore) return;
+    const batchSize = this.options.batchSize || this.options.pageSize || 100;
+    const docs = await this.fetchWithCursor(subscription, batchSize);
+    subscription.updateProgressiveCache({
+      docIds: [...subscription.getProgressiveCache().docIds, ...docs.map(doc => doc.name)],
+      nextCursor: docs.length > 0 ? docs[docs.length - 1].name : subscription.getProgressiveCache().nextCursor,
+      hasMore: docs.length === batchSize
+    });
+    for (const doc of docs) {
+      this.normalizedCache.upsert(doc);
+    }
+  }
+  async fetchWithCursor(subscription, limit) {
+    return this.db.getDocList(subscription.query.doctype, {
+      fields: subscription.query.fields,
+      filters: subscription.query.filters,
+      orderBy: subscription.query.orderBy,
+      cursor: subscription.getProgressiveCache().nextCursor,
+      limit
+    });
+  }
+  queueBackfill(subscriptionId, reason) {
+    if (!this.backfillQueue.has(subscriptionId)) {
+      this.backfillQueue.set(subscriptionId, []);
+    }
+    const queue = this.backfillQueue.get(subscriptionId);
+    if (!queue.includes(reason)) queue.push(reason);
+  }
+}