@grabbit-labs/dynafetch 0.2.5 → 0.2.7

package/README.md

@@ -119,7 +119,7 @@ const page = await dynafetch({
 |-|-|-|
 | `minWaitMs` | `75` | Minimum ms before checking idle state |
 | `idleWaitMs` | `100` | Ms of zero pending requests to consider settled |
-| `maxWaitMs` | `2000` | Hard cap on wait time |
+| `maxWaitMs` | `3000` | Hard cap on wait time |
 | `moduleWaitMs` | `6000` | Max wait for ES module bundling |
 | `timeoutMs` | none | Overall operation timeout |
 

package/dist/index.js

@@ -829,9 +829,11 @@ function shouldSkipDynamicScriptUrl(url, pageUrl, policy) {
 // ../../src/phantom/execute.ts
 var esbuildModulePromise = null;
 var esbuildRefCount = 0;
+var runtimeImport = new Function("specifier", "return import(specifier)");
+var ESBUILD_MODULE_NAME = Buffer.from("ZXNidWlsZA==", "base64").toString("utf8");
 async function acquireEsbuildModule() {
   if (!esbuildModulePromise) {
-    esbuildModulePromise = import("esbuild");
+    esbuildModulePromise = runtimeImport(ESBUILD_MODULE_NAME);
   }
   esbuildRefCount++;
   try {
@@ -890,6 +892,8 @@ var Executor = class {
     this.originalGlobalMessageChannel = void 0;
     this.originalGlobalMessagePort = void 0;
     this.activeTimerHandles = /* @__PURE__ */ new Set();
+    this.domMutationVersion = 0;
+    this.domMutationObserver = null;
     // Simple telemetry counters (useful for debugging).
     this.telemetry_stubbed = 0;
     this.telemetry_proxy = 0;
@@ -1008,6 +1012,29 @@ var Executor = class {
     }
     this.activeTimerHandles.clear();
   }
+  noteDomMutation() {
+    this.domMutationVersion++;
+  }
+  async waitForDomSettle(window) {
+    const quietMs = this.clampMs(
+      Number(process.env.PHANTOM_DOM_SETTLE_MS ?? Math.max(this.quiescenceOptions.idleWaitMs * 4, 400)),
+      50,
+      5e3
+    );
+    const maxMs = this.clampMs(
+      Number(process.env.PHANTOM_DOM_SETTLE_MAX_MS ?? Math.max(quietMs * 4, 2e3)),
+      quietMs,
+      1e4
+    );
+    const start = Date.now();
+    let seenVersion = this.domMutationVersion;
+    while (!this.windowClosed) {
+      await new Promise((resolve) => setTimeout(resolve, quietMs));
+      if (this.domMutationVersion === seenVersion) return;
+      seenVersion = this.domMutationVersion;
+      if (Date.now() - start >= maxMs) return;
+    }
+  }
   async performAutoScroll(window) {
     const steps = this.clampMs(Number(process.env.PHANTOM_AUTO_SCROLL_STEPS ?? 3), 0, 12);
     if (steps === 0) return false;
@@ -1065,7 +1092,7 @@ var Executor = class {
     const hardMaxCap = this.clampMs(Number(process.env.PHANTOM_QUIESCENCE_MAX_CAP_MS ?? 8e3), 500, 6e4);
     const minWaitMs = this.clampMs(quiescence?.minWaitMs ?? 75, 0, 1e4);
     const idleWaitMs = this.clampMs(quiescence?.idleWaitMs ?? 100, 0, 1e4);
-    const maxWaitMs = this.clampMs(quiescence?.maxWaitMs ?? 2e3, 0, hardMaxCap);
+    const maxWaitMs = this.clampMs(quiescence?.maxWaitMs ?? 3e3, 0, hardMaxCap);
     const hardModuleCap = this.clampMs(Number(process.env.PHANTOM_MODULE_WAIT_MAX_CAP_MS ?? 3e4), 1e3, 12e4);
     this.moduleWaitMs = this.clampMs(Number(process.env.PHANTOM_MODULE_WAIT_MS ?? moduleWaitMsOverride ?? 6e3), 1e3, hardModuleCap);
     this.quiescenceOptions = { minWaitMs, idleWaitMs, maxWaitMs };
@@ -2272,6 +2299,23 @@ var Executor = class {
         }
       });
       const { window } = dom;
+      this.domMutationVersion = 0;
+      try {
+        const MutationObserverCtor = window.MutationObserver;
+        const targetNode = window.document?.documentElement || window.document?.body;
+        if (MutationObserverCtor && targetNode) {
+          this.domMutationObserver = new MutationObserverCtor(() => {
+            this.noteDomMutation();
+          });
+          this.domMutationObserver.observe(targetNode, {
+            subtree: true,
+            childList: true,
+            characterData: true,
+            attributes: true
+          });
+        }
+      } catch {
+      }
       let readyStateValue = "loading";
       try {
         Object.defineProperty(window.document, "readyState", {
@@ -2414,6 +2458,11 @@ var Executor = class {
           this.timings.quiescence_ms += Date.now() - postScrollStart;
         }
       }
+      try {
+        await this.waitForDomSettle(window);
+      } catch (e) {
+        warn("[Executor] DOM settle wait failed:", e);
+      }
       const renderedHtml = this.serializeDocument(window);
       this.windowClosed = true;
       const result = {
@@ -2430,6 +2479,11 @@ var Executor = class {
       this.unrefNewMessagePorts(initialActiveHandles);
       return result;
     } finally {
+      try {
+        this.domMutationObserver?.disconnect?.();
+      } catch {
+      }
+      this.domMutationObserver = null;
       const g = globalThis;
       if (this.originalGlobalMessageChannel !== void 0) {
         g.MessageChannel = this.originalGlobalMessageChannel;