@grabbit-labs/dynafetch 0.2.4 → 0.2.6

package/README.md

@@ -119,7 +119,7 @@ const page = await dynafetch({
 |-|-|-|
 | `minWaitMs` | `75` | Minimum ms before checking idle state |
 | `idleWaitMs` | `100` | Ms of zero pending requests to consider settled |
-| `maxWaitMs` | `2000` | Hard cap on wait time |
+| `maxWaitMs` | `3000` | Hard cap on wait time |
 | `moduleWaitMs` | `6000` | Max wait for ES module bundling |
 | `timeoutMs` | none | Overall operation timeout |
 

package/dist/index.js

@@ -764,8 +764,19 @@ var TELEMETRY_INLINE_PATTERNS = [
   /\bhj\s*\(/i,
   /\bnewrelic\b/i,
   /\bdatadog\b/i,
-  /\bSentry\b/i,
-  /\bLogRocket\b/i
+  /\bSentry\.(?:init|capture|captureException|configureScope|withScope)\b/i,
+  /\bLogRocket\.(?:init|identify|track)\b/i
+];
+var INLINE_APPLICATION_PATTERNS = [
+  /\bwebpackChunk\w*/i,
+  /\b__webpack_require__\b/i,
+  /\b__SCRIPTS_LOADED__\b/i,
+  /\b__LOADABLE_LOADED_CHUNKS__\b/i,
+  /\bparcelRequire\b/i,
+  /\bwindow\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bglobalThis\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bself\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bperformance\.mark\s*\(/i
 ];
 function siteKey(hostname) {
   const parts = hostname.split(".").filter(Boolean);
@@ -799,6 +810,7 @@ function classifyScriptAsset(script, pageUrl) {
     if (script.scriptKind === "module") return "application";
     return "unknown";
   }
+  if (matchesAny(content, INLINE_APPLICATION_PATTERNS)) return "application";
   if (matchesAny(content, TELEMETRY_INLINE_PATTERNS)) return "telemetry";
   return "application";
 }
@@ -878,6 +890,8 @@ var Executor = class {
     this.originalGlobalMessageChannel = void 0;
     this.originalGlobalMessagePort = void 0;
     this.activeTimerHandles = /* @__PURE__ */ new Set();
+    this.domMutationVersion = 0;
+    this.domMutationObserver = null;
     // Simple telemetry counters (useful for debugging).
     this.telemetry_stubbed = 0;
     this.telemetry_proxy = 0;
@@ -996,11 +1010,87 @@ var Executor = class {
     }
     this.activeTimerHandles.clear();
   }
+  noteDomMutation() {
+    this.domMutationVersion++;
+  }
+  async waitForDomSettle(window) {
+    const quietMs = this.clampMs(
+      Number(process.env.PHANTOM_DOM_SETTLE_MS ?? Math.max(this.quiescenceOptions.idleWaitMs * 4, 400)),
+      50,
+      5e3
+    );
+    const maxMs = this.clampMs(
+      Number(process.env.PHANTOM_DOM_SETTLE_MAX_MS ?? Math.max(quietMs * 4, 2e3)),
+      quietMs,
+      1e4
+    );
+    const start = Date.now();
+    let seenVersion = this.domMutationVersion;
+    while (!this.windowClosed) {
+      await new Promise((resolve) => setTimeout(resolve, quietMs));
+      if (this.domMutationVersion === seenVersion) return;
+      seenVersion = this.domMutationVersion;
+      if (Date.now() - start >= maxMs) return;
+    }
+  }
+  async performAutoScroll(window) {
+    const steps = this.clampMs(Number(process.env.PHANTOM_AUTO_SCROLL_STEPS ?? 3), 0, 12);
+    if (steps === 0) return false;
+    const stepPx = this.clampMs(
+      Number(process.env.PHANTOM_AUTO_SCROLL_STEP_PX ?? Math.max(Number(window.innerHeight) || 0, 900)),
+      100,
+      1e4
+    );
+    const delayMs = this.clampMs(Number(process.env.PHANTOM_AUTO_SCROLL_DELAY_MS ?? 150), 0, 5e3);
+    const dispatchScroll = () => {
+      try {
+        window.dispatchEvent(new window.Event("scroll"));
+      } catch {
+      }
+      try {
+        window.document?.dispatchEvent?.(new window.Event("scroll"));
+      } catch {
+      }
+      try {
+        window.document?.documentElement?.dispatchEvent?.(new window.Event("scroll"));
+      } catch {
+      }
+    };
+    let didScroll = false;
+    for (let step = 0; step < steps && !this.windowClosed; step++) {
+      const nextY = (Number(window.scrollY) || 0) + stepPx;
+      try {
+        window.scrollTo(0, nextY);
+      } catch {
+        try {
+          window.scrollY = nextY;
+        } catch {
+        }
+      }
+      dispatchScroll();
+      didScroll = true;
+      if (delayMs > 0) {
+        await new Promise((resolve) => window.setTimeout(resolve, delayMs));
+      }
+    }
+    if (didScroll) {
+      try {
+        window.scrollTo(0, 0);
+      } catch {
+        try {
+          window.scrollY = 0;
+        } catch {
+        }
+      }
+      dispatchScroll();
+    }
+    return didScroll;
+  }
   applyDefaults(quiescence, moduleWaitMsOverride) {
     const hardMaxCap = this.clampMs(Number(process.env.PHANTOM_QUIESCENCE_MAX_CAP_MS ?? 8e3), 500, 6e4);
     const minWaitMs = this.clampMs(quiescence?.minWaitMs ?? 75, 0, 1e4);
     const idleWaitMs = this.clampMs(quiescence?.idleWaitMs ?? 100, 0, 1e4);
-    const maxWaitMs = this.clampMs(quiescence?.maxWaitMs ?? 2e3, 0, hardMaxCap);
+    const maxWaitMs = this.clampMs(quiescence?.maxWaitMs ?? 3e3, 0, hardMaxCap);
     const hardModuleCap = this.clampMs(Number(process.env.PHANTOM_MODULE_WAIT_MAX_CAP_MS ?? 3e4), 1e3, 12e4);
     this.moduleWaitMs = this.clampMs(Number(process.env.PHANTOM_MODULE_WAIT_MS ?? moduleWaitMsOverride ?? 6e3), 1e3, hardModuleCap);
     this.quiescenceOptions = { minWaitMs, idleWaitMs, maxWaitMs };
@@ -1539,9 +1629,10 @@ var Executor = class {
       };
     }
     if (!window.crypto) window.crypto = {};
+    const nodeWebCrypto = g.crypto || nodeCrypto.webcrypto;
     if (!window.crypto.getRandomValues) {
-      if (g.crypto && typeof g.crypto.getRandomValues === "function") {
-        window.crypto.getRandomValues = g.crypto.getRandomValues.bind(g.crypto);
+      if (nodeWebCrypto && typeof nodeWebCrypto.getRandomValues === "function") {
+        window.crypto.getRandomValues = nodeWebCrypto.getRandomValues.bind(nodeWebCrypto);
       } else {
         window.crypto.getRandomValues = (arr) => {
           const buf = nodeCrypto.randomBytes(arr.length);
@@ -1550,6 +1641,22 @@ var Executor = class {
         };
       }
     }
+    if (!window.crypto.subtle && nodeWebCrypto?.subtle) {
+      window.crypto.subtle = nodeWebCrypto.subtle;
+    }
+    if (!window.crypto.randomUUID) {
+      if (typeof nodeWebCrypto?.randomUUID === "function") {
+        window.crypto.randomUUID = nodeWebCrypto.randomUUID.bind(nodeWebCrypto);
+      } else {
+        window.crypto.randomUUID = () => nodeCrypto.randomUUID();
+      }
+    }
+    if (!window.CryptoKey && g.CryptoKey) {
+      window.CryptoKey = g.CryptoKey;
+    }
+    if (!window.SubtleCrypto && g.SubtleCrypto) {
+      window.SubtleCrypto = g.SubtleCrypto;
+    }
     {
       const _g = globalThis;
       if (_g.MessageChannel) {
@@ -1629,6 +1736,81 @@ var Executor = class {
         availHeight: 900
       };
     }
+    const parsePx = (value) => {
+      if (typeof value === "number" && Number.isFinite(value)) return value;
+      if (typeof value === "string") {
+        const parsed = Number.parseFloat(value.replace(/px$/i, ""));
+        if (Number.isFinite(parsed)) return parsed;
+      }
+      return 0;
+    };
+    const fallbackViewportHeight = Math.max(Number(window.innerHeight) || 0, 900);
+    const fallbackViewportWidth = Math.max(Number(window.innerWidth) || 0, 1440);
+    const fallbackDocumentHeight = Math.max(fallbackViewportHeight * 6, 5400);
+    const makeRect = (width, height) => ({
+      x: 0,
+      y: 0,
+      top: 0,
+      left: 0,
+      bottom: height,
+      right: width,
+      width,
+      height,
+      toJSON() {
+        return { x: 0, y: 0, top: 0, left: 0, bottom: height, right: width, width, height };
+      }
+    });
+    const computeElementBox = (el) => {
+      const tag = String(el?.tagName || "").toLowerCase();
+      if (tag === "html" || tag === "body") {
+        return {
+          width: fallbackViewportWidth,
+          height: fallbackDocumentHeight
+        };
+      }
+      const width = parsePx(el?.style?.width) || parsePx(el?.getAttribute?.("width")) || fallbackViewportWidth;
+      const height = parsePx(el?.style?.height) || parsePx(el?.getAttribute?.("height")) || 240;
+      return { width, height };
+    };
+    try {
+      const elementProto = window.Element?.prototype;
+      if (elementProto) {
+        const originalGetBoundingClientRect = elementProto.getBoundingClientRect;
+        elementProto.getBoundingClientRect = function() {
+          try {
+            const rect = originalGetBoundingClientRect?.call(this);
+            if (rect && (rect.width > 0 || rect.height > 0)) return rect;
+          } catch {
+          }
+          const { width, height } = computeElementBox(this);
+          return makeRect(width, height);
+        };
+      }
+      const htmlEl = window.document?.documentElement;
+      const bodyEl = window.document?.body;
+      const defineSize = (target, name, value) => {
+        try {
+          Object.defineProperty(target, name, {
+            configurable: true,
+            get: () => value
+          });
+        } catch {
+        }
+      };
+      if (htmlEl) {
+        defineSize(htmlEl, "clientWidth", fallbackViewportWidth);
+        defineSize(htmlEl, "clientHeight", fallbackViewportHeight);
+        defineSize(htmlEl, "scrollWidth", fallbackViewportWidth);
+        defineSize(htmlEl, "scrollHeight", fallbackDocumentHeight);
+      }
+      if (bodyEl) {
+        defineSize(bodyEl, "clientWidth", fallbackViewportWidth);
+        defineSize(bodyEl, "clientHeight", fallbackViewportHeight);
+        defineSize(bodyEl, "scrollWidth", fallbackViewportWidth);
+        defineSize(bodyEl, "scrollHeight", fallbackDocumentHeight);
+      }
+    } catch {
+    }
     if (!window.visualViewport) {
       window.visualViewport = {
         width: window.innerWidth,
@@ -2115,6 +2297,23 @@ var Executor = class {
         }
       });
       const { window } = dom;
+      this.domMutationVersion = 0;
+      try {
+        const MutationObserverCtor = window.MutationObserver;
+        const targetNode = window.document?.documentElement || window.document?.body;
+        if (MutationObserverCtor && targetNode) {
+          this.domMutationObserver = new MutationObserverCtor(() => {
+            this.noteDomMutation();
+          });
+          this.domMutationObserver.observe(targetNode, {
+            subtree: true,
+            childList: true,
+            characterData: true,
+            attributes: true
+          });
+        }
+      } catch {
+      }
       let readyStateValue = "loading";
       try {
         Object.defineProperty(window.document, "readyState", {
@@ -2241,6 +2440,27 @@ var Executor = class {
       this.timings.quiescence_ms = Date.now() - quiescenceStart;
       const reason = this.matchFound && !this.findAll ? "(early exit on match)" : "";
       log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
+      if (!this.matchFound || this.findAll) {
+        const didScroll = await this.performAutoScroll(window);
+        if (didScroll) {
+          if (this.moduleInFlight.size > 0) {
+            await this.waitForModuleWork(this.moduleWaitMs);
+          }
+          log("[Executor] Waiting for post-scroll quiescence...");
+          const postScrollStart = Date.now();
+          try {
+            await this.waitForQuiescence();
+          } catch (e) {
+            warn("[Executor] Post-scroll quiescence wait failed:", e);
+          }
+          this.timings.quiescence_ms += Date.now() - postScrollStart;
+        }
+      }
+      try {
+        await this.waitForDomSettle(window);
+      } catch (e) {
+        warn("[Executor] DOM settle wait failed:", e);
+      }
       const renderedHtml = this.serializeDocument(window);
       this.windowClosed = true;
       const result = {
@@ -2257,6 +2477,11 @@ var Executor = class {
       this.unrefNewMessagePorts(initialActiveHandles);
       return result;
     } finally {
+      try {
+        this.domMutationObserver?.disconnect?.();
+      } catch {
+      }
+      this.domMutationObserver = null;
       const g = globalThis;
       if (this.originalGlobalMessageChannel !== void 0) {
         g.MessageChannel = this.originalGlobalMessageChannel;