@grabbit-labs/dynafetch 0.2.3 → 0.2.5

package/dist/index.js

@@ -325,7 +325,8 @@ async function withDynafetchSession(options, run) {
   try {
     return await sessionStore.run({ sessionId: session.sessionId }, run);
   } finally {
-    callWorker("closeSession", { sessionId: session.sessionId }).catch(() => {
+    const closeTimeoutMs = Math.min(resolveRpcTimeoutMs(options), 5e3);
+    await callWorker("closeSession", { sessionId: session.sessionId }, closeTimeoutMs).catch(() => {
     });
     transport.release();
   }
@@ -763,8 +764,19 @@ var TELEMETRY_INLINE_PATTERNS = [
   /\bhj\s*\(/i,
   /\bnewrelic\b/i,
   /\bdatadog\b/i,
-  /\bSentry\b/i,
-  /\bLogRocket\b/i
+  /\bSentry\.(?:init|capture|captureException|configureScope|withScope)\b/i,
+  /\bLogRocket\.(?:init|identify|track)\b/i
+];
+var INLINE_APPLICATION_PATTERNS = [
+  /\bwebpackChunk\w*/i,
+  /\b__webpack_require__\b/i,
+  /\b__SCRIPTS_LOADED__\b/i,
+  /\b__LOADABLE_LOADED_CHUNKS__\b/i,
+  /\bparcelRequire\b/i,
+  /\bwindow\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bglobalThis\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bself\.__[A-Z0-9_]{3,}\s*=/i,
+  /\bperformance\.mark\s*\(/i
 ];
 function siteKey(hostname) {
   const parts = hostname.split(".").filter(Boolean);
@@ -798,6 +810,7 @@ function classifyScriptAsset(script, pageUrl) {
     if (script.scriptKind === "module") return "application";
     return "unknown";
   }
+  if (matchesAny(content, INLINE_APPLICATION_PATTERNS)) return "application";
   if (matchesAny(content, TELEMETRY_INLINE_PATTERNS)) return "telemetry";
   return "application";
 }
@@ -876,6 +889,7 @@ var Executor = class {
     this.esbuildModule = null;
     this.originalGlobalMessageChannel = void 0;
     this.originalGlobalMessagePort = void 0;
+    this.activeTimerHandles = /* @__PURE__ */ new Set();
     // Simple telemetry counters (useful for debugging).
     this.telemetry_stubbed = 0;
     this.telemetry_proxy = 0;
@@ -977,6 +991,76 @@ var Executor = class {
       }
     }
   }
+  clearTrackedTimers(window) {
+    for (const handle of this.activeTimerHandles) {
+      try {
+        window.clearTimeout?.(handle);
+      } catch {
+      }
+      try {
+        window.clearInterval?.(handle);
+      } catch {
+      }
+      try {
+        handle?.unref?.();
+      } catch {
+      }
+    }
+    this.activeTimerHandles.clear();
+  }
+  async performAutoScroll(window) {
+    const steps = this.clampMs(Number(process.env.PHANTOM_AUTO_SCROLL_STEPS ?? 3), 0, 12);
+    if (steps === 0) return false;
+    const stepPx = this.clampMs(
+      Number(process.env.PHANTOM_AUTO_SCROLL_STEP_PX ?? Math.max(Number(window.innerHeight) || 0, 900)),
+      100,
+      1e4
+    );
+    const delayMs = this.clampMs(Number(process.env.PHANTOM_AUTO_SCROLL_DELAY_MS ?? 150), 0, 5e3);
+    const dispatchScroll = () => {
+      try {
+        window.dispatchEvent(new window.Event("scroll"));
+      } catch {
+      }
+      try {
+        window.document?.dispatchEvent?.(new window.Event("scroll"));
+      } catch {
+      }
+      try {
+        window.document?.documentElement?.dispatchEvent?.(new window.Event("scroll"));
+      } catch {
+      }
+    };
+    let didScroll = false;
+    for (let step = 0; step < steps && !this.windowClosed; step++) {
+      const nextY = (Number(window.scrollY) || 0) + stepPx;
+      try {
+        window.scrollTo(0, nextY);
+      } catch {
+        try {
+          window.scrollY = nextY;
+        } catch {
+        }
+      }
+      dispatchScroll();
+      didScroll = true;
+      if (delayMs > 0) {
+        await new Promise((resolve) => window.setTimeout(resolve, delayMs));
+      }
+    }
+    if (didScroll) {
+      try {
+        window.scrollTo(0, 0);
+      } catch {
+        try {
+          window.scrollY = 0;
+        } catch {
+        }
+      }
+      dispatchScroll();
+    }
+    return didScroll;
+  }
   applyDefaults(quiescence, moduleWaitMsOverride) {
     const hardMaxCap = this.clampMs(Number(process.env.PHANTOM_QUIESCENCE_MAX_CAP_MS ?? 8e3), 500, 6e4);
     const minWaitMs = this.clampMs(quiescence?.minWaitMs ?? 75, 0, 1e4);
@@ -1414,14 +1498,66 @@ var Executor = class {
       };
     }
     const g = globalThis;
-    if (!window.Headers && g.Headers) window.Headers = g.Headers;
-    if (!window.Request && g.Request) window.Request = g.Request;
-    if (!window.Response && g.Response) window.Response = g.Response;
-    if (!window.AbortController && g.AbortController) window.AbortController = g.AbortController;
-    if (!window.AbortSignal && g.AbortSignal) window.AbortSignal = g.AbortSignal;
+    if (g.Headers) window.Headers = g.Headers;
+    if (g.Response) window.Response = g.Response;
+    if (g.AbortController) window.AbortController = g.AbortController;
+    if (g.AbortSignal) window.AbortSignal = g.AbortSignal;
     if (!window.TextEncoder && g.TextEncoder) window.TextEncoder = g.TextEncoder;
     if (!window.TextDecoder && g.TextDecoder) window.TextDecoder = g.TextDecoder;
     if (!window.structuredClone && g.structuredClone) window.structuredClone = g.structuredClone.bind(g);
+    const normalizeUrlForRequest = (value) => {
+      if (typeof value !== "string") return value;
+      try {
+        return new URL(value, window.location.href).toString();
+      } catch {
+        return value;
+      }
+    };
+    const normalizeAbortSignalForRequest = (signal) => {
+      if (!signal || !g.AbortController || !g.AbortSignal) return signal;
+      if (signal instanceof g.AbortSignal) return signal;
+      if (typeof signal.aborted !== "boolean") return void 0;
+      const controller = new g.AbortController();
+      if (signal.aborted) {
+        try {
+          controller.abort(signal.reason);
+        } catch {
+          controller.abort();
+        }
+        return controller.signal;
+      }
+      if (typeof signal.addEventListener === "function") {
+        try {
+          signal.addEventListener("abort", () => {
+            try {
+              controller.abort(signal.reason);
+            } catch {
+              controller.abort();
+            }
+          }, { once: true });
+        } catch {
+        }
+      }
+      return controller.signal;
+    };
+    if (g.Request) {
+      const NativeRequest = g.Request;
+      const RequestShim = class Request extends NativeRequest {
+        constructor(input, init) {
+          const normalizedInput = typeof input === "string" ? normalizeUrlForRequest(input) : input instanceof URL ? normalizeUrlForRequest(input.toString()) : input;
+          const normalizedInit = init && typeof init === "object" ? {
+            ...init,
+            signal: normalizeAbortSignalForRequest(init.signal)
+          } : init;
+          super(normalizedInput, normalizedInit);
+        }
+      };
+      try {
+        Object.setPrototypeOf(RequestShim, NativeRequest);
+      } catch {
+      }
+      window.Request = RequestShim;
+    }
     const makeStorage = () => {
       const store = /* @__PURE__ */ new Map();
       return {
@@ -1468,9 +1604,10 @@ var Executor = class {
       };
     }
     if (!window.crypto) window.crypto = {};
+    const nodeWebCrypto = g.crypto || nodeCrypto.webcrypto;
     if (!window.crypto.getRandomValues) {
-      if (g.crypto && typeof g.crypto.getRandomValues === "function") {
-        window.crypto.getRandomValues = g.crypto.getRandomValues.bind(g.crypto);
+      if (nodeWebCrypto && typeof nodeWebCrypto.getRandomValues === "function") {
+        window.crypto.getRandomValues = nodeWebCrypto.getRandomValues.bind(nodeWebCrypto);
       } else {
         window.crypto.getRandomValues = (arr) => {
           const buf = nodeCrypto.randomBytes(arr.length);
@@ -1479,6 +1616,22 @@ var Executor = class {
         };
       }
     }
+    if (!window.crypto.subtle && nodeWebCrypto?.subtle) {
+      window.crypto.subtle = nodeWebCrypto.subtle;
+    }
+    if (!window.crypto.randomUUID) {
+      if (typeof nodeWebCrypto?.randomUUID === "function") {
+        window.crypto.randomUUID = nodeWebCrypto.randomUUID.bind(nodeWebCrypto);
+      } else {
+        window.crypto.randomUUID = () => nodeCrypto.randomUUID();
+      }
+    }
+    if (!window.CryptoKey && g.CryptoKey) {
+      window.CryptoKey = g.CryptoKey;
+    }
+    if (!window.SubtleCrypto && g.SubtleCrypto) {
+      window.SubtleCrypto = g.SubtleCrypto;
+    }
     {
       const _g = globalThis;
       if (_g.MessageChannel) {
@@ -1558,6 +1711,81 @@ var Executor = class {
         availHeight: 900
       };
     }
+    const parsePx = (value) => {
+      if (typeof value === "number" && Number.isFinite(value)) return value;
+      if (typeof value === "string") {
+        const parsed = Number.parseFloat(value.replace(/px$/i, ""));
+        if (Number.isFinite(parsed)) return parsed;
+      }
+      return 0;
+    };
+    const fallbackViewportHeight = Math.max(Number(window.innerHeight) || 0, 900);
+    const fallbackViewportWidth = Math.max(Number(window.innerWidth) || 0, 1440);
+    const fallbackDocumentHeight = Math.max(fallbackViewportHeight * 6, 5400);
+    const makeRect = (width, height) => ({
+      x: 0,
+      y: 0,
+      top: 0,
+      left: 0,
+      bottom: height,
+      right: width,
+      width,
+      height,
+      toJSON() {
+        return { x: 0, y: 0, top: 0, left: 0, bottom: height, right: width, width, height };
+      }
+    });
+    const computeElementBox = (el) => {
+      const tag = String(el?.tagName || "").toLowerCase();
+      if (tag === "html" || tag === "body") {
+        return {
+          width: fallbackViewportWidth,
+          height: fallbackDocumentHeight
+        };
+      }
+      const width = parsePx(el?.style?.width) || parsePx(el?.getAttribute?.("width")) || fallbackViewportWidth;
+      const height = parsePx(el?.style?.height) || parsePx(el?.getAttribute?.("height")) || 240;
+      return { width, height };
+    };
+    try {
+      const elementProto = window.Element?.prototype;
+      if (elementProto) {
+        const originalGetBoundingClientRect = elementProto.getBoundingClientRect;
+        elementProto.getBoundingClientRect = function() {
+          try {
+            const rect = originalGetBoundingClientRect?.call(this);
+            if (rect && (rect.width > 0 || rect.height > 0)) return rect;
+          } catch {
+          }
+          const { width, height } = computeElementBox(this);
+          return makeRect(width, height);
+        };
+      }
+      const htmlEl = window.document?.documentElement;
+      const bodyEl = window.document?.body;
+      const defineSize = (target, name, value) => {
+        try {
+          Object.defineProperty(target, name, {
+            configurable: true,
+            get: () => value
+          });
+        } catch {
+        }
+      };
+      if (htmlEl) {
+        defineSize(htmlEl, "clientWidth", fallbackViewportWidth);
+        defineSize(htmlEl, "clientHeight", fallbackViewportHeight);
+        defineSize(htmlEl, "scrollWidth", fallbackViewportWidth);
+        defineSize(htmlEl, "scrollHeight", fallbackDocumentHeight);
+      }
+      if (bodyEl) {
+        defineSize(bodyEl, "clientWidth", fallbackViewportWidth);
+        defineSize(bodyEl, "clientHeight", fallbackViewportHeight);
+        defineSize(bodyEl, "scrollWidth", fallbackViewportWidth);
+        defineSize(bodyEl, "scrollHeight", fallbackDocumentHeight);
+      }
+    } catch {
+    }
     if (!window.visualViewport) {
       window.visualViewport = {
         width: window.innerWidth,
@@ -1620,8 +1848,41 @@ var Executor = class {
     };
     const _setTimeout = window.setTimeout?.bind(window);
     const _setInterval = window.setInterval?.bind(window);
-    if (_setTimeout) window.setTimeout = (cb, ms, ...rest) => _setTimeout(wrapCb(cb), ms, ...rest);
-    if (_setInterval) window.setInterval = (cb, ms, ...rest) => _setInterval(wrapCb(cb), ms, ...rest);
+    const _clearTimeout = window.clearTimeout?.bind(window);
+    const _clearInterval = window.clearInterval?.bind(window);
+    if (_setTimeout) {
+      window.setTimeout = (cb, ms, ...rest) => {
+        let handle;
+        const wrapped = wrapCb((...args) => {
+          this.activeTimerHandles.delete(handle);
+          return typeof cb === "function" ? cb(...args) : cb;
+        });
+        handle = _setTimeout(wrapped, ms, ...rest);
+        this.activeTimerHandles.add(handle);
+        handle?.unref?.();
+        return handle;
+      };
+    }
+    if (_setInterval) {
+      window.setInterval = (cb, ms, ...rest) => {
+        const handle = _setInterval(wrapCb(cb), ms, ...rest);
+        this.activeTimerHandles.add(handle);
+        handle?.unref?.();
+        return handle;
+      };
+    }
+    if (_clearTimeout) {
+      window.clearTimeout = (handle) => {
+        this.activeTimerHandles.delete(handle);
+        return _clearTimeout(handle);
+      };
+    }
+    if (_clearInterval) {
+      window.clearInterval = (handle) => {
+        this.activeTimerHandles.delete(handle);
+        return _clearInterval(handle);
+      };
+    }
     if (window.queueMicrotask) {
       const _q = window.queueMicrotask.bind(window);
       window.queueMicrotask = (cb) => _q(wrapCb(cb));
@@ -1731,7 +1992,7 @@ var Executor = class {
     try {
       const virtualConsole = new VirtualConsole();
       virtualConsole.on("log", (...args) => log("[JSDOM Log]", ...args));
-      virtualConsole.on("error", (...args) => console.error("[JSDOM Error]", ...args));
+      virtualConsole.on("error", (...args) => warn("[JSDOM Error]", ...args));
       virtualConsole.on("warn", (...args) => warn("[JSDOM Warn]", ...args));
       const cookieJar = new CookieJar();
       this.harvestData.cookies.forEach((c) => {
@@ -2137,12 +2398,24 @@ var Executor = class {
       this.timings.quiescence_ms = Date.now() - quiescenceStart;
       const reason = this.matchFound && !this.findAll ? "(early exit on match)" : "";
       log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
+      if (!this.matchFound || this.findAll) {
+        const didScroll = await this.performAutoScroll(window);
+        if (didScroll) {
+          if (this.moduleInFlight.size > 0) {
+            await this.waitForModuleWork(this.moduleWaitMs);
+          }
+          log("[Executor] Waiting for post-scroll quiescence...");
+          const postScrollStart = Date.now();
+          try {
+            await this.waitForQuiescence();
+          } catch (e) {
+            warn("[Executor] Post-scroll quiescence wait failed:", e);
+          }
+          this.timings.quiescence_ms += Date.now() - postScrollStart;
+        }
+      }
       const renderedHtml = this.serializeDocument(window);
       this.windowClosed = true;
-      try {
-        window.close();
-      } catch {
-      }
       const result = {
         logs: this.logs,
         matchedRequests: this.earlyMatches,
@@ -2153,6 +2426,7 @@ var Executor = class {
       };
       const shutdownGraceMs = this.clampMs(Number(process.env.PHANTOM_SHUTDOWN_GRACE_MS ?? 50), 10, 5e3);
       await new Promise((r) => setTimeout(r, shutdownGraceMs));
+      this.clearTrackedTimers(window);
       this.unrefNewMessagePorts(initialActiveHandles);
       return result;
     } finally {
@@ -2259,6 +2533,12 @@ var Executor = class {
     };
     window.__phantom = {
       fetch: async (input, opts = {}) => {
+        if (that.windowClosed) {
+          return new (window.Response || global.Response)("", {
+            status: 200,
+            headers: toSafeResponseHeaders({})
+          });
+        }
         const norm = await normalizeFetchInput(input, opts);
         const start = Date.now();
         const headers = {
@@ -2336,6 +2616,7 @@ var Executor = class {
         }
       },
       dynamicImport: async (url) => {
+        if (that.windowClosed) return {};
         const fullUrl = new URL(url, window.location.href).toString();
         const start = Date.now();
         const logEntry = {
@@ -2448,6 +2729,12 @@ var Executor = class {
           this.onloadend?.({ type: "loadend" });
         }
         async send(body) {
+          if (that.windowClosed) {
+            this.readyState = 4;
+            this.status = 0;
+            this.onloadend?.({ type: "loadend" });
+            return;
+          }
           const start = Date.now();
           const headers = {
             "User-Agent": window.navigator.userAgent,