@grabbit-labs/dynafetch 0.2.0 → 0.2.2

package/dist/index.js

@@ -1,13 +1,12 @@
 import { createRequire } from "node:module"; import { fileURLToPath as __fileURLToPath } from "node:url"; import { dirname as __dirname_fn } from "node:path"; const __filename = __fileURLToPath(import.meta.url); const __dirname = __dirname_fn(__filename); const require = createRequire(import.meta.url);
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined") return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
 
-// ../dynafetch-core/src/index.ts
-import * as net from "node:net";
+// ../../src/phantom/log.ts
+var enabled = process.env.DYNAFETCH_DEBUG === "1";
+var log = enabled ? console.log.bind(console) : () => {
+};
+var warn = enabled ? console.warn.bind(console) : () => {
+};
+var error = console.error.bind(console);
 
 // ../../src/phantom/execute.ts
 import { JSDOM, VirtualConsole, CookieJar } from "jsdom";
@@ -115,29 +114,32 @@ var Transformer = class {
 import { AsyncLocalStorage } from "node:async_hooks";
 import { spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
+import { accessSync, constants } from "node:fs";
 import path2 from "node:path";
 import readline from "node:readline";
+import { fileURLToPath } from "node:url";
 var sessionStore = new AsyncLocalStorage();
 var transportPromise = null;
+var workerDir = path2.dirname(fileURLToPath(import.meta.url));
 function findPrecompiledBinary() {
   const platform = process.platform;
-  const arch = process.arch === "x64" ? "x64" : "arm64";
+  const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : null;
+  if (!arch) return null;
   const ext = platform === "win32" ? ".exe" : "";
   const name = `dynafetch-net-${platform}-${arch}${ext}`;
   const candidates = [
-    path2.resolve(__dirname, "../bin", name),
+    path2.resolve(workerDir, "../bin", name),
     // installed: dist/../bin
-    path2.resolve(__dirname, "../../../dynafetch-net/bin", name),
+    path2.resolve(workerDir, "../../../dynafetch-net/bin", name),
     // dev: dynafetch-core/src/net -> dynafetch-net/bin
-    path2.resolve(__dirname, "../../../../packages/dynafetch-net/bin", name),
+    path2.resolve(workerDir, "../../../../packages/dynafetch-net/bin", name),
     // dev: alt layout
     path2.resolve(process.cwd(), "packages/dynafetch-net/bin", name)
     // dev: from workspace root
   ];
   for (const candidate of candidates) {
     try {
-      const fs2 = __require("fs");
-      fs2.accessSync(candidate, fs2.constants.X_OK);
+      accessSync(candidate, constants.X_OK);
       return candidate;
     } catch {
     }
@@ -161,12 +163,44 @@ function createWorkerCommand() {
 }
 function createWorkerTransport() {
   const { command, args, cwd } = createWorkerCommand();
-  const child = spawn(command, args, {
-    cwd,
-    stdio: ["pipe", "pipe", "pipe"],
-    env: process.env
-  });
+  let child;
+  try {
+    child = spawn(command, args, {
+      cwd,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+  } catch (err) {
+    return Promise.reject(
+      new Error(`Failed to start dynafetch-net TLS proxy: ${err instanceof Error ? err.message : String(err)}. Binary: ${command}`)
+    );
+  }
   const pending = /* @__PURE__ */ new Map();
+  let holdCount = 0;
+  const updateRef = () => {
+    if (pending.size === 0 && holdCount === 0) {
+      child.unref();
+      child.stdin.unref?.();
+      child.stdout.unref?.();
+      child.stderr.unref?.();
+    } else {
+      child.ref();
+      child.stdin.ref?.();
+      child.stdout.ref?.();
+      child.stderr.ref?.();
+    }
+  };
+  const hold = () => {
+    holdCount++;
+    updateRef();
+  };
+  const release = () => {
+    holdCount = Math.max(0, holdCount - 1);
+    updateRef();
+  };
+  child.stdin.on("error", () => {
+  });
+  updateRef();
   const rl = readline.createInterface({ input: child.stdout });
   rl.on("line", (line) => {
     const trimmed = line.trim();
@@ -174,16 +208,18 @@ function createWorkerTransport() {
     let payload;
     try {
       payload = JSON.parse(trimmed);
-    } catch (error) {
+    } catch (error2) {
       for (const entry of pending.values()) {
-        entry.reject(new Error(`Invalid dynafetch-net response: ${String(error)}`));
+        entry.reject(new Error(`Invalid dynafetch-net response: ${String(error2)}`));
       }
       pending.clear();
+      updateRef();
       return;
     }
     const request = pending.get(payload.id);
     if (!request) return;
     pending.delete(payload.id);
+    updateRef();
     if (payload.error) {
       request.reject(new Error(payload.error.message || payload.error.code || "dynafetch-net request failed"));
       return;
@@ -193,26 +229,42 @@ function createWorkerTransport() {
   child.stderr.on("data", (chunk) => {
     const message = chunk.toString().trim();
     if (message) {
-      console.warn(`[dynafetch-net] ${message}`);
+      if (process.env.DYNAFETCH_DEBUG === "1") console.warn(`[dynafetch-net] ${message}`);
     }
   });
   const onExit = (code, signal) => {
-    const reason = `dynafetch-net exited (code=${code ?? "null"}, signal=${signal ?? "null"})`;
-    for (const entry of pending.values()) {
-      entry.reject(new Error(reason));
+    if (pending.size > 0 && signal !== "SIGKILL") {
+      const reason = `dynafetch-net exited (code=${code ?? "null"}, signal=${signal ?? "null"})`;
+      for (const entry of pending.values()) {
+        entry.reject(new Error(reason));
+      }
     }
     pending.clear();
     transportPromise = null;
   };
-  child.once("error", (error) => {
+  child.once("error", (error2) => {
     for (const entry of pending.values()) {
-      entry.reject(error);
+      entry.reject(error2);
     }
     pending.clear();
     transportPromise = null;
   });
   child.once("exit", onExit);
-  return Promise.resolve({ child, pending });
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    child.once("error", (err) => {
+      if (!settled) {
+        settled = true;
+        reject(new Error(`Failed to start dynafetch-net TLS proxy: ${err.message}. Binary: ${command}`));
+      }
+    });
+    child.once("spawn", () => {
+      if (!settled) {
+        settled = true;
+        resolve({ child, pending, updateRef, holdCount, hold, release });
+      }
+    });
+  });
 }
 async function getWorkerTransport() {
   if (!transportPromise) {
@@ -220,27 +272,62 @@ async function getWorkerTransport() {
   }
   return transportPromise;
 }
-async function callWorker(method, params) {
+async function callWorker(method, params, timeoutMs = 3e4) {
   const transport = await getWorkerTransport();
   const id = randomUUID();
   const payload = JSON.stringify({ id, method, params });
   return await new Promise((resolve, reject) => {
-    transport.pending.set(id, { resolve, reject });
+    const timer = setTimeout(() => {
+      transport.pending.delete(id);
+      transport.updateRef();
+      reject(new Error(`dynafetch-net request timed out after ${timeoutMs}ms (method: ${method})`));
+    }, timeoutMs);
+    timer.unref();
+    transport.pending.set(id, {
+      resolve: (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      reject: (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    });
+    transport.updateRef();
     transport.child.stdin.write(`${payload}
-`, (error) => {
-      if (!error) return;
+`, (error2) => {
+      if (!error2) return;
+      clearTimeout(timer);
       transport.pending.delete(id);
-      reject(error);
+      transport.updateRef();
+      reject(error2);
     });
   });
 }
+function resolveRpcTimeoutMs(options) {
+  if (options.rpcTimeoutMs != null) {
+    return Math.max(1, Math.ceil(options.rpcTimeoutMs));
+  }
+  if (options.timeoutSeconds != null) {
+    return Math.max(1e3, Math.ceil(options.timeoutSeconds * 1e3) + 1e3);
+  }
+  return 3e4;
+}
 async function withDynafetchSession(options, run) {
-  const session = await callWorker("openSession", options);
+  const transport = await getWorkerTransport();
+  transport.hold();
+  const { rpcTimeoutMs, ...sessionOptions } = options;
+  const session = await callWorker(
+    "openSession",
+    sessionOptions,
+    resolveRpcTimeoutMs(options)
+  );
   try {
     return await sessionStore.run({ sessionId: session.sessionId }, run);
   } finally {
-    await callWorker("closeSession", { sessionId: session.sessionId }).catch(() => {
+    callWorker("closeSession", { sessionId: session.sessionId }).catch(() => {
     });
+    transport.release();
   }
 }
 async function dynafetchNetHealth() {
@@ -256,7 +343,7 @@ async function dynafetchNetFetch(request, options = {}) {
     browserProfile: options.browserProfile,
     timeoutSeconds: options.timeoutSeconds,
     proxy: options.proxy
-  });
+  }, resolveRpcTimeoutMs(options));
 }
 async function dynafetchNetBatchFetch(requests, options = {}) {
   const session = sessionStore.getStore();
@@ -268,7 +355,73 @@ async function dynafetchNetBatchFetch(requests, options = {}) {
     browserProfile: options.browserProfile,
     timeoutSeconds: options.timeoutSeconds,
     proxy: options.proxy
-  });
+  }, resolveRpcTimeoutMs(options));
+}
+
+// ../../src/phantom/url-safety.ts
+import { lookup } from "node:dns/promises";
+import * as net from "node:net";
+var BLOCKED_HOSTNAMES = /* @__PURE__ */ new Set([
+  "0.0.0.0",
+  "localhost",
+  "metadata.google.internal"
+]);
+var hostnameLookupCache = /* @__PURE__ */ new Map();
+function normalizeHostname(hostname) {
+  return hostname.trim().replace(/^\[|\]$/g, "").replace(/\.+$/g, "").toLowerCase();
+}
+function isPrivateOrLocalHost(hostname) {
+  const h = normalizeHostname(hostname);
+  if (!h) return false;
+  if (BLOCKED_HOSTNAMES.has(h) || h.endsWith(".localhost")) return true;
+  const ipVer = net.isIP(h);
+  if (!ipVer) return false;
+  if (ipVer === 4) {
+    const [a, b] = h.split(".").map((value) => Number(value));
+    if (a === 10) return true;
+    if (a === 127) return true;
+    if (a === 0) return true;
+    if (a === 169 && b === 254) return true;
+    if (a === 172 && b >= 16 && b <= 31) return true;
+    if (a === 192 && b === 168) return true;
+    return false;
+  }
+  if (h === "::1") return true;
+  if (h.startsWith("fe80:")) return true;
+  if (h.startsWith("fc") || h.startsWith("fd")) return true;
+  return false;
+}
+function assertSafeHttpUrlSync(input) {
+  let parsedUrl;
+  try {
+    parsedUrl = new URL(input);
+  } catch {
+    throw new Error("Invalid URL");
+  }
+  if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
+    throw new Error("Only http(s) URLs are allowed");
+  }
+  if (isPrivateOrLocalHost(parsedUrl.hostname)) {
+    throw new Error("Refusing to fetch local/private addresses");
+  }
+  return parsedUrl;
+}
+async function hostnameResolvesToPrivateAddress(hostname) {
+  const normalized = normalizeHostname(hostname);
+  if (!normalized || net.isIP(normalized)) return false;
+  let pending = hostnameLookupCache.get(normalized);
+  if (!pending) {
+    pending = lookup(normalized, { all: true, verbatim: true }).then((records) => records.some((record) => isPrivateOrLocalHost(record.address))).catch(() => false);
+    hostnameLookupCache.set(normalized, pending);
+  }
+  return pending;
+}
+async function assertSafeRemoteUrl(input) {
+  const parsedUrl = assertSafeHttpUrlSync(input);
+  if (await hostnameResolvesToPrivateAddress(parsedUrl.hostname)) {
+    throw new Error("Refusing to fetch local/private addresses");
+  }
+  return parsedUrl;
 }
 
 // ../../src/phantom/phantom-proxy.ts
@@ -284,64 +437,115 @@ function headersToRecord(h) {
   }
   return out;
 }
-async function directFetch(payload) {
+var DIRECT_FALLBACK_WARNING = "dynafetch-net was unavailable for one or more requests; fell back to Node fetch without TLS/browser impersonation";
+var DIRECT_PROXY_ERROR = "Direct fallback cannot honor proxy configuration; dynafetch-net is required when proxy is set";
+function createTimeoutController(timeoutMs) {
+  if (!timeoutMs || !Number.isFinite(timeoutMs)) {
+    return {
+      dispose: () => {
+      },
+      didTimeout: () => false
+    };
+  }
+  const controller = new AbortController();
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    controller.abort();
+  }, Math.max(1, Math.ceil(timeoutMs)));
+  timer.unref?.();
+  return {
+    signal: controller.signal,
+    dispose: () => clearTimeout(timer),
+    didTimeout: () => timedOut
+  };
+}
+async function directFetch(payload, options = {}) {
+  if (payload.proxy) {
+    throw new Error(DIRECT_PROXY_ERROR);
+  }
   const init = {
     method: payload.method,
     headers: payload.headers,
     redirect: "manual"
   };
   if (payload.body) init.body = payload.body;
-  const resp = await fetch(payload.url, init);
-  const body = await resp.text().catch(() => "");
-  return { status: resp.status, body, headers: headersToRecord(resp.headers) };
+  const timeout = createTimeoutController(options.timeoutMs);
+  if (timeout.signal) init.signal = timeout.signal;
+  try {
+    const resp = await fetch(payload.url, init);
+    const body = await resp.text().catch(() => "");
+    return {
+      status: resp.status,
+      body,
+      headers: headersToRecord(resp.headers),
+      transport: "direct",
+      warning: DIRECT_FALLBACK_WARNING
+    };
+  } catch (error2) {
+    if (timeout.didTimeout()) {
+      throw new Error(`dynafetch request timed out after ${Math.max(1, Math.ceil(options.timeoutMs ?? 0))}ms`);
+    }
+    throw error2;
+  } finally {
+    timeout.dispose();
+  }
 }
-async function dynafetchWorkerFetch(payload) {
+async function dynafetchWorkerFetch(payload, options = {}) {
   const response = await dynafetchNetFetch(payload, {
-    followRedirect: false
+    followRedirect: false,
+    rpcTimeoutMs: options.timeoutMs
   });
   return {
     status: response.status,
     body: response.body,
     headers: response.headers,
     finalUrl: response.finalUrl,
-    error: response.error
+    error: response.error,
+    transport: "dynafetch-net"
   };
 }
-async function phantomFetch(payload) {
+async function phantomFetch(payload, options = {}) {
+  await assertSafeRemoteUrl(payload.url);
   if (process.env.PHANTOM_DISABLE_PROXY === "1" || process.env.DYNAFETCH_DISABLE_NET === "1") {
-    return directFetch(payload);
+    return directFetch(payload, options);
   }
   try {
-    return await dynafetchWorkerFetch(payload);
-  } catch (error) {
+    return await dynafetchWorkerFetch(payload, options);
+  } catch (error2) {
     if (process.env.DYNAFETCH_DISABLE_DIRECT_FALLBACK === "1") {
-      throw error;
+      throw error2;
     }
-    return await directFetch(payload);
+    return await directFetch(payload, options);
   }
 }
-async function phantomBatchFetch(payloads) {
+async function phantomBatchFetch(payloads, options = {}) {
   if (payloads.length === 0) return [];
+  await Promise.all(payloads.map((payload) => assertSafeRemoteUrl(payload.url)));
   if (process.env.PHANTOM_DISABLE_PROXY === "1" || process.env.DYNAFETCH_DISABLE_NET === "1") {
-    return Promise.all(payloads.map((p) => directFetch(p)));
+    return Promise.all(payloads.map((payload) => directFetch(payload, options)));
   }
   try {
     const responses = await dynafetchNetBatchFetch(
       payloads,
-      { followRedirect: false }
+      {
+        followRedirect: false,
+        rpcTimeoutMs: options.timeoutMs
+      }
     );
     return responses.map((r) => ({
       status: r.status,
       body: r.body,
       headers: r.headers,
       finalUrl: r.finalUrl,
-      error: r.error
+      error: r.error,
+      transport: "dynafetch-net"
     }));
-  } catch (error) {
+  } catch (error2) {
     if (process.env.DYNAFETCH_DISABLE_DIRECT_FALLBACK === "1") {
-      throw error;
+      throw error2;
     }
-    return Promise.all(payloads.map((p) => directFetch(p)));
+    return Promise.all(payloads.map((payload) => directFetch(payload, options)));
   }
 }
 
@@ -657,6 +861,7 @@ var Executor = class {
     };
     this.executionErrors = [];
     this.thirdPartyPolicy = "skip-noncritical";
+    this.warnings = /* @__PURE__ */ new Set();
     // Early exit tracking
     this.findAll = false;
     this.fuzzyMatch = true;
@@ -673,6 +878,8 @@ var Executor = class {
       this.fuzzyMatch = options.fuzzyMatch ?? true;
       this.thirdPartyPolicy = options.thirdPartyPolicy ?? "skip-noncritical";
       this.proxy = options.proxy;
+      this.timeoutMs = options.timeoutMs;
+      this.deadlineAt = options.deadlineAt;
       this.applyDefaults(options.quiescence, options.moduleWaitMs);
     }
     if (this.targetValue !== null && this.targetValue !== void 0) {
@@ -709,6 +916,26 @@ var Executor = class {
     if (!Number.isFinite(v)) return min;
     return Math.max(min, Math.min(max, Math.trunc(v)));
   }
+  createTimeoutError() {
+    const timeoutMs = Math.max(1, Math.ceil(this.timeoutMs ?? 1));
+    return new Error(`dynafetch timed out after ${timeoutMs}ms`);
+  }
+  remainingTimeMs() {
+    if (this.deadlineAt == null) return this.timeoutMs;
+    const remaining = this.deadlineAt - Date.now();
+    if (remaining <= 0) throw this.createTimeoutError();
+    return Math.max(1, Math.ceil(remaining));
+  }
+  boundedDurationMs(durationMs) {
+    if (this.deadlineAt == null) return durationMs;
+    const remaining = this.deadlineAt - Date.now();
+    if (remaining <= 0) return 0;
+    return Math.max(0, Math.min(durationMs, Math.ceil(remaining)));
+  }
+  recordWarning(warning) {
+    if (!warning) return;
+    this.warnings.add(warning);
+  }
   applyDefaults(quiescence, moduleWaitMsOverride) {
     const hardMaxCap = this.clampMs(Number(process.env.PHANTOM_QUIESCENCE_MAX_CAP_MS ?? 8e3), 500, 6e4);
     const minWaitMs = this.clampMs(quiescence?.minWaitMs ?? 75, 0, 1e4);
@@ -839,7 +1066,7 @@ var Executor = class {
   async waitForModuleWork(timeoutMs) {
     const pending = Array.from(this.moduleInFlight.values());
     if (!pending.length) return;
-    const timeout = this.clampMs(timeoutMs, 0, 6e4);
+    const timeout = this.clampMs(this.boundedDurationMs(timeoutMs), 0, 6e4);
     if (timeout === 0) return;
     const all = Promise.allSettled(pending).then(() => {
     });
@@ -856,7 +1083,11 @@ var Executor = class {
     try {
       this.telemetry_proxy++;
       const payload = { method, url, headers, headerOrder: Object.keys(headers), body, proxy: this.proxyUrlForScope(proxyScope) };
-      return await phantomFetch(payload);
+      const response = await phantomFetch(payload, {
+        timeoutMs: this.remainingTimeMs()
+      });
+      this.recordWarning(response.warning);
+      return response;
     } catch (e) {
       return { status: 0, body: e.message, headers: {}, error: e.message };
     }
@@ -972,10 +1203,10 @@ var Executor = class {
     const existing = this.moduleInFlight.get(cacheKey);
     if (existing) return existing;
     const p = (async () => {
-      const taskId = this.trackTaskStart("module_bundle", cacheKey, this.moduleWaitMs);
+      const taskId = this.trackTaskStart("module_bundle", cacheKey, this.boundedDurationMs(this.moduleWaitMs));
       try {
         if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-          console.log("[Executor] Bundling module entry:", cacheKey);
+          log("[Executor] Bundling module entry:", cacheKey);
         }
         const cached = this.moduleBundleCache.get(cacheKey);
         if (cached) {
@@ -1100,7 +1331,7 @@ var Executor = class {
         this.moduleBundleCache.set(cacheKey, transformed);
         if (!this.windowClosed) window.eval(transformed);
         if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-          console.log("[Executor] Module bundle eval complete:", cacheKey);
+          log("[Executor] Module bundle eval complete:", cacheKey);
         }
       } catch (e) {
         this.recordExecutionError(e, "unhandledRejection");
@@ -1437,9 +1668,9 @@ var Executor = class {
     process.on("unhandledRejection", onNodeUnhandled);
     try {
       const virtualConsole = new VirtualConsole();
-      virtualConsole.on("log", (...args) => console.log("[JSDOM Log]", ...args));
+      virtualConsole.on("log", (...args) => log("[JSDOM Log]", ...args));
       virtualConsole.on("error", (...args) => console.error("[JSDOM Error]", ...args));
-      virtualConsole.on("warn", (...args) => console.warn("[JSDOM Warn]", ...args));
+      virtualConsole.on("warn", (...args) => warn("[JSDOM Warn]", ...args));
       const cookieJar = new CookieJar();
       this.harvestData.cookies.forEach((c) => {
         try {
@@ -1619,7 +1850,7 @@ var Executor = class {
                       if (that.handledModuleScriptUrls.has(abs)) return;
                       that.handledModuleScriptUrls.add(abs);
                       if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-                        console.log("[Executor] Detected module script:", abs);
+                        log("[Executor] Detected module script:", abs);
                       }
                       void that.handleModuleScript(abs, window2);
                       return;
@@ -1778,7 +2009,7 @@ var Executor = class {
               });
             }
           } catch (e) {
-            console.warn(`[Executor] Module script ${script.id} failed:`, e);
+            warn(`[Executor] Module script ${script.id} failed:`, e);
           } finally {
             currentScriptState.value = prevCurrentScript;
           }
@@ -1794,7 +2025,7 @@ var Executor = class {
         try {
           window.eval(code);
         } catch (e) {
-          console.warn(`[Executor] Script ${script.id} failed:`, e);
+          warn(`[Executor] Script ${script.id} failed:`, e);
         } finally {
           currentScriptState.value = prevCurrentScript;
         }
@@ -1834,16 +2065,16 @@ var Executor = class {
       if (this.moduleInFlight.size > 0) {
         await this.waitForModuleWork(this.moduleWaitMs);
       }
-      console.log("[Executor] Waiting for network quiescence...");
+      log("[Executor] Waiting for network quiescence...");
       const quiescenceStart = Date.now();
       try {
         await this.waitForQuiescence();
       } catch (e) {
-        console.warn("[Executor] Quiescence wait failed:", e);
+        warn("[Executor] Quiescence wait failed:", e);
       }
       this.timings.quiescence_ms = Date.now() - quiescenceStart;
       const reason = this.matchFound && !this.findAll ? "(early exit on match)" : "";
-      console.log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
+      log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
       const renderedHtml = this.serializeDocument(window);
       this.windowClosed = true;
       try {
@@ -1855,7 +2086,8 @@ var Executor = class {
         matchedRequests: this.earlyMatches,
         renderedHtml,
         timings: { ...this.timings },
-        errors: this.executionErrors.length ? this.executionErrors : void 0
+        errors: this.executionErrors.length ? this.executionErrors : void 0,
+        warnings: Array.from(this.warnings)
       };
       const shutdownGraceMs = this.clampMs(Number(process.env.PHANTOM_SHUTDOWN_GRACE_MS ?? 50), 10, 5e3);
       await new Promise((r) => setTimeout(r, shutdownGraceMs));
@@ -2106,7 +2338,7 @@ var Executor = class {
           this.asyncFlag = async !== false;
           this.aborted = false;
           if (process.env.PHANTOM_DEBUG_XHR === "1") {
-            console.log("[XHR open]", this.method, this.url);
+            log("[XHR open]", this.method, this.url);
           }
           this.readyState = 1;
           this.responseURL = this.url;
@@ -2159,7 +2391,7 @@ var Executor = class {
           };
           that.logRequest(logEntry);
           if (process.env.PHANTOM_DEBUG_XHR === "1") {
-            console.log("[XHR send]", this.method, this.url, {
+            log("[XHR send]", this.method, this.url, {
               hasBody: body != null,
               headers
             });
@@ -2326,7 +2558,9 @@ async function prefetchModuleGraph(rootUrls, cache, pageUrl, opts) {
   }
   for (let round = 0; round < maxRounds && toFetch.length > 0; round++) {
     if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-      console.log(`[prefetch] Round ${round}: ${toFetch.length} modules`);
+      if (process.env.DYNAFETCH_DEBUG === "1") {
+        console.log(`[prefetch] Round ${round}: ${toFetch.length} modules`);
+      }
     }
     const payloads = toFetch.map((u) => ({
       method: "GET",
@@ -2336,11 +2570,12 @@ async function prefetchModuleGraph(rootUrls, cache, pageUrl, opts) {
       body: "",
       proxy: opts?.proxyUrl
     }));
-    const responses = await phantomBatchFetch(payloads);
+    const responses = await phantomBatchFetch(payloads, { timeoutMs: opts?.timeoutMs });
     const newToScan = [];
     for (let i = 0; i < toFetch.length; i++) {
       const u = toFetch[i];
       const r = responses[i];
+      if (r.warning) opts?.onWarning?.(r.warning);
       if (r.status < 400 && r.body) {
         cache.set(u, r.body);
         totalFetched++;
@@ -2378,6 +2613,7 @@ var Harvester = class {
     this.requestHeaders = {};
     this.thirdPartyPolicy = "skip-noncritical";
     this.prefetchModulePreloads = true;
+    this.warnings = /* @__PURE__ */ new Set();
     this.targetUrl = url;
     this.prefetchExternalScripts = opts.prefetchExternalScripts !== false;
     this.externalScriptConcurrency = opts.externalScriptConcurrency ?? 8;
@@ -2386,6 +2622,8 @@ var Harvester = class {
     this.thirdPartyPolicy = opts.thirdPartyPolicy ?? "skip-noncritical";
     this.prefetchModulePreloads = opts.prefetchModulePreloads !== false;
     this.proxy = opts.proxy;
+    this.timeoutMs = opts.timeoutMs;
+    this.deadlineAt = opts.deadlineAt;
   }
   proxyUrlForScope(scope) {
     if (!this.proxy) return void 0;
@@ -2401,6 +2639,20 @@ var Harvester = class {
     }
     return pairs.join("; ");
   }
+  createTimeoutError() {
+    const timeoutMs = Math.max(1, Math.ceil(this.timeoutMs ?? 1));
+    return new Error(`dynafetch timed out after ${timeoutMs}ms`);
+  }
+  remainingTimeMs() {
+    if (this.deadlineAt == null) return this.timeoutMs;
+    const remaining = this.deadlineAt - Date.now();
+    if (remaining <= 0) throw this.createTimeoutError();
+    return Math.max(1, Math.ceil(remaining));
+  }
+  recordWarning(warning) {
+    if (!warning) return;
+    this.warnings.add(warning);
+  }
   async fetchViaProxy(url, headers = {}, followRedirects = false, maxRedirects = 5, method = "GET", body = "", proxyScope = "page") {
     let currentUrl = url;
     let redirectCount = 0;
@@ -2421,7 +2673,10 @@ var Harvester = class {
           body: currentBody,
           proxy: this.proxyUrlForScope(proxyScope)
         };
-        const data = await phantomFetch(payload);
+        const data = await phantomFetch(payload, {
+          timeoutMs: this.remainingTimeMs()
+        });
+        this.recordWarning(data.warning);
         if (data.error) throw new Error(`Proxy Error: ${data.error}`);
         const setCookie = data.headers["Set-Cookie"] || data.headers["set-cookie"];
         if (setCookie) this.cookies.push(setCookie);
@@ -2429,7 +2684,7 @@ var Harvester = class {
           const location = data.headers["Location"] || data.headers["location"];
           if (location) {
             currentUrl = new URL(location, currentUrl).toString();
-            console.log(`[Harvest] Following redirect to: ${currentUrl}`);
+            log(`[Harvest] Following redirect to: ${currentUrl}`);
             redirectCount++;
             if (data.status === 302 || data.status === 303) {
               currentMethod = "GET";
@@ -2500,10 +2755,10 @@ var Harvester = class {
     return b.includes("just a moment") || b.includes("challenge-platform") || b.includes("__cf_chl") || b.includes("cf-browser-verification") || b.includes("enable javascript and cookies to continue") || b.includes("security verification") || b.includes("captcha") || b.includes("trkcode=") || b.includes("trkinfo=");
   }
   async harvest() {
-    console.log(`[Harvest] Fetching ${this.targetUrl} via TLS Proxy...`);
+    log(`[Harvest] Fetching ${this.targetUrl} via TLS Proxy...`);
     let response = await this.fetchViaProxy(this.targetUrl, { ...DEFAULT_HEADERS, ...this.requestHeaders }, true);
     if (response.status >= 400) {
-      console.log(`[Harvest] Response Body on Error:`, response.body.substring(0, 500));
+      log(`[Harvest] Response Body on Error:`, response.body.substring(0, 500));
       if (this.looksBlocked(response.status, response.body || "")) {
         throw new BlockedByBotProtectionError(
           this.targetUrl,
@@ -2515,7 +2770,7 @@ var Harvester = class {
     let finalUrl = response.finalUrl;
     let html = response.body;
     if (this.isConsentWall(finalUrl, html)) {
-      console.log(`[Harvest] Consent wall detected at ${finalUrl}, attempting bypass...`);
+      log(`[Harvest] Consent wall detected at ${finalUrl}, attempting bypass...`);
       const form = this.parseConsentForm(html, finalUrl);
       if (form) {
         try {
@@ -2529,32 +2784,32 @@ var Harvester = class {
           };
           const consentResp = await this.fetchViaProxy(form.action, postHeaders, true, 10, "POST", formBody);
           if (consentResp.status < 400) {
-            console.log(`[Harvest] Consent POST succeeded (${consentResp.status}), final URL: ${consentResp.finalUrl}`);
+            log(`[Harvest] Consent POST succeeded (${consentResp.status}), final URL: ${consentResp.finalUrl}`);
             if (!this.isConsentWall(consentResp.finalUrl, consentResp.body)) {
               response = consentResp;
               finalUrl = consentResp.finalUrl;
               html = consentResp.body;
-              console.log(`[Harvest] Consent bypass successful (from redirect), got real page at ${finalUrl}`);
+              log(`[Harvest] Consent bypass successful (from redirect), got real page at ${finalUrl}`);
             } else {
-              console.log(`[Harvest] Consent redirect still on consent page, re-fetching original URL...`);
+              log(`[Harvest] Consent redirect still on consent page, re-fetching original URL...`);
               const retryResp = await this.fetchViaProxy(this.targetUrl, { ...DEFAULT_HEADERS, ...this.requestHeaders }, true);
               if (retryResp.status < 400 && !this.isConsentWall(retryResp.finalUrl, retryResp.body)) {
                 response = retryResp;
                 finalUrl = retryResp.finalUrl;
                 html = retryResp.body;
-                console.log(`[Harvest] Consent bypass successful (re-fetch), got real page at ${finalUrl}`);
+                log(`[Harvest] Consent bypass successful (re-fetch), got real page at ${finalUrl}`);
               } else {
-                console.warn(`[Harvest] Re-fetch after consent still returned consent wall, proceeding with original`);
+                warn(`[Harvest] Re-fetch after consent still returned consent wall, proceeding with original`);
               }
             }
           } else {
-            console.warn(`[Harvest] Consent POST returned ${consentResp.status}, proceeding with consent page`);
+            warn(`[Harvest] Consent POST returned ${consentResp.status}, proceeding with consent page`);
           }
         } catch (e) {
-          console.warn(`[Harvest] Consent bypass failed, proceeding with consent page:`, e);
+          warn(`[Harvest] Consent bypass failed, proceeding with consent page:`, e);
         }
       } else {
-        console.warn(`[Harvest] Could not parse consent form, proceeding with consent page`);
+        warn(`[Harvest] Could not parse consent form, proceeding with consent page`);
       }
     }
     const $ = cheerio.load(html);
@@ -2663,11 +2918,14 @@ var Harvester = class {
     }));
     const allPayloads = [...scriptPayloads, ...preloadPayloads];
     if (allPayloads.length > 0) {
-      console.log(`[Harvest] Batch-fetching ${scriptPayloads.length} scripts + ${preloadPayloads.length} modulepreloads...`);
-      const allResponses = await phantomBatchFetch(allPayloads);
+      log(`[Harvest] Batch-fetching ${scriptPayloads.length} scripts + ${preloadPayloads.length} modulepreloads...`);
+      const allResponses = await phantomBatchFetch(allPayloads, {
+        timeoutMs: this.remainingTimeMs()
+      });
       for (let i = 0; i < batchScriptMeta.length; i++) {
         const meta = batchScriptMeta[i];
         const resp = allResponses[i];
+        this.recordWarning(resp.warning);
         const logEntry = {
           type: "resource_load",
           url: meta.absoluteUrl,
@@ -2690,12 +2948,13 @@ var Harvester = class {
             execution: meta.execution
           });
         } else {
-          console.warn(`[Harvest] Failed to fetch script ${meta.absoluteUrl}: status ${resp.status}`);
+          warn(`[Harvest] Failed to fetch script ${meta.absoluteUrl}: status ${resp.status}`);
         }
       }
       for (let i = 0; i < modulePreloadUrls.length; i++) {
         const url = modulePreloadUrls[i];
         const resp = allResponses[batchScriptMeta.length + i];
+        this.recordWarning(resp.warning);
         const logEntry = {
           type: "resource_load",
           url,
@@ -2760,7 +3019,9 @@ var Harvester = class {
     if (moduleEntryUrls.length > 0 || modulePreloads.length > 0) {
       const rootUrls = [...moduleEntryUrls, ...modulePreloads.map((mp) => mp.url)];
       await prefetchModuleGraph(rootUrls, moduleGraphCache, finalUrl, {
-        proxyUrl: this.proxyUrlForScope("assets")
+        proxyUrl: this.proxyUrlForScope("assets"),
+        timeoutMs: this.remainingTimeMs(),
+        onWarning: (warning) => this.recordWarning(warning)
       });
     }
     return {
@@ -2774,7 +3035,8 @@ var Harvester = class {
       cookies: this.cookies,
       headers: response.headers,
       logs: this.logs,
-      moduleGraphCache
+      moduleGraphCache,
+      warnings: Array.from(this.warnings)
     };
   }
 };
@@ -2851,27 +3113,6 @@ var DynafetchInputError = class extends Error {
     this.status = status;
   }
 };
-function isPrivateOrLocalHost(hostname) {
-  const h = hostname.toLowerCase();
-  if (h === "localhost" || h.endsWith(".localhost") || h === "0.0.0.0") return true;
-  if (h === "metadata.google.internal") return true;
-  const ipVer = net.isIP(h);
-  if (!ipVer) return false;
-  if (ipVer === 4) {
-    const [a, b] = h.split(".").map((x) => Number(x));
-    if (a === 10) return true;
-    if (a === 127) return true;
-    if (a === 0) return true;
-    if (a === 169 && b === 254) return true;
-    if (a === 172 && b >= 16 && b <= 31) return true;
-    if (a === 192 && b === 168) return true;
-    return false;
-  }
-  if (h === "::1") return true;
-  if (h.startsWith("fe80:")) return true;
-  if (h.startsWith("fc") || h.startsWith("fd")) return true;
-  return false;
-}
 function normalizeProxy(input) {
   if (!input) return void 0;
   if (typeof input === "string") {
@@ -2901,15 +3142,10 @@ function normalizeOptions(input) {
   }
   let parsedUrl;
   try {
-    parsedUrl = new URL(options.url);
-  } catch {
-    throw new DynafetchInputError("Invalid URL");
-  }
-  if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
-    throw new DynafetchInputError("Only http(s) URLs are allowed");
-  }
-  if (isPrivateOrLocalHost(parsedUrl.hostname)) {
-    throw new DynafetchInputError("Refusing to fetch local/private addresses");
+    parsedUrl = assertSafeHttpUrlSync(options.url);
+  } catch (error2) {
+    const message = error2 instanceof Error ? error2.message : "Invalid URL";
+    throw new DynafetchInputError(message);
   }
   return {
     ...options,
@@ -2921,8 +3157,13 @@ function normalizeOptions(input) {
     thirdPartyPolicy: options.thirdPartyPolicy ?? "skip-noncritical"
   };
 }
-function toWarnings(plan, errors, options) {
+function toWarnings(plan, errors, options, runtimeWarnings = []) {
   const warnings = [plan.reason];
+  for (const warning of runtimeWarnings) {
+    if (warning && !warnings.includes(warning)) {
+      warnings.push(warning);
+    }
+  }
   if (plan.strategy === "jsdom-fallback" || plan.strategy === "framework-probe") {
     warnings.push("runtime execution used the legacy JSDOM-based renderer while lightweight adapters are still being built");
   }
@@ -2933,8 +3174,8 @@ function toWarnings(plan, errors, options) {
     warnings.push("non-critical third-party scripts are skipped on the critical render path");
   }
   if (errors?.length) {
-    for (const error of errors.slice(0, 3)) {
-      warnings.push(`${error.source}: ${error.message}`);
+    for (const error2 of errors.slice(0, 3)) {
+      warnings.push(`${error2.source}: ${error2.message}`);
     }
   }
   return warnings;
@@ -2956,85 +3197,125 @@ function computeConfidence(params) {
   confidence -= Math.min(0.28, params.executionErrors * 0.07);
   return Math.max(0.05, Math.min(0.98, Number(confidence.toFixed(2))));
 }
+function createTimeoutError(timeoutMs) {
+  const error2 = new Error(`dynafetch timed out after ${timeoutMs}ms`);
+  error2.name = "DynafetchTimeoutError";
+  return error2;
+}
+async function withOperationTimeout(operation, timeoutMs) {
+  if (!timeoutMs || !Number.isFinite(timeoutMs)) {
+    return await operation;
+  }
+  return await new Promise((resolve, reject) => {
+    const timer = setTimeout(() => reject(createTimeoutError(Math.max(1, Math.ceil(timeoutMs)))), Math.max(1, Math.ceil(timeoutMs)));
+    timer.unref?.();
+    operation.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (error2) => {
+        clearTimeout(timer);
+        reject(error2);
+      }
+    );
+  });
+}
 async function dynafetch(input) {
   const options = normalizeOptions(input);
   const timeoutSeconds = options.timeoutMs ? Math.max(1, Math.ceil(options.timeoutMs / 1e3)) : void 0;
+  const deadlineAt = options.timeoutMs ? Date.now() + options.timeoutMs : void 0;
   const initialCookies = normalizeCookies(options.cookies);
   const proxy = normalizeProxy(options.proxy);
-  return await withDynafetchSession(
-    {
-      browserProfile: options.browserProfile,
-      timeoutSeconds,
-      proxy: proxy?.url
-    },
-    async () => {
-      const totalStart = Date.now();
-      const harvestStart = Date.now();
-      const harvester = new Harvester(options.url, {
-        prefetchExternalScripts: options.prefetchExternalScripts,
-        prefetchModulePreloads: options.prefetchModulePreloads,
-        requestHeaders: options.headers,
-        initialCookies,
-        thirdPartyPolicy: options.thirdPartyPolicy,
-        proxy
-      });
-      const harvest = await harvester.harvest();
-      const harvestMs = Date.now() - harvestStart;
-      const framework = detectFramework(harvest);
-      const plan = planDynafetch(framework, harvest, options.allowJsdomFallback !== false);
-      let html = harvest.html;
-      let requestCount = harvest.logs.length;
-      let executionErrors;
-      let executeMs = 0;
-      let quiescenceMs = 0;
-      let scriptsTransformed = 0;
-      if (plan.strategy !== "static-html") {
-        const executeStart = Date.now();
-        const executor = new Executor(harvest, {
+  return await withOperationTimeout(
+    withDynafetchSession(
+      {
+        browserProfile: options.browserProfile,
+        timeoutSeconds,
+        proxy: proxy?.url,
+        rpcTimeoutMs: options.timeoutMs
+      },
+      async () => {
+        const totalStart = Date.now();
+        const harvestStart = Date.now();
+        const harvester = new Harvester(options.url, {
+          prefetchExternalScripts: options.prefetchExternalScripts,
+          prefetchModulePreloads: options.prefetchModulePreloads,
+          requestHeaders: options.headers,
+          initialCookies,
           thirdPartyPolicy: options.thirdPartyPolicy,
-          quiescence: {
-            minWaitMs: options.minWaitMs,
-            idleWaitMs: options.idleWaitMs,
-            maxWaitMs: options.maxWaitMs
-          },
-          moduleWaitMs: options.moduleWaitMs,
-          proxy
+          proxy,
+          timeoutMs: options.timeoutMs,
+          deadlineAt
         });
-        const execution = await executor.execute();
-        executeMs = Date.now() - executeStart;
-        html = execution.renderedHtml ?? harvest.html;
-        requestCount = execution.logs.length;
-        executionErrors = execution.errors;
-        quiescenceMs = execution.timings?.quiescence_ms ?? 0;
-        scriptsTransformed = execution.timings?.scripts_transformed_count ?? 0;
-      }
-      const totalMs = Date.now() - totalStart;
-      const warnings = toWarnings(plan, executionErrors, options);
-      const confidence = computeConfidence({
-        plan,
-        initialStateCount: Object.keys(harvest.initialState).length,
-        executionErrors: executionErrors?.length ?? 0,
-        htmlLength: html.length
-      });
-      return {
-        url: options.url,
-        finalUrl: harvest.url,
-        status: harvest.status,
-        html,
-        framework,
-        strategy: plan.strategy,
-        confidence,
-        warnings,
-        timings: {
-          total: totalMs,
-          harvest: harvestMs,
-          execute: executeMs,
-          quiescence: quiescenceMs,
-          scriptsTransformed
-        },
-        requestCount
-      };
-    }
+        const harvest = await harvester.harvest();
+        const harvestMs = Date.now() - harvestStart;
+        const framework = detectFramework(harvest);
+        const plan = planDynafetch(framework, harvest, options.allowJsdomFallback !== false);
+        let html = harvest.html;
+        let requestCount = harvest.logs.length;
+        let executionErrors;
+        let executionWarnings = [];
+        let executeMs = 0;
+        let quiescenceMs = 0;
+        let scriptsTransformed = 0;
+        if (plan.strategy !== "static-html") {
+          const executeStart = Date.now();
+          const executor = new Executor(harvest, {
+            thirdPartyPolicy: options.thirdPartyPolicy,
+            quiescence: {
+              minWaitMs: options.minWaitMs,
+              idleWaitMs: options.idleWaitMs,
+              maxWaitMs: options.maxWaitMs
+            },
+            moduleWaitMs: options.moduleWaitMs,
+            proxy,
+            timeoutMs: options.timeoutMs,
+            deadlineAt
+          });
+          const execution = await executor.execute();
+          executeMs = Date.now() - executeStart;
+          html = execution.renderedHtml ?? harvest.html;
+          requestCount = execution.logs.length;
+          executionErrors = execution.errors;
+          executionWarnings = execution.warnings ?? [];
+          quiescenceMs = execution.timings?.quiescence_ms ?? 0;
+          scriptsTransformed = execution.timings?.scripts_transformed_count ?? 0;
+        }
+        const totalMs = Date.now() - totalStart;
+        const warnings = toWarnings(
+          plan,
+          executionErrors,
+          options,
+          [...harvest.warnings ?? [], ...executionWarnings]
+        );
+        const confidence = computeConfidence({
+          plan,
+          initialStateCount: Object.keys(harvest.initialState).length,
+          executionErrors: executionErrors?.length ?? 0,
+          htmlLength: html.length
+        });
+        return {
+          url: options.url,
+          finalUrl: harvest.url,
+          status: harvest.status,
+          html,
+          framework,
+          strategy: plan.strategy,
+          confidence,
+          warnings,
+          timings: {
+            total: totalMs,
+            harvest: harvestMs,
+            execute: executeMs,
+            quiescence: quiescenceMs,
+            scriptsTransformed
+          },
+          requestCount
+        };
+      }
+    ),
+    options.timeoutMs
   );
 }
 export {