@grabbit-labs/dynafetch 0.2.0 → 0.2.1

package/dist/index.js

@@ -9,6 +9,14 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
 // ../dynafetch-core/src/index.ts
 import * as net from "node:net";
 
+// ../../src/phantom/log.ts
+var enabled = process.env.DYNAFETCH_DEBUG === "1";
+var log = enabled ? console.log.bind(console) : () => {
+};
+var warn = enabled ? console.warn.bind(console) : () => {
+};
+var error = console.error.bind(console);
+
 // ../../src/phantom/execute.ts
 import { JSDOM, VirtualConsole, CookieJar } from "jsdom";
 import WebSocket from "ws";
@@ -161,11 +169,18 @@ function createWorkerCommand() {
 }
 function createWorkerTransport() {
   const { command, args, cwd } = createWorkerCommand();
-  const child = spawn(command, args, {
-    cwd,
-    stdio: ["pipe", "pipe", "pipe"],
-    env: process.env
-  });
+  let child;
+  try {
+    child = spawn(command, args, {
+      cwd,
+      stdio: ["pipe", "pipe", "pipe"],
+      env: process.env
+    });
+  } catch (err) {
+    return Promise.reject(
+      new Error(`Failed to start dynafetch-net TLS proxy: ${err instanceof Error ? err.message : String(err)}. Binary: ${command}`)
+    );
+  }
   const pending = /* @__PURE__ */ new Map();
   const rl = readline.createInterface({ input: child.stdout });
   rl.on("line", (line) => {
@@ -174,9 +189,9 @@ function createWorkerTransport() {
     let payload;
     try {
       payload = JSON.parse(trimmed);
-    } catch (error) {
+    } catch (error2) {
       for (const entry of pending.values()) {
-        entry.reject(new Error(`Invalid dynafetch-net response: ${String(error)}`));
+        entry.reject(new Error(`Invalid dynafetch-net response: ${String(error2)}`));
       }
       pending.clear();
       return;
@@ -193,7 +208,7 @@ function createWorkerTransport() {
   child.stderr.on("data", (chunk) => {
     const message = chunk.toString().trim();
     if (message) {
-      console.warn(`[dynafetch-net] ${message}`);
+      if (process.env.DYNAFETCH_DEBUG === "1") console.warn(`[dynafetch-net] ${message}`);
     }
   });
   const onExit = (code, signal) => {
@@ -204,15 +219,29 @@ function createWorkerTransport() {
     pending.clear();
     transportPromise = null;
   };
-  child.once("error", (error) => {
+  child.once("error", (error2) => {
     for (const entry of pending.values()) {
-      entry.reject(error);
+      entry.reject(error2);
     }
     pending.clear();
     transportPromise = null;
   });
   child.once("exit", onExit);
-  return Promise.resolve({ child, pending });
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    child.once("error", (err) => {
+      if (!settled) {
+        settled = true;
+        reject(new Error(`Failed to start dynafetch-net TLS proxy: ${err.message}. Binary: ${command}`));
+      }
+    });
+    child.once("spawn", () => {
+      if (!settled) {
+        settled = true;
+        resolve({ child, pending });
+      }
+    });
+  });
 }
 async function getWorkerTransport() {
   if (!transportPromise) {
@@ -220,17 +249,31 @@ async function getWorkerTransport() {
   }
   return transportPromise;
 }
-async function callWorker(method, params) {
+async function callWorker(method, params, timeoutMs = 3e4) {
   const transport = await getWorkerTransport();
   const id = randomUUID();
   const payload = JSON.stringify({ id, method, params });
   return await new Promise((resolve, reject) => {
-    transport.pending.set(id, { resolve, reject });
+    const timer = setTimeout(() => {
+      transport.pending.delete(id);
+      reject(new Error(`dynafetch-net request timed out after ${timeoutMs}ms (method: ${method})`));
+    }, timeoutMs);
+    transport.pending.set(id, {
+      resolve: (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      reject: (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    });
     transport.child.stdin.write(`${payload}
-`, (error) => {
-      if (!error) return;
+`, (error2) => {
+      if (!error2) return;
+      clearTimeout(timer);
       transport.pending.delete(id);
-      reject(error);
+      reject(error2);
     });
   });
 }
@@ -313,9 +356,9 @@ async function phantomFetch(payload) {
   }
   try {
     return await dynafetchWorkerFetch(payload);
-  } catch (error) {
+  } catch (error2) {
     if (process.env.DYNAFETCH_DISABLE_DIRECT_FALLBACK === "1") {
-      throw error;
+      throw error2;
     }
     return await directFetch(payload);
   }
@@ -337,9 +380,9 @@ async function phantomBatchFetch(payloads) {
       finalUrl: r.finalUrl,
       error: r.error
     }));
-  } catch (error) {
+  } catch (error2) {
     if (process.env.DYNAFETCH_DISABLE_DIRECT_FALLBACK === "1") {
-      throw error;
+      throw error2;
     }
     return Promise.all(payloads.map((p) => directFetch(p)));
   }
@@ -975,7 +1018,7 @@ var Executor = class {
       const taskId = this.trackTaskStart("module_bundle", cacheKey, this.moduleWaitMs);
       try {
         if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-          console.log("[Executor] Bundling module entry:", cacheKey);
+          log("[Executor] Bundling module entry:", cacheKey);
         }
         const cached = this.moduleBundleCache.get(cacheKey);
         if (cached) {
@@ -1100,7 +1143,7 @@ var Executor = class {
         this.moduleBundleCache.set(cacheKey, transformed);
         if (!this.windowClosed) window.eval(transformed);
         if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-          console.log("[Executor] Module bundle eval complete:", cacheKey);
+          log("[Executor] Module bundle eval complete:", cacheKey);
         }
       } catch (e) {
         this.recordExecutionError(e, "unhandledRejection");
@@ -1437,9 +1480,9 @@ var Executor = class {
     process.on("unhandledRejection", onNodeUnhandled);
     try {
       const virtualConsole = new VirtualConsole();
-      virtualConsole.on("log", (...args) => console.log("[JSDOM Log]", ...args));
+      virtualConsole.on("log", (...args) => log("[JSDOM Log]", ...args));
       virtualConsole.on("error", (...args) => console.error("[JSDOM Error]", ...args));
-      virtualConsole.on("warn", (...args) => console.warn("[JSDOM Warn]", ...args));
+      virtualConsole.on("warn", (...args) => warn("[JSDOM Warn]", ...args));
       const cookieJar = new CookieJar();
       this.harvestData.cookies.forEach((c) => {
         try {
@@ -1619,7 +1662,7 @@ var Executor = class {
                       if (that.handledModuleScriptUrls.has(abs)) return;
                       that.handledModuleScriptUrls.add(abs);
                       if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-                        console.log("[Executor] Detected module script:", abs);
+                        log("[Executor] Detected module script:", abs);
                       }
                       void that.handleModuleScript(abs, window2);
                       return;
@@ -1778,7 +1821,7 @@ var Executor = class {
               });
             }
           } catch (e) {
-            console.warn(`[Executor] Module script ${script.id} failed:`, e);
+            warn(`[Executor] Module script ${script.id} failed:`, e);
           } finally {
             currentScriptState.value = prevCurrentScript;
           }
@@ -1794,7 +1837,7 @@ var Executor = class {
         try {
           window.eval(code);
         } catch (e) {
-          console.warn(`[Executor] Script ${script.id} failed:`, e);
+          warn(`[Executor] Script ${script.id} failed:`, e);
         } finally {
           currentScriptState.value = prevCurrentScript;
         }
@@ -1834,16 +1877,16 @@ var Executor = class {
       if (this.moduleInFlight.size > 0) {
         await this.waitForModuleWork(this.moduleWaitMs);
       }
-      console.log("[Executor] Waiting for network quiescence...");
+      log("[Executor] Waiting for network quiescence...");
       const quiescenceStart = Date.now();
       try {
         await this.waitForQuiescence();
       } catch (e) {
-        console.warn("[Executor] Quiescence wait failed:", e);
+        warn("[Executor] Quiescence wait failed:", e);
       }
       this.timings.quiescence_ms = Date.now() - quiescenceStart;
       const reason = this.matchFound && !this.findAll ? "(early exit on match)" : "";
-      console.log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
+      log(`[Executor] Quiescence reached in ${Date.now() - quiescenceStart}ms ${reason}`);
       const renderedHtml = this.serializeDocument(window);
       this.windowClosed = true;
       try {
@@ -2106,7 +2149,7 @@ var Executor = class {
           this.asyncFlag = async !== false;
           this.aborted = false;
           if (process.env.PHANTOM_DEBUG_XHR === "1") {
-            console.log("[XHR open]", this.method, this.url);
+            log("[XHR open]", this.method, this.url);
           }
           this.readyState = 1;
           this.responseURL = this.url;
@@ -2159,7 +2202,7 @@ var Executor = class {
           };
           that.logRequest(logEntry);
           if (process.env.PHANTOM_DEBUG_XHR === "1") {
-            console.log("[XHR send]", this.method, this.url, {
+            log("[XHR send]", this.method, this.url, {
               hasBody: body != null,
               headers
             });
@@ -2326,7 +2369,9 @@ async function prefetchModuleGraph(rootUrls, cache, pageUrl, opts) {
   }
   for (let round = 0; round < maxRounds && toFetch.length > 0; round++) {
     if (process.env.PHANTOM_DEBUG_MODULES === "1") {
-      console.log(`[prefetch] Round ${round}: ${toFetch.length} modules`);
+      if (process.env.DYNAFETCH_DEBUG === "1") {
+        console.log(`[prefetch] Round ${round}: ${toFetch.length} modules`);
+      }
     }
     const payloads = toFetch.map((u) => ({
       method: "GET",
@@ -2429,7 +2474,7 @@ var Harvester = class {
           const location = data.headers["Location"] || data.headers["location"];
           if (location) {
             currentUrl = new URL(location, currentUrl).toString();
-            console.log(`[Harvest] Following redirect to: ${currentUrl}`);
+            log(`[Harvest] Following redirect to: ${currentUrl}`);
             redirectCount++;
             if (data.status === 302 || data.status === 303) {
               currentMethod = "GET";
@@ -2500,10 +2545,10 @@ var Harvester = class {
     return b.includes("just a moment") || b.includes("challenge-platform") || b.includes("__cf_chl") || b.includes("cf-browser-verification") || b.includes("enable javascript and cookies to continue") || b.includes("security verification") || b.includes("captcha") || b.includes("trkcode=") || b.includes("trkinfo=");
   }
   async harvest() {
-    console.log(`[Harvest] Fetching ${this.targetUrl} via TLS Proxy...`);
+    log(`[Harvest] Fetching ${this.targetUrl} via TLS Proxy...`);
     let response = await this.fetchViaProxy(this.targetUrl, { ...DEFAULT_HEADERS, ...this.requestHeaders }, true);
     if (response.status >= 400) {
-      console.log(`[Harvest] Response Body on Error:`, response.body.substring(0, 500));
+      log(`[Harvest] Response Body on Error:`, response.body.substring(0, 500));
       if (this.looksBlocked(response.status, response.body || "")) {
         throw new BlockedByBotProtectionError(
           this.targetUrl,
@@ -2515,7 +2560,7 @@ var Harvester = class {
     let finalUrl = response.finalUrl;
     let html = response.body;
     if (this.isConsentWall(finalUrl, html)) {
-      console.log(`[Harvest] Consent wall detected at ${finalUrl}, attempting bypass...`);
+      log(`[Harvest] Consent wall detected at ${finalUrl}, attempting bypass...`);
       const form = this.parseConsentForm(html, finalUrl);
       if (form) {
         try {
@@ -2529,32 +2574,32 @@ var Harvester = class {
           };
           const consentResp = await this.fetchViaProxy(form.action, postHeaders, true, 10, "POST", formBody);
           if (consentResp.status < 400) {
-            console.log(`[Harvest] Consent POST succeeded (${consentResp.status}), final URL: ${consentResp.finalUrl}`);
+            log(`[Harvest] Consent POST succeeded (${consentResp.status}), final URL: ${consentResp.finalUrl}`);
             if (!this.isConsentWall(consentResp.finalUrl, consentResp.body)) {
               response = consentResp;
               finalUrl = consentResp.finalUrl;
               html = consentResp.body;
-              console.log(`[Harvest] Consent bypass successful (from redirect), got real page at ${finalUrl}`);
+              log(`[Harvest] Consent bypass successful (from redirect), got real page at ${finalUrl}`);
             } else {
-              console.log(`[Harvest] Consent redirect still on consent page, re-fetching original URL...`);
+              log(`[Harvest] Consent redirect still on consent page, re-fetching original URL...`);
               const retryResp = await this.fetchViaProxy(this.targetUrl, { ...DEFAULT_HEADERS, ...this.requestHeaders }, true);
               if (retryResp.status < 400 && !this.isConsentWall(retryResp.finalUrl, retryResp.body)) {
                 response = retryResp;
                 finalUrl = retryResp.finalUrl;
                 html = retryResp.body;
-                console.log(`[Harvest] Consent bypass successful (re-fetch), got real page at ${finalUrl}`);
+                log(`[Harvest] Consent bypass successful (re-fetch), got real page at ${finalUrl}`);
               } else {
-                console.warn(`[Harvest] Re-fetch after consent still returned consent wall, proceeding with original`);
+                warn(`[Harvest] Re-fetch after consent still returned consent wall, proceeding with original`);
               }
             }
           } else {
-            console.warn(`[Harvest] Consent POST returned ${consentResp.status}, proceeding with consent page`);
+            warn(`[Harvest] Consent POST returned ${consentResp.status}, proceeding with consent page`);
           }
         } catch (e) {
-          console.warn(`[Harvest] Consent bypass failed, proceeding with consent page:`, e);
+          warn(`[Harvest] Consent bypass failed, proceeding with consent page:`, e);
         }
       } else {
-        console.warn(`[Harvest] Could not parse consent form, proceeding with consent page`);
+        warn(`[Harvest] Could not parse consent form, proceeding with consent page`);
       }
     }
     const $ = cheerio.load(html);
@@ -2663,7 +2708,7 @@ var Harvester = class {
     }));
     const allPayloads = [...scriptPayloads, ...preloadPayloads];
     if (allPayloads.length > 0) {
-      console.log(`[Harvest] Batch-fetching ${scriptPayloads.length} scripts + ${preloadPayloads.length} modulepreloads...`);
+      log(`[Harvest] Batch-fetching ${scriptPayloads.length} scripts + ${preloadPayloads.length} modulepreloads...`);
       const allResponses = await phantomBatchFetch(allPayloads);
       for (let i = 0; i < batchScriptMeta.length; i++) {
         const meta = batchScriptMeta[i];
@@ -2690,7 +2735,7 @@ var Harvester = class {
             execution: meta.execution
           });
         } else {
-          console.warn(`[Harvest] Failed to fetch script ${meta.absoluteUrl}: status ${resp.status}`);
+          warn(`[Harvest] Failed to fetch script ${meta.absoluteUrl}: status ${resp.status}`);
         }
       }
       for (let i = 0; i < modulePreloadUrls.length; i++) {
@@ -2933,8 +2978,8 @@ function toWarnings(plan, errors, options) {
     warnings.push("non-critical third-party scripts are skipped on the critical render path");
   }
   if (errors?.length) {
-    for (const error of errors.slice(0, 3)) {
-      warnings.push(`${error.source}: ${error.message}`);
+    for (const error2 of errors.slice(0, 3)) {
+      warnings.push(`${error2.source}: ${error2.message}`);
     }
   }
   return warnings;