@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/fetch/browser-types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/fetch/browser-types.ts"],"sourcesContent":["import type { BrowserType } from '../types.js';\n\nconst VALID_BROWSER_TYPES: ReadonlySet<string> = new Set(['chromium', 'firefox', 'webkit', 'lightpanda']);\n\nconst DEFAULT_BROWSER_TYPES: BrowserType[] = ['chromium'];\n\nfunction warn(msg: string, data: Record<string, unknown>): void {\n  const line = JSON.stringify({ ts: new Date().toISOString(), level: 'warn', msg, module: 'fetch', data });\n  process.stderr.write(line + '\\n');\n}\n\nexport function parseBrowserTypes(input: string | undefined | null): BrowserType[] {\n  if (!input || typeof input !== 'string') {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const trimmed = input.trim();\n  if (trimmed.length === 0) {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const parts = trimmed\n    .split(',')\n    .map(s => s.trim())\n    .filter(s => s.length > 0);\n\n  if (parts.length === 0) {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const seen = new Set<string>();\n  const valid: BrowserType[] = [];\n  const invalid: string[] = [];\n\n  for (const part of parts) {\n    if (!VALID_BROWSER_TYPES.has(part)) {\n      invalid.push(part);\n      continue;\n    }\n    if (seen.has(part)) {\n      continue;\n    }\n    seen.add(part);\n    valid.push(part as BrowserType);\n  }\n\n  if (invalid.length > 0) {\n    warn('ignored invalid browser types', { invalid, valid: [...valid] });\n  }\n\n  if (valid.length === 0) {\n    warn('no valid browser types found, falling back to chromium', { input });\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  return valid;\n}\n"],"mappings":"AAEA,MAAM,sBAA2C,oBAAI,IAAI,CAAC,YAAY,WAAW,UAAU,YAAY,CAAC;AAExG,MAAM,wBAAuC,CAAC,UAAU;AAExD,SAAS,KAAK,KAAa,MAAqC;AAC9D,QAAM,OAAO,KAAK,UAAU,EAAE,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,OAAO,QAAQ,KAAK,QAAQ,SAAS,KAAK,CAAC;AACvG,UAAQ,OAAO,MAAM,OAAO,IAAI;AAClC;AAEO,SAAS,kBAAkB,OAAiD;AACjF,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,QAAQ,QACX,MAAM,GAAG,EACT,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAK,EAAE,SAAS,CAAC;AAE3B,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,QAAuB,CAAC;AAC9B,QAAM,UAAoB,CAAC;AAE3B,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,oBAAoB,IAAI,IAAI,GAAG;AAClC,cAAQ,KAAK,IAAI;AACjB;AAAA,IACF;AACA,QAAI,KAAK,IAAI,IAAI,GAAG;AAClB;AAAA,IACF;AACA,SAAK,IAAI,IAAI;AACb,UAAM,KAAK,IAAmB;AAAA,EAChC;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,SAAK,iCAAiC,EAAE,SAAS,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,EACtE;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,SAAK,0DAA0D,EAAE,MAAM,CAAC;AACxE,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,SAAO;AACT;","names":[]}

package/dist/fetch/cdp-client.d.ts

@@ -0,0 +1,9 @@
+import type { CDPSession } from '../types.js';
+export interface DiscoverOptions {
+    timeoutMs?: number;
+    filterPages?: boolean;
+}
+export declare function parseCDPResponse(raw: string): CDPSession[];
+export declare function discoverSessions(cdpUrl: string, options?: DiscoverOptions): Promise<CDPSession[]>;
+export declare function isCDPReachable(cdpUrl: string, timeoutMs?: number): Promise<boolean>;
+//# sourceMappingURL=cdp-client.d.ts.map

package/dist/fetch/cdp-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdp-client.d.ts","sourceRoot":"","sources":["../../src/fetch/cdp-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAM9C,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAiCD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,EAAE,CA2B1D;AAED,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC,CA6BvB;AAED,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,SAAqB,GAAG,OAAO,CAAC,OAAO,CAAC,CAQrG"}

package/dist/fetch/cdp-client.js

@@ -0,0 +1,89 @@
+import * as http from "node:http";
+import { createLogger } from "../logger.js";
+const log = createLogger("fetch");
+const DEFAULT_TIMEOUT_MS = 3e3;
+function httpGet(url, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const req = http.get(url, { timeout: timeoutMs }, (res) => {
+      let body = "";
+      res.on("data", (chunk) => {
+        body += chunk;
+      });
+      res.on("end", () => {
+        resolve({ statusCode: res.statusCode ?? 0, body });
+      });
+    });
+    req.on("error", (err) => {
+      reject(err);
+    });
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error(`CDP request timed out after ${timeoutMs}ms`));
+    });
+  });
+}
+function parseCDPResponse(raw) {
+  try {
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) {
+      log.debug("CDP response is not an array");
+      return [];
+    }
+    return parsed.filter((entry) => {
+      if (!entry || typeof entry !== "object") return false;
+      if (!entry.id || typeof entry.id !== "string") return false;
+      if (!entry.webSocketDebuggerUrl || typeof entry.webSocketDebuggerUrl !== "string") return false;
+      return true;
+    }).map((entry) => ({
+      id: entry.id,
+      url: entry.url ?? "",
+      title: entry.title ?? "",
+      webSocketDebuggerUrl: entry.webSocketDebuggerUrl,
+      type: entry.type,
+      devtoolsFrontendUrl: entry.devtoolsFrontendUrl
+    }));
+  } catch (err) {
+    log.debug("failed to parse CDP response", { error: err instanceof Error ? err.message : String(err) });
+    return [];
+  }
+}
+async function discoverSessions(cdpUrl, options) {
+  const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const filterPages = options?.filterPages ?? false;
+  const jsonUrl = cdpUrl.endsWith("/") ? `${cdpUrl}json` : `${cdpUrl}/json`;
+  try {
+    log.debug("discovering CDP sessions", { url: jsonUrl, timeoutMs });
+    const { statusCode, body } = await httpGet(jsonUrl, timeoutMs);
+    if (statusCode !== 200) {
+      log.warn("CDP endpoint returned non-200", { statusCode, url: jsonUrl });
+      return [];
+    }
+    let sessions = parseCDPResponse(body);
+    if (filterPages) {
+      sessions = sessions.filter((s) => s.type === "page" || s.type === void 0);
+    }
+    log.info("CDP sessions discovered", { count: sessions.length, url: jsonUrl });
+    return sessions;
+  } catch (err) {
+    log.debug("CDP discovery failed", {
+      url: jsonUrl,
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return [];
+  }
+}
+async function isCDPReachable(cdpUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const jsonUrl = cdpUrl.endsWith("/") ? `${cdpUrl}json` : `${cdpUrl}/json`;
+  try {
+    const { statusCode } = await httpGet(jsonUrl, timeoutMs);
+    return statusCode === 200;
+  } catch {
+    return false;
+  }
+}
+export {
+  discoverSessions,
+  isCDPReachable,
+  parseCDPResponse
+};
+//# sourceMappingURL=cdp-client.js.map

package/dist/fetch/cdp-client.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/fetch/cdp-client.ts"],"sourcesContent":["import * as http from 'node:http';\nimport { createLogger } from '../logger.js';\nimport type { CDPSession } from '../types.js';\n\nconst log = createLogger('fetch');\n\nconst DEFAULT_TIMEOUT_MS = 3000;\n\nexport interface DiscoverOptions {\n  timeoutMs?: number;\n  filterPages?: boolean;\n}\n\nfunction httpGet(url: string, timeoutMs: number): Promise<{ statusCode: number; body: string }> {\n  return new Promise((resolve, reject) => {\n    const req = http.get(url, { timeout: timeoutMs }, (res) => {\n      let body = '';\n      res.on('data', (chunk: string) => { body += chunk; });\n      res.on('end', () => {\n        resolve({ statusCode: res.statusCode ?? 0, body });\n      });\n    });\n\n    req.on('error', (err) => {\n      reject(err);\n    });\n\n    req.on('timeout', () => {\n      req.destroy();\n      reject(new Error(`CDP request timed out after ${timeoutMs}ms`));\n    });\n  });\n}\n\ninterface RawCDPEntry {\n  id?: string;\n  url?: string;\n  title?: string;\n  webSocketDebuggerUrl?: string;\n  type?: string;\n  devtoolsFrontendUrl?: string;\n  description?: string;\n}\n\nexport function parseCDPResponse(raw: string): CDPSession[] {\n  try {\n    const parsed = JSON.parse(raw);\n    if (!Array.isArray(parsed)) {\n      log.debug('CDP response is not an array');\n      return [];\n    }\n\n    return parsed\n      .filter((entry: RawCDPEntry) => {\n        if (!entry || typeof entry !== 'object') return false;\n        if (!entry.id || typeof entry.id !== 'string') return false;\n        if (!entry.webSocketDebuggerUrl || typeof entry.webSocketDebuggerUrl !== 'string') return false;\n        return true;\n      })\n      .map((entry: RawCDPEntry) => ({\n        id: entry.id!,\n        url: entry.url ?? '',\n        title: entry.title ?? '',\n        webSocketDebuggerUrl: entry.webSocketDebuggerUrl!,\n        type: entry.type,\n        devtoolsFrontendUrl: entry.devtoolsFrontendUrl,\n      }));\n  } catch (err) {\n    log.debug('failed to parse CDP response', { error: err instanceof Error ? err.message : String(err) });\n    return [];\n  }\n}\n\nexport async function discoverSessions(\n  cdpUrl: string,\n  options?: DiscoverOptions,\n): Promise<CDPSession[]> {\n  const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n  const filterPages = options?.filterPages ?? false;\n  const jsonUrl = cdpUrl.endsWith('/') ? `${cdpUrl}json` : `${cdpUrl}/json`;\n\n  try {\n    log.debug('discovering CDP sessions', { url: jsonUrl, timeoutMs });\n    const { statusCode, body } = await httpGet(jsonUrl, timeoutMs);\n\n    if (statusCode !== 200) {\n      log.warn('CDP endpoint returned non-200', { statusCode, url: jsonUrl });\n      return [];\n    }\n\n    let sessions = parseCDPResponse(body);\n\n    if (filterPages) {\n      sessions = sessions.filter(s => s.type === 'page' || s.type === undefined);\n    }\n\n    log.info('CDP sessions discovered', { count: sessions.length, url: jsonUrl });\n    return sessions;\n  } catch (err) {\n    log.debug('CDP discovery failed', {\n      url: jsonUrl,\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return [];\n  }\n}\n\nexport async function isCDPReachable(cdpUrl: string, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<boolean> {\n  const jsonUrl = cdpUrl.endsWith('/') ? `${cdpUrl}json` : `${cdpUrl}/json`;\n  try {\n    const { statusCode } = await httpGet(jsonUrl, timeoutMs);\n    return statusCode === 200;\n  } catch {\n    return false;\n  }\n}\n"],"mappings":"AAAA,YAAY,UAAU;AACtB,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,qBAAqB;AAO3B,SAAS,QAAQ,KAAa,WAAkE;AAC9F,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,MAAM,KAAK,IAAI,KAAK,EAAE,SAAS,UAAU,GAAG,CAAC,QAAQ;AACzD,UAAI,OAAO;AACX,UAAI,GAAG,QAAQ,CAAC,UAAkB;AAAE,gBAAQ;AAAA,MAAO,CAAC;AACpD,UAAI,GAAG,OAAO,MAAM;AAClB,gBAAQ,EAAE,YAAY,IAAI,cAAc,GAAG,KAAK,CAAC;AAAA,MACnD,CAAC;AAAA,IACH,CAAC;AAED,QAAI,GAAG,SAAS,CAAC,QAAQ;AACvB,aAAO,GAAG;AAAA,IACZ,CAAC;AAED,QAAI,GAAG,WAAW,MAAM;AACtB,UAAI,QAAQ;AACZ,aAAO,IAAI,MAAM,+BAA+B,SAAS,IAAI,CAAC;AAAA,IAChE,CAAC;AAAA,EACH,CAAC;AACH;AAYO,SAAS,iBAAiB,KAA2B;AAC1D,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,QAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAI,MAAM,8BAA8B;AACxC,aAAO,CAAC;AAAA,IACV;AAEA,WAAO,OACJ,OAAO,CAAC,UAAuB;AAC9B,UAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,UAAI,CAAC,MAAM,MAAM,OAAO,MAAM,OAAO,SAAU,QAAO;AACtD,UAAI,CAAC,MAAM,wBAAwB,OAAO,MAAM,yBAAyB,SAAU,QAAO;AAC1F,aAAO;AAAA,IACT,CAAC,EACA,IAAI,CAAC,WAAwB;AAAA,MAC5B,IAAI,MAAM;AAAA,MACV,KAAK,MAAM,OAAO;AAAA,MAClB,OAAO,MAAM,SAAS;AAAA,MACtB,sBAAsB,MAAM;AAAA,MAC5B,MAAM,MAAM;AAAA,MACZ,qBAAqB,MAAM;AAAA,IAC7B,EAAE;AAAA,EACN,SAAS,KAAK;AACZ,QAAI,MAAM,gCAAgC,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;AACrG,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,iBACpB,QACA,SACuB;AACvB,QAAM,YAAY,SAAS,aAAa;AACxC,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,UAAU,OAAO,SAAS,GAAG,IAAI,GAAG,MAAM,SAAS,GAAG,MAAM;AAElE,MAAI;AACF,QAAI,MAAM,4BAA4B,EAAE,KAAK,SAAS,UAAU,CAAC;AACjE,UAAM,EAAE,YAAY,KAAK,IAAI,MAAM,QAAQ,SAAS,SAAS;AAE7D,QAAI,eAAe,KAAK;AACtB,UAAI,KAAK,iCAAiC,EAAE,YAAY,KAAK,QAAQ,CAAC;AACtE,aAAO,CAAC;AAAA,IACV;AAEA,QAAI,WAAW,iBAAiB,IAAI;AAEpC,QAAI,aAAa;AACf,iBAAW,SAAS,OAAO,OAAK,EAAE,SAAS,UAAU,EAAE,SAAS,MAAS;AAAA,IAC3E;AAEA,QAAI,KAAK,2BAA2B,EAAE,OAAO,SAAS,QAAQ,KAAK,QAAQ,CAAC;AAC5E,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,wBAAwB;AAAA,MAChC,KAAK;AAAA,MACL,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,eAAe,QAAgB,YAAY,oBAAsC;AACrG,QAAM,UAAU,OAAO,SAAS,GAAG,IAAI,GAAG,MAAM,SAAS,GAAG,MAAM;AAClE,MAAI;AACF,UAAM,EAAE,WAAW,IAAI,MAAM,QAAQ,SAAS,SAAS;AACvD,WAAO,eAAe;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/fetch/content-check.js

@@ -1,62 +1,55 @@
 const VISIBLE_TEXT_THRESHOLD = 200;
 const SCRIPT_RATIO_THRESHOLD = 0.8;
 function stripScriptsAndStyles(html) {
-    return html
-        .replace(/<script[\s\S]*?<\/script>/gi, '')
-        .replace(/<style[\s\S]*?<\/style>/gi, '');
+  return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
 }
 function extractVisibleText(html) {
-    const stripped = stripScriptsAndStyles(html);
-    const noTags = stripped.replace(/<[^>]+>/g, ' ');
-    return noTags.replace(/\s+/g, ' ').trim();
+  const stripped = stripScriptsAndStyles(html);
+  const noTags = stripped.replace(/<[^>]+>/g, " ");
+  return noTags.replace(/\s+/g, " ").trim();
 }
 function hasSpaShellIndicator(html) {
-    const spaPatterns = [
-        /<div[^>]+id=["']root["'][^>]*>\s*<\/div>/i,
-        /<div[^>]+id=["']app["'][^>]*>\s*<\/div>/i,
-        /<div[^>]+id=["']__next["'][^>]*>\s*<\/div>/i,
-    ];
-    return spaPatterns.some((pattern) => pattern.test(html));
+  const spaPatterns = [
+    /<div[^>]+id=["']root["'][^>]*>\s*<\/div>/i,
+    /<div[^>]+id=["']app["'][^>]*>\s*<\/div>/i,
+    /<div[^>]+id=["']__next["'][^>]*>\s*<\/div>/i
+  ];
+  return spaPatterns.some((pattern) => pattern.test(html));
 }
 function hasNextData(html) {
-    if (!/__NEXT_DATA__/.test(html))
-        return false;
-    const withoutScripts = stripScriptsAndStyles(html);
-    const visibleText = withoutScripts.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
-    return visibleText.length < VISIBLE_TEXT_THRESHOLD;
+  if (!/__NEXT_DATA__/.test(html)) return false;
+  const withoutScripts = stripScriptsAndStyles(html);
+  const visibleText = withoutScripts.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  return visibleText.length < VISIBLE_TEXT_THRESHOLD;
 }
 function hasNoscriptRequired(html) {
-    const noscriptMatches = html.match(/<noscript[^>]*>([\s\S]*?)<\/noscript>/gi);
-    if (!noscriptMatches)
-        return false;
-    return noscriptMatches.some((tag) => {
-        const inner = tag.replace(/<[^>]+>/g, '').toLowerCase();
-        return inner.includes('javascript') || inner.includes('enable');
-    });
+  const noscriptMatches = html.match(/<noscript[^>]*>([\s\S]*?)<\/noscript>/gi);
+  if (!noscriptMatches) return false;
+  return noscriptMatches.some((tag) => {
+    const inner = tag.replace(/<[^>]+>/g, "").toLowerCase();
+    return inner.includes("javascript") || inner.includes("enable");
+  });
 }
 function hasHighScriptRatio(html) {
-    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-    const bodyContent = bodyMatch ? bodyMatch[1] : html;
-    const scriptMatches = bodyContent.match(/<script[\s\S]*?<\/script>/gi) ?? [];
-    const scriptText = scriptMatches.join('');
-    const scriptLen = scriptText.length;
-    const totalLen = bodyContent.length;
-    if (totalLen === 0)
-        return false;
-    return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+  const bodyContent = bodyMatch ? bodyMatch[1] : html;
+  const scriptMatches = bodyContent.match(/<script[\s\S]*?<\/script>/gi) ?? [];
+  const scriptText = scriptMatches.join("");
+  const scriptLen = scriptText.length;
+  const totalLen = bodyContent.length;
+  if (totalLen === 0) return false;
+  return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;
 }
-export function contentAppearsEmpty(html) {
-    const visibleText = extractVisibleText(html);
-    if (visibleText.length < VISIBLE_TEXT_THRESHOLD)
-        return true;
-    if (hasSpaShellIndicator(html))
-        return true;
-    if (hasNextData(html))
-        return true;
-    if (hasNoscriptRequired(html))
-        return true;
-    if (hasHighScriptRatio(html))
-        return true;
-    return false;
+function contentAppearsEmpty(html) {
+  const visibleText = extractVisibleText(html);
+  if (visibleText.length < VISIBLE_TEXT_THRESHOLD) return true;
+  if (hasSpaShellIndicator(html)) return true;
+  if (hasNextData(html)) return true;
+  if (hasNoscriptRequired(html)) return true;
+  if (hasHighScriptRatio(html)) return true;
+  return false;
 }
+export {
+  contentAppearsEmpty
+};
 //# sourceMappingURL=content-check.js.map

package/dist/fetch/content-check.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-check.js","sourceRoot":"","sources":["../../src/fetch/content-check.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,WAAW,GAAG;QAClB,2CAA2C;QAC3C,0CAA0C;QAC1C,6CAA6C;KAC9C,CAAC;IACF,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,OAAO,WAAW,CAAC,MAAM,GAAG,sBAAsB,CAAC;AACrD,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC9E,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;IACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC;IAEpC,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,OAAO,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,WAAW,CAAC,MAAM,GAAG,sBAAsB;QAAE,OAAO,IAAI,CAAC;IAE7D,IAAI,oBAAoB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,IAAI,WAAW,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"sources":["../../src/fetch/content-check.ts"],"sourcesContent":["const VISIBLE_TEXT_THRESHOLD = 200;\nconst SCRIPT_RATIO_THRESHOLD = 0.8;\n\nfunction stripScriptsAndStyles(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, '')\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, '');\n}\n\nfunction extractVisibleText(html: string): string {\n  const stripped = stripScriptsAndStyles(html);\n  const noTags = stripped.replace(/<[^>]+>/g, ' ');\n  return noTags.replace(/\\s+/g, ' ').trim();\n}\n\nfunction hasSpaShellIndicator(html: string): boolean {\n  const spaPatterns = [\n    /<div[^>]+id=[\"']root[\"'][^>]*>\\s*<\\/div>/i,\n    /<div[^>]+id=[\"']app[\"'][^>]*>\\s*<\\/div>/i,\n    /<div[^>]+id=[\"']__next[\"'][^>]*>\\s*<\\/div>/i,\n  ];\n  return spaPatterns.some((pattern) => pattern.test(html));\n}\n\nfunction hasNextData(html: string): boolean {\n  if (!/__NEXT_DATA__/.test(html)) return false;\n  const withoutScripts = stripScriptsAndStyles(html);\n  const visibleText = withoutScripts.replace(/<[^>]+>/g, ' ').replace(/\\s+/g, ' ').trim();\n  return visibleText.length < VISIBLE_TEXT_THRESHOLD;\n}\n\nfunction hasNoscriptRequired(html: string): boolean {\n  const noscriptMatches = html.match(/<noscript[^>]*>([\\s\\S]*?)<\\/noscript>/gi);\n  if (!noscriptMatches) return false;\n  return noscriptMatches.some((tag) => {\n    const inner = tag.replace(/<[^>]+>/g, '').toLowerCase();\n    return inner.includes('javascript') || inner.includes('enable');\n  });\n}\n\nfunction hasHighScriptRatio(html: string): boolean {\n  const bodyMatch = html.match(/<body[^>]*>([\\s\\S]*?)<\\/body>/i);\n  const bodyContent = bodyMatch ? bodyMatch[1] : html;\n\n  const scriptMatches = bodyContent.match(/<script[\\s\\S]*?<\\/script>/gi) ?? [];\n  const scriptText = scriptMatches.join('');\n\n  const scriptLen = scriptText.length;\n  const totalLen = bodyContent.length;\n\n  if (totalLen === 0) return false;\n  return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;\n}\n\nexport function contentAppearsEmpty(html: string): boolean {\n  const visibleText = extractVisibleText(html);\n  if (visibleText.length < VISIBLE_TEXT_THRESHOLD) return true;\n\n  if (hasSpaShellIndicator(html)) return true;\n  if (hasNextData(html)) return true;\n  if (hasNoscriptRequired(html)) return true;\n  if (hasHighScriptRatio(html)) return true;\n\n  return false;\n}\n"],"mappings":"AAAA,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,SAAS,sBAAsB,MAAsB;AACnD,SAAO,KACJ,QAAQ,+BAA+B,EAAE,EACzC,QAAQ,6BAA6B,EAAE;AAC5C;AAEA,SAAS,mBAAmB,MAAsB;AAChD,QAAM,WAAW,sBAAsB,IAAI;AAC3C,QAAM,SAAS,SAAS,QAAQ,YAAY,GAAG;AAC/C,SAAO,OAAO,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC1C;AAEA,SAAS,qBAAqB,MAAuB;AACnD,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,YAAY,KAAK,CAAC,YAAY,QAAQ,KAAK,IAAI,CAAC;AACzD;AAEA,SAAS,YAAY,MAAuB;AAC1C,MAAI,CAAC,gBAAgB,KAAK,IAAI,EAAG,QAAO;AACxC,QAAM,iBAAiB,sBAAsB,IAAI;AACjD,QAAM,cAAc,eAAe,QAAQ,YAAY,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACtF,SAAO,YAAY,SAAS;AAC9B;AAEA,SAAS,oBAAoB,MAAuB;AAClD,QAAM,kBAAkB,KAAK,MAAM,yCAAyC;AAC5E,MAAI,CAAC,gBAAiB,QAAO;AAC7B,SAAO,gBAAgB,KAAK,CAAC,QAAQ;AACnC,UAAM,QAAQ,IAAI,QAAQ,YAAY,EAAE,EAAE,YAAY;AACtD,WAAO,MAAM,SAAS,YAAY,KAAK,MAAM,SAAS,QAAQ;AAAA,EAChE,CAAC;AACH;AAEA,SAAS,mBAAmB,MAAuB;AACjD,QAAM,YAAY,KAAK,MAAM,gCAAgC;AAC7D,QAAM,cAAc,YAAY,UAAU,CAAC,IAAI;AAE/C,QAAM,gBAAgB,YAAY,MAAM,6BAA6B,KAAK,CAAC;AAC3E,QAAM,aAAa,cAAc,KAAK,EAAE;AAExC,QAAM,YAAY,WAAW;AAC7B,QAAM,WAAW,YAAY;AAE7B,MAAI,aAAa,EAAG,QAAO;AAC3B,SAAO,YAAY,WAAW;AAChC;AAEO,SAAS,oBAAoB,MAAuB;AACzD,QAAM,cAAc,mBAAmB,IAAI;AAC3C,MAAI,YAAY,SAAS,uBAAwB,QAAO;AAExD,MAAI,qBAAqB,IAAI,EAAG,QAAO;AACvC,MAAI,YAAY,IAAI,EAAG,QAAO;AAC9B,MAAI,oBAAoB,IAAI,EAAG,QAAO;AACtC,MAAI,mBAAmB,IAAI,EAAG,QAAO;AAErC,SAAO;AACT;","names":[]}

package/dist/fetch/error-describe.d.ts

@@ -0,0 +1,7 @@
+interface DescribedError {
+    reason: string;
+    hint?: string;
+}
+export declare function describeFetchError(err: unknown): DescribedError;
+export {};
+//# sourceMappingURL=error-describe.d.ts.map

package/dist/fetch/error-describe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"error-describe.d.ts","sourceRoot":"","sources":["../../src/fetch/error-describe.ts"],"names":[],"mappings":"AAKA,UAAU,cAAc;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAwBD,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,cAAc,CAa/D"}

package/dist/fetch/error-describe.js

@@ -0,0 +1,37 @@
+const CODE_DESCRIPTIONS = {
+  ENOTFOUND: { reason: "DNS resolution failed (ENOTFOUND)", hint: "Check the domain name; the host could not be resolved" },
+  ECONNREFUSED: { reason: "Connection refused (ECONNREFUSED)", hint: "Target host rejected the connection \u2014 server may be down or port closed" },
+  ECONNRESET: { reason: "Connection reset (ECONNRESET)", hint: "Remote peer closed the connection mid-request \u2014 retry may succeed" },
+  ETIMEDOUT: { reason: "Connection timed out (ETIMEDOUT)", hint: "Increase timeoutMs or check network reachability" },
+  EAI_AGAIN: { reason: "DNS lookup temporarily failed (EAI_AGAIN)", hint: "DNS resolver issue \u2014 retry shortly" },
+  EHOSTUNREACH: { reason: "Host unreachable (EHOSTUNREACH)" },
+  ENETUNREACH: { reason: "Network unreachable (ENETUNREACH)" },
+  CERT_HAS_EXPIRED: { reason: "TLS certificate expired" },
+  UNABLE_TO_VERIFY_LEAF_SIGNATURE: { reason: "TLS chain verification failed" },
+  SELF_SIGNED_CERT_IN_CHAIN: { reason: "Self-signed TLS certificate in chain" }
+};
+function extractCode(err, depth = 0) {
+  if (!err || depth > 5) return null;
+  if (typeof err !== "object") return null;
+  const e = err;
+  if (typeof e.code === "string" && e.code) return e.code;
+  if (e.cause) return extractCode(e.cause, depth + 1);
+  return null;
+}
+function describeFetchError(err) {
+  const code = extractCode(err);
+  if (code && CODE_DESCRIPTIONS[code]) return CODE_DESCRIPTIONS[code];
+  if (err instanceof Error) {
+    if (err.name === "TimeoutError" || err.name === "AbortError") {
+      return { reason: "Request timed out", hint: "Increase timeoutMs or check network reachability" };
+    }
+    const msg = err.message;
+    if (code) return { reason: `${msg} (${code})` };
+    return { reason: msg || "fetch failed" };
+  }
+  return { reason: String(err) };
+}
+export {
+  describeFetchError
+};
+//# sourceMappingURL=error-describe.js.map

package/dist/fetch/error-describe.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/fetch/error-describe.ts"],"sourcesContent":["// Translate a thrown fetch error into a stable, specific reason string.\n// Node's undici surfaces \"fetch failed\" on the outer TypeError while the\n// actual code (ENOTFOUND/ECONNREFUSED/etc.) hides on err.cause. Drill the\n// chain so callers see what actually broke instead of a generic phrase.\n\ninterface DescribedError {\n  reason: string;\n  hint?: string;\n}\n\nconst CODE_DESCRIPTIONS: Record<string, { reason: string; hint?: string }> = {\n  ENOTFOUND: { reason: 'DNS resolution failed (ENOTFOUND)', hint: 'Check the domain name; the host could not be resolved' },\n  ECONNREFUSED: { reason: 'Connection refused (ECONNREFUSED)', hint: 'Target host rejected the connection — server may be down or port closed' },\n  ECONNRESET: { reason: 'Connection reset (ECONNRESET)', hint: 'Remote peer closed the connection mid-request — retry may succeed' },\n  ETIMEDOUT: { reason: 'Connection timed out (ETIMEDOUT)', hint: 'Increase timeoutMs or check network reachability' },\n  EAI_AGAIN: { reason: 'DNS lookup temporarily failed (EAI_AGAIN)', hint: 'DNS resolver issue — retry shortly' },\n  EHOSTUNREACH: { reason: 'Host unreachable (EHOSTUNREACH)' },\n  ENETUNREACH: { reason: 'Network unreachable (ENETUNREACH)' },\n  CERT_HAS_EXPIRED: { reason: 'TLS certificate expired' },\n  UNABLE_TO_VERIFY_LEAF_SIGNATURE: { reason: 'TLS chain verification failed' },\n  SELF_SIGNED_CERT_IN_CHAIN: { reason: 'Self-signed TLS certificate in chain' },\n};\n\nfunction extractCode(err: unknown, depth = 0): string | null {\n  if (!err || depth > 5) return null;\n  if (typeof err !== 'object') return null;\n  const e = err as { code?: unknown; cause?: unknown };\n  if (typeof e.code === 'string' && e.code) return e.code;\n  if (e.cause) return extractCode(e.cause, depth + 1);\n  return null;\n}\n\nexport function describeFetchError(err: unknown): DescribedError {\n  const code = extractCode(err);\n  if (code && CODE_DESCRIPTIONS[code]) return CODE_DESCRIPTIONS[code];\n\n  if (err instanceof Error) {\n    if (err.name === 'TimeoutError' || err.name === 'AbortError') {\n      return { reason: 'Request timed out', hint: 'Increase timeoutMs or check network reachability' };\n    }\n    const msg = err.message;\n    if (code) return { reason: `${msg} (${code})` };\n    return { reason: msg || 'fetch failed' };\n  }\n  return { reason: String(err) };\n}\n"],"mappings":"AAUA,MAAM,oBAAuE;AAAA,EAC3E,WAAW,EAAE,QAAQ,qCAAqC,MAAM,wDAAwD;AAAA,EACxH,cAAc,EAAE,QAAQ,qCAAqC,MAAM,+EAA0E;AAAA,EAC7I,YAAY,EAAE,QAAQ,iCAAiC,MAAM,yEAAoE;AAAA,EACjI,WAAW,EAAE,QAAQ,oCAAoC,MAAM,mDAAmD;AAAA,EAClH,WAAW,EAAE,QAAQ,6CAA6C,MAAM,0CAAqC;AAAA,EAC7G,cAAc,EAAE,QAAQ,kCAAkC;AAAA,EAC1D,aAAa,EAAE,QAAQ,oCAAoC;AAAA,EAC3D,kBAAkB,EAAE,QAAQ,0BAA0B;AAAA,EACtD,iCAAiC,EAAE,QAAQ,gCAAgC;AAAA,EAC3E,2BAA2B,EAAE,QAAQ,uCAAuC;AAC9E;AAEA,SAAS,YAAY,KAAc,QAAQ,GAAkB;AAC3D,MAAI,CAAC,OAAO,QAAQ,EAAG,QAAO;AAC9B,MAAI,OAAO,QAAQ,SAAU,QAAO;AACpC,QAAM,IAAI;AACV,MAAI,OAAO,EAAE,SAAS,YAAY,EAAE,KAAM,QAAO,EAAE;AACnD,MAAI,EAAE,MAAO,QAAO,YAAY,EAAE,OAAO,QAAQ,CAAC;AAClD,SAAO;AACT;AAEO,SAAS,mBAAmB,KAA8B;AAC/D,QAAM,OAAO,YAAY,GAAG;AAC5B,MAAI,QAAQ,kBAAkB,IAAI,EAAG,QAAO,kBAAkB,IAAI;AAElE,MAAI,eAAe,OAAO;AACxB,QAAI,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAc;AAC5D,aAAO,EAAE,QAAQ,qBAAqB,MAAM,mDAAmD;AAAA,IACjG;AACA,UAAM,MAAM,IAAI;AAChB,QAAI,KAAM,QAAO,EAAE,QAAQ,GAAG,GAAG,KAAK,IAAI,IAAI;AAC9C,WAAO,EAAE,QAAQ,OAAO,eAAe;AAAA,EACzC;AACA,SAAO,EAAE,QAAQ,OAAO,GAAG,EAAE;AAC/B;","names":[]}

package/dist/fetch/http-client.d.ts

@@ -1,6 +1,10 @@
 export interface HttpFetchOptions {
     headers?: Record<string, string>;
     timeoutMs?: number;
+    conditionalHeaders?: {
+        ifNoneMatch?: string;
+        ifModifiedSince?: string;
+    };
 }
 export interface HttpFetchResult {
     url: string;

package/dist/fetch/http-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmCD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyCrG"}
+{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmCD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyCrG"}

package/dist/fetch/http-client.js

@@ -1,146 +1,165 @@
-import { getConfig } from '../config.js';
-import { createLogger } from '../logger.js';
-const RETRYABLE_STATUSES = new Set([429, 502, 503]);
-const RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);
-const REDIRECT_STATUSES = new Set([301, 302, 307, 308]);
+import { getConfig } from "../config.js";
+import { createLogger } from "../logger.js";
+const RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 502, 503]);
+const RETRYABLE_ERROR_CODES = /* @__PURE__ */ new Set(["ECONNRESET", "ETIMEDOUT", "ECONNREFUSED"]);
+const REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 307, 308]);
 const DEFAULT_USER_AGENTS = [
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
 ];
 function getRotatingUserAgent(config) {
-    if (config.userAgent)
-        return config.userAgent;
-    return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];
+  if (config.userAgent) return config.userAgent;
+  return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];
 }
 function isRetryableError(err) {
-    if (err instanceof Error) {
-        const code = err.code;
-        if (code && RETRYABLE_ERROR_CODES.has(code))
-            return true;
-        // AbortSignal timeout throws DOMException with name TimeoutError
-        if (err.name === 'TimeoutError')
-            return true;
-    }
-    return false;
+  if (err instanceof Error) {
+    const code = err.code;
+    if (code && RETRYABLE_ERROR_CODES.has(code)) return true;
+    if (err.name === "TimeoutError") return true;
+  }
+  return false;
 }
 function backoffMs(attempt) {
-    return 500 * Math.pow(2, attempt) + Math.random() * 500;
+  return 500 * Math.pow(2, attempt) + Math.random() * 500;
 }
 function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
-export async function httpFetch(url, options = {}) {
-    const config = getConfig();
-    const logger = createLogger('fetch');
-    const maxRetries = config.fetchMaxRetries;
-    const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;
-    const maxRedirects = config.maxRedirects;
-    let lastError;
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-        if (attempt > 0) {
-            const delay = backoffMs(attempt - 1);
-            logger.debug('retrying after backoff', { attempt, delayMs: delay, url });
-            await sleep(delay);
-        }
-        try {
-            const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);
-            return result;
-        }
-        catch (err) {
-            lastError = err;
-            if (err instanceof HttpFetchError && !err.retryable) {
-                throw err;
-            }
-            const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);
-            if (!retryable || attempt >= maxRetries) {
-                throw err;
-            }
-            logger.warn('fetch failed, will retry', {
-                attempt,
-                url,
-                error: err instanceof Error ? err.message : String(err),
-            });
-        }
+async function httpFetch(url, options = {}) {
+  const config = getConfig();
+  const logger = createLogger("fetch");
+  const maxRetries = config.fetchMaxRetries;
+  const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;
+  const maxRedirects = config.maxRedirects;
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    if (attempt > 0) {
+      const delay = backoffMs(attempt - 1);
+      logger.debug("retrying after backoff", { attempt, delayMs: delay, url });
+      await sleep(delay);
+    }
+    try {
+      const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);
+      return result;
+    } catch (err) {
+      lastError = err;
+      if (err instanceof HttpFetchError && !err.retryable) {
+        throw err;
+      }
+      const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);
+      if (!retryable || attempt >= maxRetries) {
+        throw err;
+      }
+      logger.warn("fetch failed, will retry", {
+        attempt,
+        url,
+        error: err instanceof Error ? err.message : String(err)
+      });
     }
-    throw lastError;
+  }
+  throw lastError;
 }
 class HttpFetchError extends Error {
-    retryable;
-    constructor(message, retryable) {
-        super(message);
-        this.retryable = retryable;
-        this.name = 'HttpFetchError';
-    }
+  constructor(message, retryable) {
+    super(message);
+    this.retryable = retryable;
+    this.name = "HttpFetchError";
+  }
+  retryable;
 }
 async function fetchWithRedirects(originalUrl, options, timeoutMs, maxRedirects, logger) {
-    const visited = new Set();
-    let currentUrl = originalUrl;
-    let redirectCount = 0;
-    while (true) {
-        if (visited.has(currentUrl)) {
-            throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);
-        }
-        visited.add(currentUrl);
-        logger.debug('fetching', { url: currentUrl, attempt: redirectCount });
-        const signal = AbortSignal.timeout(timeoutMs);
-        let response;
-        try {
-            const ua = getRotatingUserAgent(getConfig());
-            const mergedHeaders = { 'User-Agent': ua, ...options.headers };
-            response = await fetch(currentUrl, {
-                headers: mergedHeaders,
-                redirect: 'manual',
-                signal,
-            });
-        }
-        catch (err) {
-            const isTimeout = err instanceof Error && err.name === 'TimeoutError';
-            const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has(err.code ?? '');
-            const retryable = isTimeout || isConnErr;
-            throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });
-        }
-        if (REDIRECT_STATUSES.has(response.status)) {
-            const location = response.headers.get('location');
-            if (!location) {
-                throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);
-            }
-            redirectCount++;
-            if (redirectCount > maxRedirects) {
-                throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);
-            }
-            // Resolve relative redirects
-            currentUrl = new URL(location, currentUrl).toString();
-            continue;
-        }
-        if (RETRYABLE_STATUSES.has(response.status)) {
-            throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);
-        }
-        const contentType = response.headers.get('content-type') ?? '';
-        const headers = {};
-        response.headers.forEach((value, key) => {
-            headers[key] = value;
-        });
-        const isPdf = contentType.includes('application/pdf');
-        let html;
-        let rawBuffer;
-        if (isPdf) {
-            const arrayBuf = await response.arrayBuffer();
-            rawBuffer = Buffer.from(arrayBuf);
-            html = '';
-        }
-        else {
-            html = await response.text();
-        }
-        return {
-            url: originalUrl,
-            finalUrl: currentUrl,
-            html,
-            contentType,
-            statusCode: response.status,
-            headers,
-            rawBuffer,
-        };
+  const visited = /* @__PURE__ */ new Set();
+  let currentUrl = originalUrl;
+  let redirectCount = 0;
+  while (true) {
+    if (visited.has(currentUrl)) {
+      throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);
+    }
+    visited.add(currentUrl);
+    logger.debug("fetching", { url: currentUrl, attempt: redirectCount });
+    const signal = AbortSignal.timeout(timeoutMs);
+    let response;
+    try {
+      const ua = getRotatingUserAgent(getConfig());
+      const mergedHeaders = { "User-Agent": ua, ...options.headers };
+      if (options.conditionalHeaders?.ifNoneMatch) {
+        mergedHeaders["If-None-Match"] = options.conditionalHeaders.ifNoneMatch;
+      }
+      if (options.conditionalHeaders?.ifModifiedSince) {
+        mergedHeaders["If-Modified-Since"] = options.conditionalHeaders.ifModifiedSince;
+      }
+      response = await fetch(currentUrl, {
+        headers: mergedHeaders,
+        redirect: "manual",
+        signal
+      });
+    } catch (err) {
+      const isTimeout = err instanceof Error && err.name === "TimeoutError";
+      const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has(err.code ?? "");
+      const retryable = isTimeout || isConnErr;
+      throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });
+    }
+    if (response.status === 304) {
+      const headers2 = {};
+      response.headers.forEach((value, key) => {
+        headers2[key] = value;
+      });
+      try {
+        await response.arrayBuffer();
+      } catch {
+      }
+      return {
+        url: originalUrl,
+        finalUrl: currentUrl,
+        html: "",
+        contentType: response.headers.get("content-type") ?? "",
+        statusCode: 304,
+        headers: headers2
+      };
+    }
+    if (REDIRECT_STATUSES.has(response.status)) {
+      const location = response.headers.get("location");
+      if (!location) {
+        throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);
+      }
+      redirectCount++;
+      if (redirectCount > maxRedirects) {
+        throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);
+      }
+      currentUrl = new URL(location, currentUrl).toString();
+      continue;
+    }
+    if (RETRYABLE_STATUSES.has(response.status)) {
+      throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    const headers = {};
+    response.headers.forEach((value, key) => {
+      headers[key] = value;
+    });
+    const isPdf = contentType.includes("application/pdf");
+    let html;
+    let rawBuffer;
+    if (isPdf) {
+      const arrayBuf = await response.arrayBuffer();
+      rawBuffer = Buffer.from(arrayBuf);
+      html = "";
+    } else {
+      html = await response.text();
     }
+    return {
+      url: originalUrl,
+      finalUrl: currentUrl,
+      html,
+      contentType,
+      statusCode: response.status,
+      headers,
+      rawBuffer
+    };
+  }
 }
+export {
+  httpFetch
+};
 //# sourceMappingURL=http-client.js.map