@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/fetch/browser-types.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/fetch/browser-types.ts"],"sourcesContent":["import type { BrowserType } from '../types.js';\n\nconst VALID_BROWSER_TYPES: ReadonlySet<string> = new Set(['chromium', 'firefox', 'webkit', 'lightpanda']);\n\nconst DEFAULT_BROWSER_TYPES: BrowserType[] = ['chromium'];\n\nfunction warn(msg: string, data: Record<string, unknown>): void {\n  const line = JSON.stringify({ ts: new Date().toISOString(), level: 'warn', msg, module: 'fetch', data });\n  process.stderr.write(line + '\\n');\n}\n\nexport function parseBrowserTypes(input: string | undefined | null): BrowserType[] {\n  if (!input || typeof input !== 'string') {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const trimmed = input.trim();\n  if (trimmed.length === 0) {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const parts = trimmed\n    .split(',')\n    .map(s => s.trim())\n    .filter(s => s.length > 0);\n\n  if (parts.length === 0) {\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  const seen = new Set<string>();\n  const valid: BrowserType[] = [];\n  const invalid: string[] = [];\n\n  for (const part of parts) {\n    if (!VALID_BROWSER_TYPES.has(part)) {\n      invalid.push(part);\n      continue;\n    }\n    if (seen.has(part)) {\n      continue;\n    }\n    seen.add(part);\n    valid.push(part as BrowserType);\n  }\n\n  if (invalid.length > 0) {\n    warn('ignored invalid browser types', { invalid, valid: [...valid] });\n  }\n\n  if (valid.length === 0) {\n    warn('no valid browser types found, falling back to chromium', { input });\n    return [...DEFAULT_BROWSER_TYPES];\n  }\n\n  return valid;\n}\n"],"mappings":"AAEA,MAAM,sBAA2C,oBAAI,IAAI,CAAC,YAAY,WAAW,UAAU,YAAY,CAAC;AAExG,MAAM,wBAAuC,CAAC,UAAU;AAExD,SAAS,KAAK,KAAa,MAAqC;AAC9D,QAAM,OAAO,KAAK,UAAU,EAAE,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,OAAO,QAAQ,KAAK,QAAQ,SAAS,KAAK,CAAC;AACvG,UAAQ,OAAO,MAAM,OAAO,IAAI;AAClC;AAEO,SAAS,kBAAkB,OAAiD;AACjF,MAAI,CAAC,SAAS,OAAO,UAAU,UAAU;AACvC,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,UAAU,MAAM,KAAK;AAC3B,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,QAAQ,QACX,MAAM,GAAG,EACT,IAAI,OAAK,EAAE,KAAK,CAAC,EACjB,OAAO,OAAK,EAAE,SAAS,CAAC;AAE3B,MAAI,MAAM,WAAW,GAAG;AACtB,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,QAAuB,CAAC;AAC9B,QAAM,UAAoB,CAAC;AAE3B,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,oBAAoB,IAAI,IAAI,GAAG;AAClC,cAAQ,KAAK,IAAI;AACjB;AAAA,IACF;AACA,QAAI,KAAK,IAAI,IAAI,GAAG;AAClB;AAAA,IACF;AACA,SAAK,IAAI,IAAI;AACb,UAAM,KAAK,IAAmB;AAAA,EAChC;AAEA,MAAI,QAAQ,SAAS,GAAG;AACtB,SAAK,iCAAiC,EAAE,SAAS,OAAO,CAAC,GAAG,KAAK,EAAE,CAAC;AAAA,EACtE;AAEA,MAAI,MAAM,WAAW,GAAG;AACtB,SAAK,0DAA0D,EAAE,MAAM,CAAC;AACxE,WAAO,CAAC,GAAG,qBAAqB;AAAA,EAClC;AAEA,SAAO;AACT;","names":[]}

package/dist/fetch/cdp-client.d.ts

@@ -0,0 +1,9 @@
+import type { CDPSession } from '../types.js';
+export interface DiscoverOptions {
+    timeoutMs?: number;
+    filterPages?: boolean;
+}
+export declare function parseCDPResponse(raw: string): CDPSession[];
+export declare function discoverSessions(cdpUrl: string, options?: DiscoverOptions): Promise<CDPSession[]>;
+export declare function isCDPReachable(cdpUrl: string, timeoutMs?: number): Promise<boolean>;
+//# sourceMappingURL=cdp-client.d.ts.map

package/dist/fetch/cdp-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdp-client.d.ts","sourceRoot":"","sources":["../../src/fetch/cdp-client.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAM9C,MAAM,WAAW,eAAe;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAiCD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,UAAU,EAAE,CA2B1D;AAED,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,UAAU,EAAE,CAAC,CA6BvB;AAED,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,SAAqB,GAAG,OAAO,CAAC,OAAO,CAAC,CAQrG"}

package/dist/fetch/cdp-client.js

@@ -0,0 +1,89 @@
+import * as http from "node:http";
+import { createLogger } from "../logger.js";
+const log = createLogger("fetch");
+const DEFAULT_TIMEOUT_MS = 3e3;
+function httpGet(url, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const req = http.get(url, { timeout: timeoutMs }, (res) => {
+      let body = "";
+      res.on("data", (chunk) => {
+        body += chunk;
+      });
+      res.on("end", () => {
+        resolve({ statusCode: res.statusCode ?? 0, body });
+      });
+    });
+    req.on("error", (err) => {
+      reject(err);
+    });
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error(`CDP request timed out after ${timeoutMs}ms`));
+    });
+  });
+}
+function parseCDPResponse(raw) {
+  try {
+    const parsed = JSON.parse(raw);
+    if (!Array.isArray(parsed)) {
+      log.debug("CDP response is not an array");
+      return [];
+    }
+    return parsed.filter((entry) => {
+      if (!entry || typeof entry !== "object") return false;
+      if (!entry.id || typeof entry.id !== "string") return false;
+      if (!entry.webSocketDebuggerUrl || typeof entry.webSocketDebuggerUrl !== "string") return false;
+      return true;
+    }).map((entry) => ({
+      id: entry.id,
+      url: entry.url ?? "",
+      title: entry.title ?? "",
+      webSocketDebuggerUrl: entry.webSocketDebuggerUrl,
+      type: entry.type,
+      devtoolsFrontendUrl: entry.devtoolsFrontendUrl
+    }));
+  } catch (err) {
+    log.debug("failed to parse CDP response", { error: err instanceof Error ? err.message : String(err) });
+    return [];
+  }
+}
+async function discoverSessions(cdpUrl, options) {
+  const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const filterPages = options?.filterPages ?? false;
+  const jsonUrl = cdpUrl.endsWith("/") ? `${cdpUrl}json` : `${cdpUrl}/json`;
+  try {
+    log.debug("discovering CDP sessions", { url: jsonUrl, timeoutMs });
+    const { statusCode, body } = await httpGet(jsonUrl, timeoutMs);
+    if (statusCode !== 200) {
+      log.warn("CDP endpoint returned non-200", { statusCode, url: jsonUrl });
+      return [];
+    }
+    let sessions = parseCDPResponse(body);
+    if (filterPages) {
+      sessions = sessions.filter((s) => s.type === "page" || s.type === void 0);
+    }
+    log.info("CDP sessions discovered", { count: sessions.length, url: jsonUrl });
+    return sessions;
+  } catch (err) {
+    log.debug("CDP discovery failed", {
+      url: jsonUrl,
+      error: err instanceof Error ? err.message : String(err)
+    });
+    return [];
+  }
+}
+async function isCDPReachable(cdpUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
+  const jsonUrl = cdpUrl.endsWith("/") ? `${cdpUrl}json` : `${cdpUrl}/json`;
+  try {
+    const { statusCode } = await httpGet(jsonUrl, timeoutMs);
+    return statusCode === 200;
+  } catch {
+    return false;
+  }
+}
+export {
+  discoverSessions,
+  isCDPReachable,
+  parseCDPResponse
+};
+//# sourceMappingURL=cdp-client.js.map

package/dist/fetch/cdp-client.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/fetch/cdp-client.ts"],"sourcesContent":["import * as http from 'node:http';\nimport { createLogger } from '../logger.js';\nimport type { CDPSession } from '../types.js';\n\nconst log = createLogger('fetch');\n\nconst DEFAULT_TIMEOUT_MS = 3000;\n\nexport interface DiscoverOptions {\n  timeoutMs?: number;\n  filterPages?: boolean;\n}\n\nfunction httpGet(url: string, timeoutMs: number): Promise<{ statusCode: number; body: string }> {\n  return new Promise((resolve, reject) => {\n    const req = http.get(url, { timeout: timeoutMs }, (res) => {\n      let body = '';\n      res.on('data', (chunk: string) => { body += chunk; });\n      res.on('end', () => {\n        resolve({ statusCode: res.statusCode ?? 0, body });\n      });\n    });\n\n    req.on('error', (err) => {\n      reject(err);\n    });\n\n    req.on('timeout', () => {\n      req.destroy();\n      reject(new Error(`CDP request timed out after ${timeoutMs}ms`));\n    });\n  });\n}\n\ninterface RawCDPEntry {\n  id?: string;\n  url?: string;\n  title?: string;\n  webSocketDebuggerUrl?: string;\n  type?: string;\n  devtoolsFrontendUrl?: string;\n  description?: string;\n}\n\nexport function parseCDPResponse(raw: string): CDPSession[] {\n  try {\n    const parsed = JSON.parse(raw);\n    if (!Array.isArray(parsed)) {\n      log.debug('CDP response is not an array');\n      return [];\n    }\n\n    return parsed\n      .filter((entry: RawCDPEntry) => {\n        if (!entry || typeof entry !== 'object') return false;\n        if (!entry.id || typeof entry.id !== 'string') return false;\n        if (!entry.webSocketDebuggerUrl || typeof entry.webSocketDebuggerUrl !== 'string') return false;\n        return true;\n      })\n      .map((entry: RawCDPEntry) => ({\n        id: entry.id!,\n        url: entry.url ?? '',\n        title: entry.title ?? '',\n        webSocketDebuggerUrl: entry.webSocketDebuggerUrl!,\n        type: entry.type,\n        devtoolsFrontendUrl: entry.devtoolsFrontendUrl,\n      }));\n  } catch (err) {\n    log.debug('failed to parse CDP response', { error: err instanceof Error ? err.message : String(err) });\n    return [];\n  }\n}\n\nexport async function discoverSessions(\n  cdpUrl: string,\n  options?: DiscoverOptions,\n): Promise<CDPSession[]> {\n  const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS;\n  const filterPages = options?.filterPages ?? false;\n  const jsonUrl = cdpUrl.endsWith('/') ? `${cdpUrl}json` : `${cdpUrl}/json`;\n\n  try {\n    log.debug('discovering CDP sessions', { url: jsonUrl, timeoutMs });\n    const { statusCode, body } = await httpGet(jsonUrl, timeoutMs);\n\n    if (statusCode !== 200) {\n      log.warn('CDP endpoint returned non-200', { statusCode, url: jsonUrl });\n      return [];\n    }\n\n    let sessions = parseCDPResponse(body);\n\n    if (filterPages) {\n      sessions = sessions.filter(s => s.type === 'page' || s.type === undefined);\n    }\n\n    log.info('CDP sessions discovered', { count: sessions.length, url: jsonUrl });\n    return sessions;\n  } catch (err) {\n    log.debug('CDP discovery failed', {\n      url: jsonUrl,\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return [];\n  }\n}\n\nexport async function isCDPReachable(cdpUrl: string, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<boolean> {\n  const jsonUrl = cdpUrl.endsWith('/') ? `${cdpUrl}json` : `${cdpUrl}/json`;\n  try {\n    const { statusCode } = await httpGet(jsonUrl, timeoutMs);\n    return statusCode === 200;\n  } catch {\n    return false;\n  }\n}\n"],"mappings":"AAAA,YAAY,UAAU;AACtB,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,qBAAqB;AAO3B,SAAS,QAAQ,KAAa,WAAkE;AAC9F,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,MAAM,KAAK,IAAI,KAAK,EAAE,SAAS,UAAU,GAAG,CAAC,QAAQ;AACzD,UAAI,OAAO;AACX,UAAI,GAAG,QAAQ,CAAC,UAAkB;AAAE,gBAAQ;AAAA,MAAO,CAAC;AACpD,UAAI,GAAG,OAAO,MAAM;AAClB,gBAAQ,EAAE,YAAY,IAAI,cAAc,GAAG,KAAK,CAAC;AAAA,MACnD,CAAC;AAAA,IACH,CAAC;AAED,QAAI,GAAG,SAAS,CAAC,QAAQ;AACvB,aAAO,GAAG;AAAA,IACZ,CAAC;AAED,QAAI,GAAG,WAAW,MAAM;AACtB,UAAI,QAAQ;AACZ,aAAO,IAAI,MAAM,+BAA+B,SAAS,IAAI,CAAC;AAAA,IAChE,CAAC;AAAA,EACH,CAAC;AACH;AAYO,SAAS,iBAAiB,KAA2B;AAC1D,MAAI;AACF,UAAM,SAAS,KAAK,MAAM,GAAG;AAC7B,QAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,UAAI,MAAM,8BAA8B;AACxC,aAAO,CAAC;AAAA,IACV;AAEA,WAAO,OACJ,OAAO,CAAC,UAAuB;AAC9B,UAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO;AAChD,UAAI,CAAC,MAAM,MAAM,OAAO,MAAM,OAAO,SAAU,QAAO;AACtD,UAAI,CAAC,MAAM,wBAAwB,OAAO,MAAM,yBAAyB,SAAU,QAAO;AAC1F,aAAO;AAAA,IACT,CAAC,EACA,IAAI,CAAC,WAAwB;AAAA,MAC5B,IAAI,MAAM;AAAA,MACV,KAAK,MAAM,OAAO;AAAA,MAClB,OAAO,MAAM,SAAS;AAAA,MACtB,sBAAsB,MAAM;AAAA,MAC5B,MAAM,MAAM;AAAA,MACZ,qBAAqB,MAAM;AAAA,IAC7B,EAAE;AAAA,EACN,SAAS,KAAK;AACZ,QAAI,MAAM,gCAAgC,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE,CAAC;AACrG,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,iBACpB,QACA,SACuB;AACvB,QAAM,YAAY,SAAS,aAAa;AACxC,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,UAAU,OAAO,SAAS,GAAG,IAAI,GAAG,MAAM,SAAS,GAAG,MAAM;AAElE,MAAI;AACF,QAAI,MAAM,4BAA4B,EAAE,KAAK,SAAS,UAAU,CAAC;AACjE,UAAM,EAAE,YAAY,KAAK,IAAI,MAAM,QAAQ,SAAS,SAAS;AAE7D,QAAI,eAAe,KAAK;AACtB,UAAI,KAAK,iCAAiC,EAAE,YAAY,KAAK,QAAQ,CAAC;AACtE,aAAO,CAAC;AAAA,IACV;AAEA,QAAI,WAAW,iBAAiB,IAAI;AAEpC,QAAI,aAAa;AACf,iBAAW,SAAS,OAAO,OAAK,EAAE,SAAS,UAAU,EAAE,SAAS,MAAS;AAAA,IAC3E;AAEA,QAAI,KAAK,2BAA2B,EAAE,OAAO,SAAS,QAAQ,KAAK,QAAQ,CAAC;AAC5E,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,QAAI,MAAM,wBAAwB;AAAA,MAChC,KAAK;AAAA,MACL,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO,CAAC;AAAA,EACV;AACF;AAEA,eAAsB,eAAe,QAAgB,YAAY,oBAAsC;AACrG,QAAM,UAAU,OAAO,SAAS,GAAG,IAAI,GAAG,MAAM,SAAS,GAAG,MAAM;AAClE,MAAI;AACF,UAAM,EAAE,WAAW,IAAI,MAAM,QAAQ,SAAS,SAAS;AACvD,WAAO,eAAe;AAAA,EACxB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/fetch/content-check.js

@@ -1,62 +1,55 @@
 const VISIBLE_TEXT_THRESHOLD = 200;
 const SCRIPT_RATIO_THRESHOLD = 0.8;
 function stripScriptsAndStyles(html) {
-    return html
-        .replace(/<script[\s\S]*?<\/script>/gi, '')
-        .replace(/<style[\s\S]*?<\/style>/gi, '');
+  return html.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "");
 }
 function extractVisibleText(html) {
-    const stripped = stripScriptsAndStyles(html);
-    const noTags = stripped.replace(/<[^>]+>/g, ' ');
-    return noTags.replace(/\s+/g, ' ').trim();
+  const stripped = stripScriptsAndStyles(html);
+  const noTags = stripped.replace(/<[^>]+>/g, " ");
+  return noTags.replace(/\s+/g, " ").trim();
 }
 function hasSpaShellIndicator(html) {
-    const spaPatterns = [
-        /<div[^>]+id=["']root["'][^>]*>\s*<\/div>/i,
-        /<div[^>]+id=["']app["'][^>]*>\s*<\/div>/i,
-        /<div[^>]+id=["']__next["'][^>]*>\s*<\/div>/i,
-    ];
-    return spaPatterns.some((pattern) => pattern.test(html));
+  const spaPatterns = [
+    /<div[^>]+id=["']root["'][^>]*>\s*<\/div>/i,
+    /<div[^>]+id=["']app["'][^>]*>\s*<\/div>/i,
+    /<div[^>]+id=["']__next["'][^>]*>\s*<\/div>/i
+  ];
+  return spaPatterns.some((pattern) => pattern.test(html));
 }
 function hasNextData(html) {
-    if (!/__NEXT_DATA__/.test(html))
-        return false;
-    const withoutScripts = stripScriptsAndStyles(html);
-    const visibleText = withoutScripts.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
-    return visibleText.length < VISIBLE_TEXT_THRESHOLD;
+  if (!/__NEXT_DATA__/.test(html)) return false;
+  const withoutScripts = stripScriptsAndStyles(html);
+  const visibleText = withoutScripts.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+  return visibleText.length < VISIBLE_TEXT_THRESHOLD;
 }
 function hasNoscriptRequired(html) {
-    const noscriptMatches = html.match(/<noscript[^>]*>([\s\S]*?)<\/noscript>/gi);
-    if (!noscriptMatches)
-        return false;
-    return noscriptMatches.some((tag) => {
-        const inner = tag.replace(/<[^>]+>/g, '').toLowerCase();
-        return inner.includes('javascript') || inner.includes('enable');
-    });
+  const noscriptMatches = html.match(/<noscript[^>]*>([\s\S]*?)<\/noscript>/gi);
+  if (!noscriptMatches) return false;
+  return noscriptMatches.some((tag) => {
+    const inner = tag.replace(/<[^>]+>/g, "").toLowerCase();
+    return inner.includes("javascript") || inner.includes("enable");
+  });
 }
 function hasHighScriptRatio(html) {
-    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
-    const bodyContent = bodyMatch ? bodyMatch[1] : html;
-    const scriptMatches = bodyContent.match(/<script[\s\S]*?<\/script>/gi) ?? [];
-    const scriptText = scriptMatches.join('');
-    const scriptLen = scriptText.length;
-    const totalLen = bodyContent.length;
-    if (totalLen === 0)
-        return false;
-    return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;
+  const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+  const bodyContent = bodyMatch ? bodyMatch[1] : html;
+  const scriptMatches = bodyContent.match(/<script[\s\S]*?<\/script>/gi) ?? [];
+  const scriptText = scriptMatches.join("");
+  const scriptLen = scriptText.length;
+  const totalLen = bodyContent.length;
+  if (totalLen === 0) return false;
+  return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;
 }
-export function contentAppearsEmpty(html) {
-    const visibleText = extractVisibleText(html);
-    if (visibleText.length < VISIBLE_TEXT_THRESHOLD)
-        return true;
-    if (hasSpaShellIndicator(html))
-        return true;
-    if (hasNextData(html))
-        return true;
-    if (hasNoscriptRequired(html))
-        return true;
-    if (hasHighScriptRatio(html))
-        return true;
-    return false;
+function contentAppearsEmpty(html) {
+  const visibleText = extractVisibleText(html);
+  if (visibleText.length < VISIBLE_TEXT_THRESHOLD) return true;
+  if (hasSpaShellIndicator(html)) return true;
+  if (hasNextData(html)) return true;
+  if (hasNoscriptRequired(html)) return true;
+  if (hasHighScriptRatio(html)) return true;
+  return false;
 }
+export {
+  contentAppearsEmpty
+};
 //# sourceMappingURL=content-check.js.map

package/dist/fetch/content-check.js.map

@@ -1 +1 @@
-{"version":3,"file":"content-check.js","sourceRoot":"","sources":["../../src/fetch/content-check.ts"],"names":[],"mappings":"AAAA,MAAM,sBAAsB,GAAG,GAAG,CAAC;AACnC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAEnC,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,IAAI;SACR,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC;SAC1C,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC5C,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,WAAW,GAAG;QAClB,2CAA2C;QAC3C,0CAA0C;QAC1C,6CAA6C;KAC9C,CAAC;IACF,OAAO,WAAW,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAC9C,MAAM,cAAc,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,OAAO,WAAW,CAAC,MAAM,GAAG,sBAAsB,CAAC;AACrD,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC9E,IAAI,CAAC,eAAe;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAY;IACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IAC/D,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpD,MAAM,aAAa,GAAG,WAAW,CAAC,KAAK,CAAC,6BAA6B,CAAC,IAAI,EAAE,CAAC;IAC7E,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAE1C,MAAM,SAAS,GAAG,UAAU,CAAC,MAAM,CAAC;IACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC;IAEpC,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACjC,OAAO,SAAS,GAAG,QAAQ,GAAG,sBAAsB,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,MAAM,WAAW,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,WAAW,CAAC,MAAM,GAAG,sBAAsB;QAAE,OAAO,IAAI,CAAC;IAE7D,IAAI,oBAAoB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5C,IAAI,WAAW,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,mBAAmB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC3C,IAAI,kBAAkB,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAE1C,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"sources":["../../src/fetch/content-check.ts"],"sourcesContent":["const VISIBLE_TEXT_THRESHOLD = 200;\nconst SCRIPT_RATIO_THRESHOLD = 0.8;\n\nfunction stripScriptsAndStyles(html: string): string {\n  return html\n    .replace(/<script[\\s\\S]*?<\\/script>/gi, '')\n    .replace(/<style[\\s\\S]*?<\\/style>/gi, '');\n}\n\nfunction extractVisibleText(html: string): string {\n  const stripped = stripScriptsAndStyles(html);\n  const noTags = stripped.replace(/<[^>]+>/g, ' ');\n  return noTags.replace(/\\s+/g, ' ').trim();\n}\n\nfunction hasSpaShellIndicator(html: string): boolean {\n  const spaPatterns = [\n    /<div[^>]+id=[\"']root[\"'][^>]*>\\s*<\\/div>/i,\n    /<div[^>]+id=[\"']app[\"'][^>]*>\\s*<\\/div>/i,\n    /<div[^>]+id=[\"']__next[\"'][^>]*>\\s*<\\/div>/i,\n  ];\n  return spaPatterns.some((pattern) => pattern.test(html));\n}\n\nfunction hasNextData(html: string): boolean {\n  if (!/__NEXT_DATA__/.test(html)) return false;\n  const withoutScripts = stripScriptsAndStyles(html);\n  const visibleText = withoutScripts.replace(/<[^>]+>/g, ' ').replace(/\\s+/g, ' ').trim();\n  return visibleText.length < VISIBLE_TEXT_THRESHOLD;\n}\n\nfunction hasNoscriptRequired(html: string): boolean {\n  const noscriptMatches = html.match(/<noscript[^>]*>([\\s\\S]*?)<\\/noscript>/gi);\n  if (!noscriptMatches) return false;\n  return noscriptMatches.some((tag) => {\n    const inner = tag.replace(/<[^>]+>/g, '').toLowerCase();\n    return inner.includes('javascript') || inner.includes('enable');\n  });\n}\n\nfunction hasHighScriptRatio(html: string): boolean {\n  const bodyMatch = html.match(/<body[^>]*>([\\s\\S]*?)<\\/body>/i);\n  const bodyContent = bodyMatch ? bodyMatch[1] : html;\n\n  const scriptMatches = bodyContent.match(/<script[\\s\\S]*?<\\/script>/gi) ?? [];\n  const scriptText = scriptMatches.join('');\n\n  const scriptLen = scriptText.length;\n  const totalLen = bodyContent.length;\n\n  if (totalLen === 0) return false;\n  return scriptLen / totalLen > SCRIPT_RATIO_THRESHOLD;\n}\n\nexport function contentAppearsEmpty(html: string): boolean {\n  const visibleText = extractVisibleText(html);\n  if (visibleText.length < VISIBLE_TEXT_THRESHOLD) return true;\n\n  if (hasSpaShellIndicator(html)) return true;\n  if (hasNextData(html)) return true;\n  if (hasNoscriptRequired(html)) return true;\n  if (hasHighScriptRatio(html)) return true;\n\n  return false;\n}\n"],"mappings":"AAAA,MAAM,yBAAyB;AAC/B,MAAM,yBAAyB;AAE/B,SAAS,sBAAsB,MAAsB;AACnD,SAAO,KACJ,QAAQ,+BAA+B,EAAE,EACzC,QAAQ,6BAA6B,EAAE;AAC5C;AAEA,SAAS,mBAAmB,MAAsB;AAChD,QAAM,WAAW,sBAAsB,IAAI;AAC3C,QAAM,SAAS,SAAS,QAAQ,YAAY,GAAG;AAC/C,SAAO,OAAO,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC1C;AAEA,SAAS,qBAAqB,MAAuB;AACnD,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,YAAY,KAAK,CAAC,YAAY,QAAQ,KAAK,IAAI,CAAC;AACzD;AAEA,SAAS,YAAY,MAAuB;AAC1C,MAAI,CAAC,gBAAgB,KAAK,IAAI,EAAG,QAAO;AACxC,QAAM,iBAAiB,sBAAsB,IAAI;AACjD,QAAM,cAAc,eAAe,QAAQ,YAAY,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACtF,SAAO,YAAY,SAAS;AAC9B;AAEA,SAAS,oBAAoB,MAAuB;AAClD,QAAM,kBAAkB,KAAK,MAAM,yCAAyC;AAC5E,MAAI,CAAC,gBAAiB,QAAO;AAC7B,SAAO,gBAAgB,KAAK,CAAC,QAAQ;AACnC,UAAM,QAAQ,IAAI,QAAQ,YAAY,EAAE,EAAE,YAAY;AACtD,WAAO,MAAM,SAAS,YAAY,KAAK,MAAM,SAAS,QAAQ;AAAA,EAChE,CAAC;AACH;AAEA,SAAS,mBAAmB,MAAuB;AACjD,QAAM,YAAY,KAAK,MAAM,gCAAgC;AAC7D,QAAM,cAAc,YAAY,UAAU,CAAC,IAAI;AAE/C,QAAM,gBAAgB,YAAY,MAAM,6BAA6B,KAAK,CAAC;AAC3E,QAAM,aAAa,cAAc,KAAK,EAAE;AAExC,QAAM,YAAY,WAAW;AAC7B,QAAM,WAAW,YAAY;AAE7B,MAAI,aAAa,EAAG,QAAO;AAC3B,SAAO,YAAY,WAAW;AAChC;AAEO,SAAS,oBAAoB,MAAuB;AACzD,QAAM,cAAc,mBAAmB,IAAI;AAC3C,MAAI,YAAY,SAAS,uBAAwB,QAAO;AAExD,MAAI,qBAAqB,IAAI,EAAG,QAAO;AACvC,MAAI,YAAY,IAAI,EAAG,QAAO;AAC9B,MAAI,oBAAoB,IAAI,EAAG,QAAO;AACtC,MAAI,mBAAmB,IAAI,EAAG,QAAO;AAErC,SAAO;AACT;","names":[]}

package/dist/fetch/http-client.d.ts

@@ -1,6 +1,10 @@
 export interface HttpFetchOptions {
     headers?: Record<string, string>;
     timeoutMs?: number;
+    conditionalHeaders?: {
+        ifNoneMatch?: string;
+        ifModifiedSince?: string;
+    };
 }
 export interface HttpFetchResult {
     url: string;

package/dist/fetch/http-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmCD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyCrG"}
+{"version":3,"file":"http-client.d.ts","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE;QACnB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAmCD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,eAAe,CAAC,CAyCrG"}

package/dist/fetch/http-client.js

@@ -1,146 +1,165 @@
-import { getConfig } from '../config.js';
-import { createLogger } from '../logger.js';
-const RETRYABLE_STATUSES = new Set([429, 502, 503]);
-const RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);
-const REDIRECT_STATUSES = new Set([301, 302, 307, 308]);
+import { getConfig } from "../config.js";
+import { createLogger } from "../logger.js";
+const RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 502, 503]);
+const RETRYABLE_ERROR_CODES = /* @__PURE__ */ new Set(["ECONNRESET", "ETIMEDOUT", "ECONNREFUSED"]);
+const REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 307, 308]);
 const DEFAULT_USER_AGENTS = [
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+  "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
 ];
 function getRotatingUserAgent(config) {
-    if (config.userAgent)
-        return config.userAgent;
-    return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];
+  if (config.userAgent) return config.userAgent;
+  return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];
 }
 function isRetryableError(err) {
-    if (err instanceof Error) {
-        const code = err.code;
-        if (code && RETRYABLE_ERROR_CODES.has(code))
-            return true;
-        // AbortSignal timeout throws DOMException with name TimeoutError
-        if (err.name === 'TimeoutError')
-            return true;
-    }
-    return false;
+  if (err instanceof Error) {
+    const code = err.code;
+    if (code && RETRYABLE_ERROR_CODES.has(code)) return true;
+    if (err.name === "TimeoutError") return true;
+  }
+  return false;
 }
 function backoffMs(attempt) {
-    return 500 * Math.pow(2, attempt) + Math.random() * 500;
+  return 500 * Math.pow(2, attempt) + Math.random() * 500;
 }
 function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+  return new Promise((resolve) => setTimeout(resolve, ms));
 }
-export async function httpFetch(url, options = {}) {
-    const config = getConfig();
-    const logger = createLogger('fetch');
-    const maxRetries = config.fetchMaxRetries;
-    const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;
-    const maxRedirects = config.maxRedirects;
-    let lastError;
-    for (let attempt = 0; attempt <= maxRetries; attempt++) {
-        if (attempt > 0) {
-            const delay = backoffMs(attempt - 1);
-            logger.debug('retrying after backoff', { attempt, delayMs: delay, url });
-            await sleep(delay);
-        }
-        try {
-            const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);
-            return result;
-        }
-        catch (err) {
-            lastError = err;
-            if (err instanceof HttpFetchError && !err.retryable) {
-                throw err;
-            }
-            const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);
-            if (!retryable || attempt >= maxRetries) {
-                throw err;
-            }
-            logger.warn('fetch failed, will retry', {
-                attempt,
-                url,
-                error: err instanceof Error ? err.message : String(err),
-            });
-        }
+async function httpFetch(url, options = {}) {
+  const config = getConfig();
+  const logger = createLogger("fetch");
+  const maxRetries = config.fetchMaxRetries;
+  const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;
+  const maxRedirects = config.maxRedirects;
+  let lastError;
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    if (attempt > 0) {
+      const delay = backoffMs(attempt - 1);
+      logger.debug("retrying after backoff", { attempt, delayMs: delay, url });
+      await sleep(delay);
+    }
+    try {
+      const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);
+      return result;
+    } catch (err) {
+      lastError = err;
+      if (err instanceof HttpFetchError && !err.retryable) {
+        throw err;
+      }
+      const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);
+      if (!retryable || attempt >= maxRetries) {
+        throw err;
+      }
+      logger.warn("fetch failed, will retry", {
+        attempt,
+        url,
+        error: err instanceof Error ? err.message : String(err)
+      });
     }
-    throw lastError;
+  }
+  throw lastError;
 }
 class HttpFetchError extends Error {
-    retryable;
-    constructor(message, retryable) {
-        super(message);
-        this.retryable = retryable;
-        this.name = 'HttpFetchError';
-    }
+  constructor(message, retryable) {
+    super(message);
+    this.retryable = retryable;
+    this.name = "HttpFetchError";
+  }
+  retryable;
 }
 async function fetchWithRedirects(originalUrl, options, timeoutMs, maxRedirects, logger) {
-    const visited = new Set();
-    let currentUrl = originalUrl;
-    let redirectCount = 0;
-    while (true) {
-        if (visited.has(currentUrl)) {
-            throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);
-        }
-        visited.add(currentUrl);
-        logger.debug('fetching', { url: currentUrl, attempt: redirectCount });
-        const signal = AbortSignal.timeout(timeoutMs);
-        let response;
-        try {
-            const ua = getRotatingUserAgent(getConfig());
-            const mergedHeaders = { 'User-Agent': ua, ...options.headers };
-            response = await fetch(currentUrl, {
-                headers: mergedHeaders,
-                redirect: 'manual',
-                signal,
-            });
-        }
-        catch (err) {
-            const isTimeout = err instanceof Error && err.name === 'TimeoutError';
-            const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has(err.code ?? '');
-            const retryable = isTimeout || isConnErr;
-            throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });
-        }
-        if (REDIRECT_STATUSES.has(response.status)) {
-            const location = response.headers.get('location');
-            if (!location) {
-                throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);
-            }
-            redirectCount++;
-            if (redirectCount > maxRedirects) {
-                throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);
-            }
-            // Resolve relative redirects
-            currentUrl = new URL(location, currentUrl).toString();
-            continue;
-        }
-        if (RETRYABLE_STATUSES.has(response.status)) {
-            throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);
-        }
-        const contentType = response.headers.get('content-type') ?? '';
-        const headers = {};
-        response.headers.forEach((value, key) => {
-            headers[key] = value;
-        });
-        const isPdf = contentType.includes('application/pdf');
-        let html;
-        let rawBuffer;
-        if (isPdf) {
-            const arrayBuf = await response.arrayBuffer();
-            rawBuffer = Buffer.from(arrayBuf);
-            html = '';
-        }
-        else {
-            html = await response.text();
-        }
-        return {
-            url: originalUrl,
-            finalUrl: currentUrl,
-            html,
-            contentType,
-            statusCode: response.status,
-            headers,
-            rawBuffer,
-        };
+  const visited = /* @__PURE__ */ new Set();
+  let currentUrl = originalUrl;
+  let redirectCount = 0;
+  while (true) {
+    if (visited.has(currentUrl)) {
+      throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);
+    }
+    visited.add(currentUrl);
+    logger.debug("fetching", { url: currentUrl, attempt: redirectCount });
+    const signal = AbortSignal.timeout(timeoutMs);
+    let response;
+    try {
+      const ua = getRotatingUserAgent(getConfig());
+      const mergedHeaders = { "User-Agent": ua, ...options.headers };
+      if (options.conditionalHeaders?.ifNoneMatch) {
+        mergedHeaders["If-None-Match"] = options.conditionalHeaders.ifNoneMatch;
+      }
+      if (options.conditionalHeaders?.ifModifiedSince) {
+        mergedHeaders["If-Modified-Since"] = options.conditionalHeaders.ifModifiedSince;
+      }
+      response = await fetch(currentUrl, {
+        headers: mergedHeaders,
+        redirect: "manual",
+        signal
+      });
+    } catch (err) {
+      const isTimeout = err instanceof Error && err.name === "TimeoutError";
+      const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has(err.code ?? "");
+      const retryable = isTimeout || isConnErr;
+      throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });
+    }
+    if (response.status === 304) {
+      const headers2 = {};
+      response.headers.forEach((value, key) => {
+        headers2[key] = value;
+      });
+      try {
+        await response.arrayBuffer();
+      } catch {
+      }
+      return {
+        url: originalUrl,
+        finalUrl: currentUrl,
+        html: "",
+        contentType: response.headers.get("content-type") ?? "",
+        statusCode: 304,
+        headers: headers2
+      };
+    }
+    if (REDIRECT_STATUSES.has(response.status)) {
+      const location = response.headers.get("location");
+      if (!location) {
+        throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);
+      }
+      redirectCount++;
+      if (redirectCount > maxRedirects) {
+        throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);
+      }
+      currentUrl = new URL(location, currentUrl).toString();
+      continue;
+    }
+    if (RETRYABLE_STATUSES.has(response.status)) {
+      throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    const headers = {};
+    response.headers.forEach((value, key) => {
+      headers[key] = value;
+    });
+    const isPdf = contentType.includes("application/pdf");
+    let html;
+    let rawBuffer;
+    if (isPdf) {
+      const arrayBuf = await response.arrayBuffer();
+      rawBuffer = Buffer.from(arrayBuf);
+      html = "";
+    } else {
+      html = await response.text();
     }
+    return {
+      url: originalUrl,
+      finalUrl: currentUrl,
+      html,
+      contentType,
+      statusCode: response.status,
+      headers,
+      rawBuffer
+    };
+  }
 }
+export {
+  httpFetch
+};
 //# sourceMappingURL=http-client.js.map

package/dist/fetch/http-client.js.map

@@ -1 +1 @@
-{"version":3,"file":"http-client.js","sourceRoot":"","sources":["../../src/fetch/http-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAiB5C,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACpD,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;AACnF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAExD,MAAM,mBAAmB,GAAG;IAC1B,uHAAuH;IACvH,iHAAiH;IACjH,uGAAuG;CACxG,CAAC;AAEF,SAAS,oBAAoB,CAAC,MAAqC;IACjE,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,MAAM,CAAC,SAAS,CAAC;IAC9C,OAAO,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY;IACpC,IAAI,GAAG,YAAY,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAI,GAA6B,CAAC,IAAI,CAAC;QACjD,IAAI,IAAI,IAAI,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzD,iEAAiE;QACjE,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc;YAAE,OAAO,IAAI,CAAC;IAC/C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,OAAe;IAChC,OAAO,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;AAC1D,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,UAA4B,EAAE;IACzE,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,cAAc,CAAC;IAC7D,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;IAEzC,IAAI,SAAkB,CAAC;IAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YACzE,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;YACvF,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,GAAG,GAAG,CAAC;YAEhB,IAAI,GAAG,YAAY,cAAc,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;gBACpD,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,SAAS,GAAG,GAAG,YAAY,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAExF,IAAI,CAAC,SAAS,IAAI,OAAO,IAAI,UAAU,EAAE,CAAC;gBACxC,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,OAAO;gBACP,GAAG;gBACH,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,cAAe,SAAQ,KAAK;IACa;IAA7C,YAAY,OAAe,EAAkB,SAAkB;QAC7D,KAAK,CAAC,OAAO,CAAC,CAAC;QAD4B,cAAS,GAAT,SAAS,CAAS;QAE7D,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED,KAAK,UAAU,kBAAkB,CAC/B,WAAmB,EACnB,OAAyB,EACzB,SAAiB,EACjB,YAAoB,EACpB,MAAuC;IAEvC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,IAAI,UAAU,GAAG,WAAW,CAAC;IAC7B,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,cAAc,CAAC,6BAA6B,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAC;QAEtE,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAE9C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,oBAAoB,CAAC,SAAS,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/D,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACjC,OAAO,EAAE,aAAa;gBACtB,QAAQ,EAAE,QAAQ;gBAClB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,CAAC;YACtE,MAAM,SAAS,GAAG,GAAG,YAAY,KAAK,IAAI,qBAAqB,CAAC,GAAG,CAAE,GAA6B,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC/G,MAAM,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC;YACzC,MAAM,MAAM,CAAC,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,cAAc,CAAC,uCAAuC,UAAU,EAAE,EAAE,KAAK,CAAC,CAAC;YACvF,CAAC;YAED,aAAa,EAAE,CAAC;YAChB,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,cAAc,CAAC,wBAAwB,YAAY,UAAU,WAAW,EAAE,EAAE,KAAK,CAAC,CAAC;YAC/F,CAAC;YAED,6BAA6B;YAC7B,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,EAAE,CAAC;YACtD,SAAS;QACX,CAAC;QAED,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,MAAM,IAAI,cAAc,CAAC,QAAQ,QAAQ,CAAC,MAAM,SAAS,UAAU,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/E,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QAC/D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YACtC,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACtD,IAAI,IAAY,CAAC;QACjB,IAAI,SAA6B,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC9C,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;QAED,OAAO;YACL,GAAG,EAAE,WAAW;YAChB,QAAQ,EAAE,UAAU;YACpB,IAAI;YACJ,WAAW;YACX,UAAU,EAAE,QAAQ,CAAC,MAAM;YAC3B,OAAO;YACP,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/fetch/http-client.ts"],"sourcesContent":["import { getConfig } from '../config.js';\nimport { createLogger } from '../logger.js';\n\nexport interface HttpFetchOptions {\n  headers?: Record<string, string>;\n  timeoutMs?: number;\n  conditionalHeaders?: {\n    ifNoneMatch?: string;\n    ifModifiedSince?: string;\n  };\n}\n\nexport interface HttpFetchResult {\n  url: string;\n  finalUrl: string;\n  html: string;\n  contentType: string;\n  statusCode: number;\n  headers: Record<string, string>;\n  rawBuffer?: Buffer;\n}\n\nconst RETRYABLE_STATUSES = new Set([429, 502, 503]);\nconst RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED']);\nconst REDIRECT_STATUSES = new Set([301, 302, 307, 308]);\n\nconst DEFAULT_USER_AGENTS = [\n  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',\n];\n\nfunction getRotatingUserAgent(config: { userAgent?: string | null }): string {\n  if (config.userAgent) return config.userAgent;\n  return DEFAULT_USER_AGENTS[Math.floor(Math.random() * DEFAULT_USER_AGENTS.length)];\n}\n\nfunction isRetryableError(err: unknown): boolean {\n  if (err instanceof Error) {\n    const code = (err as NodeJS.ErrnoException).code;\n    if (code && RETRYABLE_ERROR_CODES.has(code)) return true;\n    // AbortSignal timeout throws DOMException with name TimeoutError\n    if (err.name === 'TimeoutError') return true;\n  }\n  return false;\n}\n\nfunction backoffMs(attempt: number): number {\n  return 500 * Math.pow(2, attempt) + Math.random() * 500;\n}\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport async function httpFetch(url: string, options: HttpFetchOptions = {}): Promise<HttpFetchResult> {\n  const config = getConfig();\n  const logger = createLogger('fetch');\n  const maxRetries = config.fetchMaxRetries;\n  const timeoutMs = options.timeoutMs ?? config.fetchTimeoutMs;\n  const maxRedirects = config.maxRedirects;\n\n  let lastError: unknown;\n\n  for (let attempt = 0; attempt <= maxRetries; attempt++) {\n    if (attempt > 0) {\n      const delay = backoffMs(attempt - 1);\n      logger.debug('retrying after backoff', { attempt, delayMs: delay, url });\n      await sleep(delay);\n    }\n\n    try {\n      const result = await fetchWithRedirects(url, options, timeoutMs, maxRedirects, logger);\n      return result;\n    } catch (err) {\n      lastError = err;\n\n      if (err instanceof HttpFetchError && !err.retryable) {\n        throw err;\n      }\n\n      const retryable = err instanceof HttpFetchError ? err.retryable : isRetryableError(err);\n\n      if (!retryable || attempt >= maxRetries) {\n        throw err;\n      }\n\n      logger.warn('fetch failed, will retry', {\n        attempt,\n        url,\n        error: err instanceof Error ? err.message : String(err),\n      });\n    }\n  }\n\n  throw lastError;\n}\n\nclass HttpFetchError extends Error {\n  constructor(message: string, public readonly retryable: boolean) {\n    super(message);\n    this.name = 'HttpFetchError';\n  }\n}\n\nasync function fetchWithRedirects(\n  originalUrl: string,\n  options: HttpFetchOptions,\n  timeoutMs: number,\n  maxRedirects: number,\n  logger: ReturnType<typeof createLogger>,\n): Promise<HttpFetchResult> {\n  const visited = new Set<string>();\n  let currentUrl = originalUrl;\n  let redirectCount = 0;\n\n  while (true) {\n    if (visited.has(currentUrl)) {\n      throw new HttpFetchError(`Redirect loop detected at ${currentUrl}`, false);\n    }\n    visited.add(currentUrl);\n\n    logger.debug('fetching', { url: currentUrl, attempt: redirectCount });\n\n    const signal = AbortSignal.timeout(timeoutMs);\n\n    let response: Response;\n    try {\n      const ua = getRotatingUserAgent(getConfig());\n      const mergedHeaders: Record<string, string> = { 'User-Agent': ua, ...options.headers };\n      // Conditional GET: inject If-None-Match / If-Modified-Since so the\n      // server can return 304 + no body when the resource hasn't changed.\n      // Callers (eg. etag-incremental crawl) wire these from the persisted\n      // crawl_etags row for the URL.\n      if (options.conditionalHeaders?.ifNoneMatch) {\n        mergedHeaders['If-None-Match'] = options.conditionalHeaders.ifNoneMatch;\n      }\n      if (options.conditionalHeaders?.ifModifiedSince) {\n        mergedHeaders['If-Modified-Since'] = options.conditionalHeaders.ifModifiedSince;\n      }\n      response = await fetch(currentUrl, {\n        headers: mergedHeaders,\n        redirect: 'manual',\n        signal,\n      });\n    } catch (err) {\n      const isTimeout = err instanceof Error && err.name === 'TimeoutError';\n      const isConnErr = err instanceof Error && RETRYABLE_ERROR_CODES.has((err as NodeJS.ErrnoException).code ?? '');\n      const retryable = isTimeout || isConnErr;\n      throw Object.assign(err instanceof Error ? err : new Error(String(err)), { retryable });\n    }\n\n    if (response.status === 304) {\n      const headers: Record<string, string> = {};\n      response.headers.forEach((value, key) => {\n        headers[key] = value;\n      });\n      // Drain so the connection can be released; ignore the (empty) body.\n      try { await response.arrayBuffer(); } catch { /* */ }\n      return {\n        url: originalUrl,\n        finalUrl: currentUrl,\n        html: '',\n        contentType: response.headers.get('content-type') ?? '',\n        statusCode: 304,\n        headers,\n      };\n    }\n\n    if (REDIRECT_STATUSES.has(response.status)) {\n      const location = response.headers.get('location');\n      if (!location) {\n        throw new HttpFetchError(`Redirect with no location header at ${currentUrl}`, false);\n      }\n\n      redirectCount++;\n      if (redirectCount > maxRedirects) {\n        throw new HttpFetchError(`Too many redirects (>${maxRedirects}) from ${originalUrl}`, false);\n      }\n\n      // Resolve relative redirects\n      currentUrl = new URL(location, currentUrl).toString();\n      continue;\n    }\n\n    if (RETRYABLE_STATUSES.has(response.status)) {\n      throw new HttpFetchError(`HTTP ${response.status} from ${currentUrl}`, true);\n    }\n\n    const contentType = response.headers.get('content-type') ?? '';\n    const headers: Record<string, string> = {};\n    response.headers.forEach((value, key) => {\n      headers[key] = value;\n    });\n\n    const isPdf = contentType.includes('application/pdf');\n    let html: string;\n    let rawBuffer: Buffer | undefined;\n\n    if (isPdf) {\n      const arrayBuf = await response.arrayBuffer();\n      rawBuffer = Buffer.from(arrayBuf);\n      html = '';\n    } else {\n      html = await response.text();\n    }\n\n    return {\n      url: originalUrl,\n      finalUrl: currentUrl,\n      html,\n      contentType,\n      statusCode: response.status,\n      headers,\n      rawBuffer,\n    };\n  }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAqB7B,MAAM,qBAAqB,oBAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC;AAClD,MAAM,wBAAwB,oBAAI,IAAI,CAAC,cAAc,aAAa,cAAc,CAAC;AACjF,MAAM,oBAAoB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,GAAG,CAAC;AAEtD,MAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AACF;AAEA,SAAS,qBAAqB,QAA+C;AAC3E,MAAI,OAAO,UAAW,QAAO,OAAO;AACpC,SAAO,oBAAoB,KAAK,MAAM,KAAK,OAAO,IAAI,oBAAoB,MAAM,CAAC;AACnF;AAEA,SAAS,iBAAiB,KAAuB;AAC/C,MAAI,eAAe,OAAO;AACxB,UAAM,OAAQ,IAA8B;AAC5C,QAAI,QAAQ,sBAAsB,IAAI,IAAI,EAAG,QAAO;AAEpD,QAAI,IAAI,SAAS,eAAgB,QAAO;AAAA,EAC1C;AACA,SAAO;AACT;AAEA,SAAS,UAAU,SAAyB;AAC1C,SAAO,MAAM,KAAK,IAAI,GAAG,OAAO,IAAI,KAAK,OAAO,IAAI;AACtD;AAEA,SAAS,MAAM,IAA2B;AACxC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AAEA,eAAsB,UAAU,KAAa,UAA4B,CAAC,GAA6B;AACrG,QAAM,SAAS,UAAU;AACzB,QAAM,SAAS,aAAa,OAAO;AACnC,QAAM,aAAa,OAAO;AAC1B,QAAM,YAAY,QAAQ,aAAa,OAAO;AAC9C,QAAM,eAAe,OAAO;AAE5B,MAAI;AAEJ,WAAS,UAAU,GAAG,WAAW,YAAY,WAAW;AACtD,QAAI,UAAU,GAAG;AACf,YAAM,QAAQ,UAAU,UAAU,CAAC;AACnC,aAAO,MAAM,0BAA0B,EAAE,SAAS,SAAS,OAAO,IAAI,CAAC;AACvE,YAAM,MAAM,KAAK;AAAA,IACnB;AAEA,QAAI;AACF,YAAM,SAAS,MAAM,mBAAmB,KAAK,SAAS,WAAW,cAAc,MAAM;AACrF,aAAO;AAAA,IACT,SAAS,KAAK;AACZ,kBAAY;AAEZ,UAAI,eAAe,kBAAkB,CAAC,IAAI,WAAW;AACnD,cAAM;AAAA,MACR;AAEA,YAAM,YAAY,eAAe,iBAAiB,IAAI,YAAY,iBAAiB,GAAG;AAEtF,UAAI,CAAC,aAAa,WAAW,YAAY;AACvC,cAAM;AAAA,MACR;AAEA,aAAO,KAAK,4BAA4B;AAAA,QACtC;AAAA,QACA;AAAA,QACA,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,MACxD,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM;AACR;AAEA,MAAM,uBAAuB,MAAM;AAAA,EACjC,YAAY,SAAiC,WAAoB;AAC/D,UAAM,OAAO;AAD8B;AAE3C,SAAK,OAAO;AAAA,EACd;AAAA,EAH6C;AAI/C;AAEA,eAAe,mBACb,aACA,SACA,WACA,cACA,QAC0B;AAC1B,QAAM,UAAU,oBAAI,IAAY;AAChC,MAAI,aAAa;AACjB,MAAI,gBAAgB;AAEpB,SAAO,MAAM;AACX,QAAI,QAAQ,IAAI,UAAU,GAAG;AAC3B,YAAM,IAAI,eAAe,6BAA6B,UAAU,IAAI,KAAK;AAAA,IAC3E;AACA,YAAQ,IAAI,UAAU;AAEtB,WAAO,MAAM,YAAY,EAAE,KAAK,YAAY,SAAS,cAAc,CAAC;AAEpE,UAAM,SAAS,YAAY,QAAQ,SAAS;AAE5C,QAAI;AACJ,QAAI;AACF,YAAM,KAAK,qBAAqB,UAAU,CAAC;AAC3C,YAAM,gBAAwC,EAAE,cAAc,IAAI,GAAG,QAAQ,QAAQ;AAKrF,UAAI,QAAQ,oBAAoB,aAAa;AAC3C,sBAAc,eAAe,IAAI,QAAQ,mBAAmB;AAAA,MAC9D;AACA,UAAI,QAAQ,oBAAoB,iBAAiB;AAC/C,sBAAc,mBAAmB,IAAI,QAAQ,mBAAmB;AAAA,MAClE;AACA,iBAAW,MAAM,MAAM,YAAY;AAAA,QACjC,SAAS;AAAA,QACT,UAAU;AAAA,QACV;AAAA,MACF,CAAC;AAAA,IACH,SAAS,KAAK;AACZ,YAAM,YAAY,eAAe,SAAS,IAAI,SAAS;AACvD,YAAM,YAAY,eAAe,SAAS,sBAAsB,IAAK,IAA8B,QAAQ,EAAE;AAC7G,YAAM,YAAY,aAAa;AAC/B,YAAM,OAAO,OAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC;AAAA,IACxF;AAEA,QAAI,SAAS,WAAW,KAAK;AAC3B,YAAMA,WAAkC,CAAC;AACzC,eAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,QAAAA,SAAQ,GAAG,IAAI;AAAA,MACjB,CAAC;AAED,UAAI;AAAE,cAAM,SAAS,YAAY;AAAA,MAAG,QAAQ;AAAA,MAAQ;AACpD,aAAO;AAAA,QACL,KAAK;AAAA,QACL,UAAU;AAAA,QACV,MAAM;AAAA,QACN,aAAa,SAAS,QAAQ,IAAI,cAAc,KAAK;AAAA,QACrD,YAAY;AAAA,QACZ,SAAAA;AAAA,MACF;AAAA,IACF;AAEA,QAAI,kBAAkB,IAAI,SAAS,MAAM,GAAG;AAC1C,YAAM,WAAW,SAAS,QAAQ,IAAI,UAAU;AAChD,UAAI,CAAC,UAAU;AACb,cAAM,IAAI,eAAe,uCAAuC,UAAU,IAAI,KAAK;AAAA,MACrF;AAEA;AACA,UAAI,gBAAgB,cAAc;AAChC,cAAM,IAAI,eAAe,wBAAwB,YAAY,UAAU,WAAW,IAAI,KAAK;AAAA,MAC7F;AAGA,mBAAa,IAAI,IAAI,UAAU,UAAU,EAAE,SAAS;AACpD;AAAA,IACF;AAEA,QAAI,mBAAmB,IAAI,SAAS,MAAM,GAAG;AAC3C,YAAM,IAAI,eAAe,QAAQ,SAAS,MAAM,SAAS,UAAU,IAAI,IAAI;AAAA,IAC7E;AAEA,UAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,UAAM,UAAkC,CAAC;AACzC,aAAS,QAAQ,QAAQ,CAAC,OAAO,QAAQ;AACvC,cAAQ,GAAG,IAAI;AAAA,IACjB,CAAC;AAED,UAAM,QAAQ,YAAY,SAAS,iBAAiB;AACpD,QAAI;AACJ,QAAI;AAEJ,QAAI,OAAO;AACT,YAAM,WAAW,MAAM,SAAS,YAAY;AAC5C,kBAAY,OAAO,KAAK,QAAQ;AAChC,aAAO;AAAA,IACT,OAAO;AACL,aAAO,MAAM,SAAS,KAAK;AAAA,IAC7B;AAEA,WAAO;AAAA,MACL,KAAK;AAAA,MACL,UAAU;AAAA,MACV;AAAA,MACA;AAAA,MACA,YAAY,SAAS;AAAA,MACrB;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;","names":["headers"]}