@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/tls-fingerprint.js

@@ -0,0 +1,81 @@
+import { Agent as HttpsAgent } from "node:https";
+import { connect as tlsConnect } from "node:tls";
+const CHROME_CIPHERS = [
+  "TLS_AES_128_GCM_SHA256",
+  "TLS_AES_256_GCM_SHA384",
+  "TLS_CHACHA20_POLY1305_SHA256",
+  "ECDHE-ECDSA-AES128-GCM-SHA256",
+  "ECDHE-RSA-AES128-GCM-SHA256",
+  "ECDHE-ECDSA-AES256-GCM-SHA384",
+  "ECDHE-RSA-AES256-GCM-SHA384",
+  "ECDHE-ECDSA-CHACHA20-POLY1305",
+  "ECDHE-RSA-CHACHA20-POLY1305",
+  "ECDHE-RSA-AES128-SHA",
+  "ECDHE-RSA-AES256-SHA",
+  "AES128-GCM-SHA256",
+  "AES256-GCM-SHA384",
+  "AES128-SHA",
+  "AES256-SHA"
+].join(":");
+const CHROME_CURVES = "X25519:P-256:P-384";
+function createChromeTlsAgent(opts = {}) {
+  return new HttpsAgent({
+    ciphers: CHROME_CIPHERS,
+    ecdhCurve: CHROME_CURVES,
+    minVersion: "TLSv1.2",
+    maxVersion: "TLSv1.3",
+    rejectUnauthorized: opts.rejectUnauthorized ?? true,
+    keepAlive: opts.keepAlive ?? true,
+    keepAliveMsecs: opts.keepAliveMsecs ?? 1e4,
+    maxSockets: opts.maxSockets ?? 50
+  });
+}
+async function probeTlsFingerprint(host, port = 443) {
+  return new Promise((resolve, reject) => {
+    const socket = tlsConnect({
+      host,
+      port,
+      servername: host,
+      ciphers: CHROME_CIPHERS,
+      ecdhCurve: CHROME_CURVES,
+      minVersion: "TLSv1.2",
+      rejectUnauthorized: false
+    }, () => {
+      const cert = socket.getPeerCertificate();
+      resolve({
+        protocol: socket.getProtocol() ?? "unknown",
+        cipher: socket.getCipher()?.name ?? "unknown",
+        authorized: socket.authorized,
+        subjectCN: cert?.subject?.CN?.toString() ?? null
+      });
+      socket.destroy();
+    });
+    socket.on("error", reject);
+    setTimeout(() => {
+      socket.destroy();
+      reject(new Error("TLS probe timeout"));
+    }, 5e3);
+  });
+}
+const chromeTlsAgent = createChromeTlsAgent();
+const CHROME_TLS_HEADERS = {
+  "Accept-Encoding": "gzip, deflate, br, zstd",
+  "Accept-Language": "en-US,en;q=0.9",
+  "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
+  "sec-ch-ua-mobile": "?0",
+  "sec-ch-ua-platform": '"Windows"',
+  "Sec-Fetch-Dest": "document",
+  "Sec-Fetch-Mode": "navigate",
+  "Sec-Fetch-Site": "none",
+  "Sec-Fetch-User": "?1",
+  "Upgrade-Insecure-Requests": "1"
+};
+export {
+  CHROME_CIPHERS,
+  CHROME_CURVES,
+  CHROME_TLS_HEADERS,
+  chromeTlsAgent,
+  createChromeTlsAgent,
+  probeTlsFingerprint
+};
+//# sourceMappingURL=tls-fingerprint.js.map

package/dist/tls-fingerprint.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/tls-fingerprint.ts"],
+  "sourcesContent": ["/**\n * TLS fingerprint spoofing \u2014 Chrome JA3 cipher suite ordering.\n *\n * Node.js TLS doesn't expose JA3 hash control directly, but ordering\n * cipher suites to match Chrome's TLS ClientHello reduces fingerprint\n * detectability vs the default OpenSSL order.\n *\n * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).\n */\n\nimport { Agent as HttpsAgent } from 'node:https';\nimport { connect as tlsConnect } from 'node:tls';\n\n// Chrome 120+ TLS 1.3 + 1.2 cipher suite order\nexport const CHROME_CIPHERS = [\n  'TLS_AES_128_GCM_SHA256',\n  'TLS_AES_256_GCM_SHA384',\n  'TLS_CHACHA20_POLY1305_SHA256',\n  'ECDHE-ECDSA-AES128-GCM-SHA256',\n  'ECDHE-RSA-AES128-GCM-SHA256',\n  'ECDHE-ECDSA-AES256-GCM-SHA384',\n  'ECDHE-RSA-AES256-GCM-SHA384',\n  'ECDHE-ECDSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-CHACHA20-POLY1305',\n  'ECDHE-RSA-AES128-SHA',\n  'ECDHE-RSA-AES256-SHA',\n  'AES128-GCM-SHA256',\n  'AES256-GCM-SHA384',\n  'AES128-SHA',\n  'AES256-SHA',\n].join(':');\n\n// Chrome's ECDH curve preference order\nexport const CHROME_CURVES = 'X25519:P-256:P-384';\n\nexport interface TlsFingerprintOptions {\n  rejectUnauthorized?: boolean;\n  keepAlive?: boolean;\n  keepAliveMsecs?: number;\n  maxSockets?: number;\n}\n\n/**\n * Returns an HTTPS agent configured with Chrome-like TLS settings.\n * Drop-in replacement for the default agent in fetch/got/axios.\n */\nexport function createChromeTlsAgent(opts: TlsFingerprintOptions = {}): HttpsAgent {\n  return new HttpsAgent({\n    ciphers: CHROME_CIPHERS,\n    ecdhCurve: CHROME_CURVES,\n    minVersion: 'TLSv1.2',\n    maxVersion: 'TLSv1.3',\n    rejectUnauthorized: opts.rejectUnauthorized ?? true,\n    keepAlive: opts.keepAlive ?? true,\n    keepAliveMsecs: opts.keepAliveMsecs ?? 10_000,\n    maxSockets: opts.maxSockets ?? 50,\n  });\n}\n\n/**\n * Probe a host and return its JA3-relevant TLS parameters.\n * Useful for verifying that the fingerprint changed vs default.\n */\nexport async function probeTlsFingerprint(host: string, port = 443): Promise<{\n  protocol: string;\n  cipher: string;\n  authorized: boolean;\n  subjectCN: string | null;\n}> {\n  return new Promise((resolve, reject) => {\n    const socket = tlsConnect({\n      host,\n      port,\n      servername: host,\n      ciphers: CHROME_CIPHERS,\n      ecdhCurve: CHROME_CURVES,\n      minVersion: 'TLSv1.2',\n      rejectUnauthorized: false,\n    }, () => {\n      const cert = socket.getPeerCertificate();\n      resolve({\n        protocol: socket.getProtocol() ?? 'unknown',\n        cipher: socket.getCipher()?.name ?? 'unknown',\n        authorized: socket.authorized,\n        subjectCN: cert?.subject?.CN?.toString() ?? null,\n      });\n      socket.destroy();\n    });\n    socket.on('error', reject);\n    setTimeout(() => { socket.destroy(); reject(new Error('TLS probe timeout')); }, 5_000);\n  });\n}\n\n/**\n * Node.js fetch dispatcher options with Chrome TLS agent.\n * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.\n */\nexport const chromeTlsAgent = createChromeTlsAgent();\n\nexport const CHROME_TLS_HEADERS = {\n  'Accept-Encoding': 'gzip, deflate, br, zstd',\n  'Accept-Language': 'en-US,en;q=0.9',\n  'sec-ch-ua': '\"Google Chrome\";v=\"120\", \"Chromium\";v=\"120\", \"Not-A.Brand\";v=\"99\"',\n  'sec-ch-ua-mobile': '?0',\n  'sec-ch-ua-platform': '\"Windows\"',\n  'Sec-Fetch-Dest': 'document',\n  'Sec-Fetch-Mode': 'navigate',\n  'Sec-Fetch-Site': 'none',\n  'Sec-Fetch-User': '?1',\n  'Upgrade-Insecure-Requests': '1',\n};\n"],
+  "mappings": "AAUA,SAAS,SAAS,kBAAkB;AACpC,SAAS,WAAW,kBAAkB;AAG/B,MAAM,iBAAiB;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,EAAE,KAAK,GAAG;AAGH,MAAM,gBAAgB;AAatB,SAAS,qBAAqB,OAA8B,CAAC,GAAe;AACjF,SAAO,IAAI,WAAW;AAAA,IACpB,SAAS;AAAA,IACT,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,oBAAoB,KAAK,sBAAsB;AAAA,IAC/C,WAAW,KAAK,aAAa;AAAA,IAC7B,gBAAgB,KAAK,kBAAkB;AAAA,IACvC,YAAY,KAAK,cAAc;AAAA,EACjC,CAAC;AACH;AAMA,eAAsB,oBAAoB,MAAc,OAAO,KAK5D;AACD,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,SAAS,WAAW;AAAA,MACxB;AAAA,MACA;AAAA,MACA,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,oBAAoB;AAAA,IACtB,GAAG,MAAM;AACP,YAAM,OAAO,OAAO,mBAAmB;AACvC,cAAQ;AAAA,QACN,UAAU,OAAO,YAAY,KAAK;AAAA,QAClC,QAAQ,OAAO,UAAU,GAAG,QAAQ;AAAA,QACpC,YAAY,OAAO;AAAA,QACnB,WAAW,MAAM,SAAS,IAAI,SAAS,KAAK;AAAA,MAC9C,CAAC;AACD,aAAO,QAAQ;AAAA,IACjB,CAAC;AACD,WAAO,GAAG,SAAS,MAAM;AACzB,eAAW,MAAM;AAAE,aAAO,QAAQ;AAAG,aAAO,IAAI,MAAM,mBAAmB,CAAC;AAAA,IAAG,GAAG,GAAK;AAAA,EACvF,CAAC;AACH;AAMO,MAAM,iBAAiB,qBAAqB;AAE5C,MAAM,qBAAqB;AAAA,EAChC,mBAAmB;AAAA,EACnB,mBAAmB;AAAA,EACnB,aAAa;AAAA,EACb,oBAAoB;AAAA,EACpB,sBAAsB;AAAA,EACtB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,kBAAkB;AAAA,EAClB,6BAA6B;AAC/B;",
+  "names": []
+}

package/dist/url-pattern-completer.d.ts

@@ -0,0 +1,28 @@
+/**
+ * P1: URL Pattern LLM Completion
+ *
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
+ * but were never linked from within the app. Validates each candidate with
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
+ *
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
+ * candidate count. Does NOT launch a browser.
+ */
+export interface UrlCandidate {
+    url: string;
+    confidence: number;
+    reason: string;
+    status: 'hit' | 'miss' | 'error';
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+    baseUrl?: string;
+}
+/**
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
+ * the pattern of already-discovered URLs.
+ */
+export declare function runUrlPatternCompletion(discoveredUrls: string[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<UrlCandidate[]>;
+export {};

package/dist/url-pattern-completer.js

@@ -0,0 +1,122 @@
+import { logger as rootLogger } from "./logger.js";
+const logger = rootLogger.child({ module: "url-pattern-completer" });
+const SUPPORTED = /* @__PURE__ */ new Set(["openai", "anthropic"]);
+function resolveTextLLM(llmCfg) {
+  if (llmCfg?.provider === "openai" && llmCfg.apiKey) {
+    return { provider: "openai", apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };
+  }
+  if (llmCfg?.provider === "anthropic" && llmCfg.apiKey) {
+    return { provider: "anthropic", apiKey: llmCfg.apiKey, model: llmCfg.model };
+  }
+  if (process.env.OPENAI_API_KEY) {
+    return { provider: "openai", apiKey: process.env.OPENAI_API_KEY, model: "gpt-4o-mini" };
+  }
+  if (process.env.ANTHROPIC_API_KEY) {
+    return { provider: "anthropic", apiKey: process.env.ANTHROPIC_API_KEY, model: "claude-haiku-4-5-20251001" };
+  }
+  return null;
+}
+async function runUrlPatternCompletion(discoveredUrls, appUrl, llmCfg) {
+  const effective = resolveTextLLM(llmCfg);
+  if (!effective || discoveredUrls.length < 2) return [];
+  const origin = (() => {
+    try {
+      return new URL(appUrl).origin;
+    } catch {
+      return "";
+    }
+  })();
+  if (!origin) return [];
+  const ASSET_EXT = /\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;
+  const internalUrls = discoveredUrls.filter((u) => {
+    try {
+      return new URL(u).origin === origin && !ASSET_EXT.test(u);
+    } catch {
+      return false;
+    }
+  }).slice(0, 60);
+  if (internalUrls.length < 2) return [];
+  const systemPrompt = "You are a URL pattern analyst. Always respond with valid JSON only.";
+  const userPrompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
+
+Discovered URLs:
+${internalUrls.map((u) => {
+    try {
+      return new URL(u).pathname + new URL(u).search;
+    } catch {
+      return u;
+    }
+  }).join("\n")}
+
+Based on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.
+
+Rules:
+- Only suggest URLs on the same domain
+- Use relative paths (starting with /)
+- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them
+- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)
+- Confidence 0.0\u20131.0: how certain you are the page exists
+
+Respond with ONLY valid JSON, no explanation:
+{"candidates":[{"path":"/settings/billing","confidence":0.85,"reason":"settings subpage pattern from /settings/profile"},{"path":"/admin/users","confidence":0.6,"reason":"admin prefix implied by /admin/dashboard"}]}`;
+  try {
+    let text = "";
+    if (effective.provider === "openai") {
+      const { OpenAI } = await import("openai");
+      const client = new OpenAI({ apiKey: effective.apiKey, ...effective.baseUrl ? { baseURL: effective.baseUrl } : {} });
+      const response = await client.chat.completions.create({
+        model: effective.model ?? "gpt-4o-mini",
+        max_tokens: 1024,
+        temperature: 0,
+        messages: [
+          { role: "system", content: systemPrompt },
+          { role: "user", content: userPrompt }
+        ]
+      });
+      text = response.choices[0]?.message?.content ?? "";
+    } else {
+      const Anthropic = (await import("@anthropic-ai/sdk")).default;
+      const client = new Anthropic({ apiKey: effective.apiKey });
+      const response = await client.messages.create({
+        model: effective.model ?? "claude-haiku-4-5-20251001",
+        max_tokens: 1024,
+        temperature: 0,
+        // @ts-ignore: cache_control supported in beta
+        system: [{ type: "text", text: systemPrompt, cache_control: { type: "ephemeral" } }],
+        messages: [{ role: "user", content: userPrompt }]
+      });
+      text = response.content[0]?.text ?? "";
+    }
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawCandidates = parsed.candidates ?? [];
+    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];
+    const results = [];
+    for (const c of rawCandidates.slice(0, 20)) {
+      if (!c.path || typeof c.path !== "string") continue;
+      const fullUrl = c.path.startsWith("http") ? c.path : `${origin}${c.path.startsWith("/") ? "" : "/"}${c.path}`;
+      try {
+        const controller = new AbortController();
+        const timeout = setTimeout(() => controller.abort(), 6e3);
+        const res = await fetch(fullUrl, { method: "HEAD", signal: controller.signal, redirect: "follow" }).catch(() => null);
+        clearTimeout(timeout);
+        const status = res?.ok ? "hit" : "miss";
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? "", status });
+        logger.info({ url: fullUrl, httpStatus: res?.status ?? "timeout", status }, "[url-completer] candidate checked");
+      } catch {
+        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? "", status: "error" });
+      }
+    }
+    const hits = results.filter((r) => r.status === "hit");
+    logger.info({ total: results.length, hits: hits.length, provider: effective.provider }, "[url-completer] pattern completion done");
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, "[url-completer] LLM call failed \u2014 skipping pattern completion");
+    return [];
+  }
+}
+export {
+  runUrlPatternCompletion
+};
+//# sourceMappingURL=url-pattern-completer.js.map

package/dist/url-pattern-completer.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/url-pattern-completer.ts"],
+  "sourcesContent": ["/**\n * P1: URL Pattern LLM Completion\n *\n * After BFS crawl completes, runs a single LLM call to predict URLs that exist\n * but were never linked from within the app. Validates each candidate with\n * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).\n *\n * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on\n * candidate count. Does NOT launch a browser.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'url-pattern-completer' });\n\nexport interface UrlCandidate {\n  url: string;\n  confidence: number;\n  reason: string;\n  status: 'hit' | 'miss' | 'error';\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n  baseUrl?: string;\n}\n\n// Only these two providers are supported by this module (they use standard SDK calls).\n// NVIDIA, groq, ollama, openrouter, etc. are skipped \u2014 fall through to env fallback.\nconst SUPPORTED = new Set(['openai', 'anthropic']);\n\n/**\n * Resolve the effective LLM to use. Priority: openai \u2192 anthropic \u2192 skip others.\n * Falls back to env vars when the platform-configured provider is unsupported.\n */\nfunction resolveTextLLM(llmCfg: LlmCfg | null | undefined): { provider: 'openai' | 'anthropic'; apiKey: string; model: string; baseUrl?: string } | null {\n  // Use platform config if it's a supported provider with a key\n  if (llmCfg?.provider === 'openai' && llmCfg.apiKey) {\n    return { provider: 'openai', apiKey: llmCfg.apiKey, model: llmCfg.model, baseUrl: llmCfg.baseUrl };\n  }\n  if (llmCfg?.provider === 'anthropic' && llmCfg.apiKey) {\n    return { provider: 'anthropic', apiKey: llmCfg.apiKey, model: llmCfg.model };\n  }\n  // Platform provider is unsupported (nvidia, groq, etc.) or no key \u2014 fall back to env vars.\n  // OpenAI takes priority over Anthropic.\n  if (process.env.OPENAI_API_KEY) {\n    return { provider: 'openai', apiKey: process.env.OPENAI_API_KEY, model: 'gpt-4o-mini' };\n  }\n  if (process.env.ANTHROPIC_API_KEY) {\n    return { provider: 'anthropic', apiKey: process.env.ANTHROPIC_API_KEY, model: 'claude-haiku-4-5-20251001' };\n  }\n  return null;\n}\n\n/**\n * Returns validated URL candidates (HEAD=200) predicted by LLM from\n * the pattern of already-discovered URLs.\n */\nexport async function runUrlPatternCompletion(\n  discoveredUrls: string[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<UrlCandidate[]> {\n  const effective = resolveTextLLM(llmCfg);\n  if (!effective || discoveredUrls.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Only send internal URLs (same origin, no assets)\n  const ASSET_EXT = /\\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;\n  const internalUrls = discoveredUrls\n    .filter(u => { try { return new URL(u).origin === origin && !ASSET_EXT.test(u); } catch { return false; } })\n    .slice(0, 60); // cap context size\n\n  if (internalUrls.length < 2) return [];\n\n  const systemPrompt = 'You are a URL pattern analyst. Always respond with valid JSON only.';\n  const userPrompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.\n\nDiscovered URLs:\n${internalUrls.map(u => { try { return new URL(u).pathname + new URL(u).search; } catch { return u; } }).join('\\n')}\n\nBased on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.\n\nRules:\n- Only suggest URLs on the same domain\n- Use relative paths (starting with /)\n- Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them\n- Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)\n- Confidence 0.0\u20131.0: how certain you are the page exists\n\nRespond with ONLY valid JSON, no explanation:\n{\"candidates\":[{\"path\":\"/settings/billing\",\"confidence\":0.85,\"reason\":\"settings subpage pattern from /settings/profile\"},{\"path\":\"/admin/users\",\"confidence\":0.6,\"reason\":\"admin prefix implied by /admin/dashboard\"}]}`;\n\n  try {\n    let text = '';\n\n    if (effective.provider === 'openai') {\n      const { OpenAI } = await import('openai');\n      const client = new OpenAI({ apiKey: effective.apiKey, ...(effective.baseUrl ? { baseURL: effective.baseUrl } : {}) });\n      const response = await client.chat.completions.create({\n        model: effective.model ?? 'gpt-4o-mini',\n        max_tokens: 1024,\n        temperature: 0,\n        messages: [\n          { role: 'system', content: systemPrompt },\n          { role: 'user', content: userPrompt },\n        ],\n      });\n      text = response.choices[0]?.message?.content ?? '';\n    } else {\n      const Anthropic = (await import('@anthropic-ai/sdk')).default;\n      const client = new Anthropic({ apiKey: effective.apiKey });\n      const response = await client.messages.create({\n        model: effective.model ?? 'claude-haiku-4-5-20251001',\n        max_tokens: 1024,\n        temperature: 0,\n        // @ts-ignore: cache_control supported in beta\n        system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }],\n        messages: [{ role: 'user', content: userPrompt }],\n      });\n      text = (response.content[0] as any)?.text ?? '';\n    }\n\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawCandidates: Array<{ path: string; confidence: number; reason: string }> =\n      parsed.candidates ?? [];\n\n    if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];\n\n    // HEAD-validate each candidate\n    const results: UrlCandidate[] = [];\n    for (const c of rawCandidates.slice(0, 20)) {\n      if (!c.path || typeof c.path !== 'string') continue;\n      const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;\n      try {\n        const controller = new AbortController();\n        const timeout = setTimeout(() => controller.abort(), 6_000);\n        const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n        clearTimeout(timeout);\n        const status: UrlCandidate['status'] = res?.ok ? 'hit' : 'miss';\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });\n        logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');\n      } catch {\n        results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });\n      }\n    }\n\n    const hits = results.filter(r => r.status === 'hit');\n    logger.info({ total: results.length, hits: hits.length, provider: effective.provider }, '[url-completer] pattern completion done');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[url-completer] LLM call failed \u2014 skipping pattern completion');\n    return [];\n  }\n}\n"],
+  "mappings": "AAWA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,wBAAwB,CAAC;AAkBnE,MAAM,YAAY,oBAAI,IAAI,CAAC,UAAU,WAAW,CAAC;AAMjD,SAAS,eAAe,QAAiI;AAEvJ,MAAI,QAAQ,aAAa,YAAY,OAAO,QAAQ;AAClD,WAAO,EAAE,UAAU,UAAU,QAAQ,OAAO,QAAQ,OAAO,OAAO,OAAO,SAAS,OAAO,QAAQ;AAAA,EACnG;AACA,MAAI,QAAQ,aAAa,eAAe,OAAO,QAAQ;AACrD,WAAO,EAAE,UAAU,aAAa,QAAQ,OAAO,QAAQ,OAAO,OAAO,MAAM;AAAA,EAC7E;AAGA,MAAI,QAAQ,IAAI,gBAAgB;AAC9B,WAAO,EAAE,UAAU,UAAU,QAAQ,QAAQ,IAAI,gBAAgB,OAAO,cAAc;AAAA,EACxF;AACA,MAAI,QAAQ,IAAI,mBAAmB;AACjC,WAAO,EAAE,UAAU,aAAa,QAAQ,QAAQ,IAAI,mBAAmB,OAAO,4BAA4B;AAAA,EAC5G;AACA,SAAO;AACT;AAMA,eAAsB,wBACpB,gBACA,QACA,QACyB;AACzB,QAAM,YAAY,eAAe,MAAM;AACvC,MAAI,CAAC,aAAa,eAAe,SAAS,EAAG,QAAO,CAAC;AAErD,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,YAAY;AAClB,QAAM,eAAe,eAClB,OAAO,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,UAAU,CAAC,UAAU,KAAK,CAAC;AAAA,IAAG,QAAQ;AAAE,aAAO;AAAA,IAAO;AAAA,EAAE,CAAC,EAC1G,MAAM,GAAG,EAAE;AAEd,MAAI,aAAa,SAAS,EAAG,QAAO,CAAC;AAErC,QAAM,eAAe;AACrB,QAAM,aAAa;AAAA;AAAA;AAAA,EAGnB,aAAa,IAAI,OAAK;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,CAAC,EAAE,WAAW,IAAI,IAAI,CAAC,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAG;AAAA,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcjH,MAAI;AACF,QAAI,OAAO;AAEX,QAAI,UAAU,aAAa,UAAU;AACnC,YAAM,EAAE,OAAO,IAAI,MAAM,OAAO,QAAQ;AACxC,YAAM,SAAS,IAAI,OAAO,EAAE,QAAQ,UAAU,QAAQ,GAAI,UAAU,UAAU,EAAE,SAAS,UAAU,QAAQ,IAAI,CAAC,EAAG,CAAC;AACpH,YAAM,WAAW,MAAM,OAAO,KAAK,YAAY,OAAO;AAAA,QACpD,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA,QACb,UAAU;AAAA,UACR,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,UACxC,EAAE,MAAM,QAAQ,SAAS,WAAW;AAAA,QACtC;AAAA,MACF,CAAC;AACD,aAAO,SAAS,QAAQ,CAAC,GAAG,SAAS,WAAW;AAAA,IAClD,OAAO;AACL,YAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,YAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,UAAU,OAAO,CAAC;AACzD,YAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,QAC5C,OAAO,UAAU,SAAS;AAAA,QAC1B,YAAY;AAAA,QACZ,aAAa;AAAA;AAAA,QAEb,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,cAAc,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,QACnF,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,WAAW,CAAC;AAAA,MAClD,CAAC;AACD,aAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AAAA,IAC/C;AAEA,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,gBACJ,OAAO,cAAc,CAAC;AAExB,QAAI,CAAC,MAAM,QAAQ,aAAa,KAAK,cAAc,WAAW,EAAG,QAAO,CAAC;AAGzE,UAAM,UAA0B,CAAC;AACjC,eAAW,KAAK,cAAc,MAAM,GAAG,EAAE,GAAG;AAC1C,UAAI,CAAC,EAAE,QAAQ,OAAO,EAAE,SAAS,SAAU;AAC3C,YAAM,UAAU,EAAE,KAAK,WAAW,MAAM,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,EAAE,KAAK,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,EAAE,IAAI;AAC3G,UAAI;AACF,cAAM,aAAa,IAAI,gBAAgB;AACvC,cAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,cAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,qBAAa,OAAO;AACpB,cAAM,SAAiC,KAAK,KAAK,QAAQ;AACzD,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,OAAO,CAAC;AAC9F,eAAO,KAAK,EAAE,KAAK,SAAS,YAAY,KAAK,UAAU,WAAW,OAAO,GAAG,mCAAmC;AAAA,MACjH,QAAQ;AACN,gBAAQ,KAAK,EAAE,KAAK,SAAS,YAAY,EAAE,cAAc,KAAK,QAAQ,EAAE,UAAU,IAAI,QAAQ,QAAQ,CAAC;AAAA,MACzG;AAAA,IACF;AAEA,UAAM,OAAO,QAAQ,OAAO,OAAK,EAAE,WAAW,KAAK;AACnD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,MAAM,KAAK,QAAQ,UAAU,UAAU,SAAS,GAAG,yCAAyC;AACjI,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,oEAA+D;AACjG,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/visual-diff-verdict.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
+ * transitively @taodigital/database, which instantiates a PrismaClient at module load and
+ * so requires a live DATABASE_URL just to import the module).
+ */
+export interface VisualDiffVerdict {
+    changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';
+    recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';
+    changes: Array<{
+        location: string;
+        severity: string;
+        description: string;
+    }>;
+    bugReportDraft?: {
+        title: string;
+        severity: string;
+        steps: string[];
+        expected: string;
+        actual: string;
+    };
+    aiSummary?: string;
+}
+/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
+export declare function pHashVerdict(pHashDistance: number, changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT', changes: Array<{
+    location: string;
+    severity: string;
+    description: string;
+}>): VisualDiffVerdict;
+/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
+ *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
+ *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
+ *  as NONE/MINOR — only what survives AI review is worth a draft). */
+export declare function mergeAiVisualVerdict(pHashResult: VisualDiffVerdict, aiResult: {
+    changeType: string;
+    changes: any[];
+    recommendation: string;
+    diffScore: number;
+    summary?: string;
+}, screenName: string): VisualDiffVerdict;

package/dist/visual-diff-verdict.js

@@ -0,0 +1,27 @@
+function pHashVerdict(pHashDistance, changeLevel, changes) {
+  const changeType = changeLevel === "IDENTICAL" ? "NONE" : changeLevel === "MINOR" ? "MINOR" : changeLevel === "MODERATE" ? "SIGNIFICANT" : "CRITICAL";
+  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? "") || 5;
+  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? "") || 20;
+  const recommendation = pHashDistance === 0 ? "APPROVE" : pHashDistance < approveThreshold ? "APPROVE" : pHashDistance < reviewThreshold ? "REVIEW" : "FLAG";
+  return { changeType, recommendation, changes };
+}
+function mergeAiVisualVerdict(pHashResult, aiResult, screenName) {
+  if (!aiResult.changeType) return pHashResult;
+  const changeType = aiResult.changeType;
+  const recommendation = aiResult.recommendation;
+  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
+  const aiSummary = aiResult.summary ?? void 0;
+  const bugReportDraft = changeType === "CRITICAL" || changeType === "SIGNIFICANT" ? {
+    title: `Visual regression on "${screenName}"`,
+    severity: changeType === "CRITICAL" ? "High" : "Medium",
+    steps: ["Navigate to the screen", "Compare current view to baseline screenshot"],
+    expected: "Screen appearance should match the stored baseline.",
+    actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`
+  } : void 0;
+  return { changeType, recommendation, changes, bugReportDraft, aiSummary };
+}
+export {
+  mergeAiVisualVerdict,
+  pHashVerdict
+};
+//# sourceMappingURL=visual-diff-verdict.js.map

package/dist/visual-diff-verdict.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/visual-diff-verdict.ts"],
+  "sourcesContent": ["/**\n * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested\n * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and\n * transitively @detiq/database, which instantiates a PrismaClient at module load and\n * so requires a live DATABASE_URL just to import the module).\n */\n\nexport interface VisualDiffVerdict {\n  changeType: 'NONE' | 'MINOR' | 'SIGNIFICANT' | 'CRITICAL';\n  recommendation: 'APPROVE' | 'REVIEW' | 'FLAG' | 'BLOCK';\n  changes: Array<{ location: string; severity: string; description: string }>;\n  bugReportDraft?: { title: string; severity: string; steps: string[]; expected: string; actual: string };\n  aiSummary?: string;\n}\n\n/** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */\nexport function pHashVerdict(\n  pHashDistance: number,\n  changeLevel: 'IDENTICAL' | 'MINOR' | 'MODERATE' | 'SIGNIFICANT',\n  changes: Array<{ location: string; severity: string; description: string }>,\n): VisualDiffVerdict {\n  const changeType: VisualDiffVerdict['changeType'] = changeLevel === 'IDENTICAL' ? 'NONE'\n    : changeLevel === 'MINOR' ? 'MINOR'\n      : changeLevel === 'MODERATE' ? 'SIGNIFICANT'\n        : 'CRITICAL';\n  const approveThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_APPROVE_THRESHOLD ?? '') || 5;\n  const reviewThreshold = parseInt(process.env.CRAWLER_VISUAL_DIFF_REVIEW_THRESHOLD ?? '') || 20;\n  const recommendation: VisualDiffVerdict['recommendation'] = pHashDistance === 0 ? 'APPROVE'\n    : pHashDistance < approveThreshold ? 'APPROVE'\n      : pHashDistance < reviewThreshold ? 'REVIEW'\n        : 'FLAG';\n  return { changeType, recommendation, changes };\n}\n\n/** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,\n *  and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review\n *  (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards\n *  as NONE/MINOR \u2014 only what survives AI review is worth a draft). */\nexport function mergeAiVisualVerdict(\n  pHashResult: VisualDiffVerdict,\n  aiResult: { changeType: string; changes: any[]; recommendation: string; diffScore: number; summary?: string },\n  screenName: string,\n): VisualDiffVerdict {\n  if (!aiResult.changeType) return pHashResult;\n  const changeType = aiResult.changeType as VisualDiffVerdict['changeType'];\n  const recommendation = aiResult.recommendation as VisualDiffVerdict['recommendation'];\n  const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;\n  const aiSummary = aiResult.summary ?? undefined;\n  const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')\n    ? {\n        title: `Visual regression on \"${screenName}\"`,\n        severity: changeType === 'CRITICAL' ? 'High' : 'Medium',\n        steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],\n        expected: 'Screen appearance should match the stored baseline.',\n        actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,\n      }\n    : undefined;\n  return { changeType, recommendation, changes, bugReportDraft, aiSummary };\n}\n"],
+  "mappings": "AAgBO,SAAS,aACd,eACA,aACA,SACmB;AACnB,QAAM,aAA8C,gBAAgB,cAAc,SAC9E,gBAAgB,UAAU,UACxB,gBAAgB,aAAa,gBAC3B;AACR,QAAM,mBAAmB,SAAS,QAAQ,IAAI,yCAAyC,EAAE,KAAK;AAC9F,QAAM,kBAAkB,SAAS,QAAQ,IAAI,wCAAwC,EAAE,KAAK;AAC5F,QAAM,iBAAsD,kBAAkB,IAAI,YAC9E,gBAAgB,mBAAmB,YACjC,gBAAgB,kBAAkB,WAChC;AACR,SAAO,EAAE,YAAY,gBAAgB,QAAQ;AAC/C;AAMO,SAAS,qBACd,aACA,UACA,YACmB;AACnB,MAAI,CAAC,SAAS,WAAY,QAAO;AACjC,QAAM,aAAa,SAAS;AAC5B,QAAM,iBAAiB,SAAS;AAChC,QAAM,UAAU,MAAM,QAAQ,SAAS,OAAO,KAAK,SAAS,QAAQ,SAAS,IAAI,SAAS,UAAU,YAAY;AAChH,QAAM,YAAY,SAAS,WAAW;AACtC,QAAM,iBAAkB,eAAe,cAAc,eAAe,gBAChE;AAAA,IACE,OAAO,yBAAyB,UAAU;AAAA,IAC1C,UAAU,eAAe,aAAa,SAAS;AAAA,IAC/C,OAAO,CAAC,0BAA0B,6CAA6C;AAAA,IAC/E,UAAU;AAAA,IACV,QAAQ,aAAa,mCAAmC,SAAS,UAAU,QAAQ,CAAC,CAAC;AAAA,EACvF,IACA;AACJ,SAAO,EAAE,YAAY,gBAAgB,SAAS,gBAAgB,UAAU;AAC1E;",
+  "names": []
+}

package/dist/webhook-notifier.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
+ *
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
+ * Retries 3x on failure with exponential backoff.
+ */
+export type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';
+export interface WebhookPayload {
+    event: CrawlEventType;
+    jobId: string;
+    projectId: string;
+    tenantId: string;
+    timestamp: string;
+    data: Record<string, unknown>;
+}
+export interface WebhookNotifierOpts {
+    url: string;
+    secret?: string;
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    maxRetries?: number;
+}
+export declare class WebhookNotifier {
+    private readonly opts;
+    constructor(opts: WebhookNotifierOpts);
+    notify(event: CrawlEventType, jobId: string, projectId: string, tenantId: string, data?: Record<string, unknown>): Promise<void>;
+    /** Slack-compatible message format */
+    static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown>;
+}

package/dist/webhook-notifier.js

@@ -0,0 +1,76 @@
+async function hmacSign(secret, payload) {
+  const key = await crypto.subtle.importKey(
+    "raw",
+    new TextEncoder().encode(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"]
+  );
+  const sig = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(payload));
+  return Array.from(new Uint8Array(sig)).map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+class WebhookNotifier {
+  opts;
+  constructor(opts) {
+    this.opts = {
+      url: opts.url,
+      secret: opts.secret,
+      headers: opts.headers ?? {},
+      timeoutMs: opts.timeoutMs ?? 1e4,
+      maxRetries: opts.maxRetries ?? 3
+    };
+  }
+  async notify(event, jobId, projectId, tenantId, data = {}) {
+    const payload = {
+      event,
+      jobId,
+      projectId,
+      tenantId,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      data
+    };
+    const body = JSON.stringify(payload);
+    const headers = {
+      "Content-Type": "application/json",
+      "User-Agent": "ZeTa-Crawler/1.0",
+      ...this.opts.headers
+    };
+    if (this.opts.secret) {
+      headers["X-ZeTa-Signature"] = `sha256=${await hmacSign(this.opts.secret, body)}`;
+    }
+    let lastErr;
+    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
+      try {
+        const res = await fetch(this.opts.url, {
+          method: "POST",
+          headers,
+          body,
+          signal: AbortSignal.timeout(this.opts.timeoutMs)
+        });
+        if (res.ok) return;
+        lastErr = new Error(`Webhook HTTP ${res.status}`);
+      } catch (e) {
+        lastErr = e;
+      }
+      if (attempt < this.opts.maxRetries) {
+        await new Promise((r) => setTimeout(r, 1e3 * Math.pow(2, attempt)));
+      }
+    }
+    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
+  }
+  /** Slack-compatible message format */
+  static slackPayload(event, data) {
+    const emoji = event === "crawl.completed" ? "\u2705" : event === "crawl.failed" ? "\u274C" : "\u{1F504}";
+    return {
+      text: `${emoji} *ZeTa Crawl Event: ${event}*`,
+      blocks: [
+        { type: "section", text: { type: "mrkdwn", text: `${emoji} *${event}*
+${Object.entries(data).map(([k, v]) => `\u2022 *${k}*: ${v}`).join("\n")}` } }
+      ]
+    };
+  }
+}
+export {
+  WebhookNotifier
+};
+//# sourceMappingURL=webhook-notifier.js.map

package/dist/webhook-notifier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/webhook-notifier.ts"],
+  "sourcesContent": ["/**\n * Webhook notifier \u2014 fire HTTP POST callbacks on crawl lifecycle events.\n *\n * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed\n * Payload format compatible with Slack, Discord, and generic REST webhooks.\n * Retries 3x on failure with exponential backoff.\n */\n\nexport type CrawlEventType = 'crawl.started' | 'crawl.progress' | 'crawl.completed' | 'crawl.failed';\n\nexport interface WebhookPayload {\n  event: CrawlEventType;\n  jobId: string;\n  projectId: string;\n  tenantId: string;\n  timestamp: string;\n  data: Record<string, unknown>;\n}\n\nexport interface WebhookNotifierOpts {\n  url: string;\n  secret?: string;       // HMAC-SHA256 signing secret \u2014 adds X-ZeTa-Signature header\n  headers?: Record<string, string>;\n  timeoutMs?: number;    // default 10s\n  maxRetries?: number;   // default 3\n}\n\nasync function hmacSign(secret: string, payload: string): Promise<string> {\n  const key = await crypto.subtle.importKey(\n    'raw',\n    new TextEncoder().encode(secret),\n    { name: 'HMAC', hash: 'SHA-256' },\n    false,\n    ['sign'],\n  );\n  const sig = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(payload));\n  return Array.from(new Uint8Array(sig)).map(b => b.toString(16).padStart(2, '0')).join('');\n}\n\nexport class WebhookNotifier {\n  private readonly opts: Required<Omit<WebhookNotifierOpts, 'secret'>> & { secret?: string };\n\n  constructor(opts: WebhookNotifierOpts) {\n    this.opts = {\n      url: opts.url,\n      secret: opts.secret,\n      headers: opts.headers ?? {},\n      timeoutMs: opts.timeoutMs ?? 10_000,\n      maxRetries: opts.maxRetries ?? 3,\n    };\n  }\n\n  async notify(\n    event: CrawlEventType,\n    jobId: string,\n    projectId: string,\n    tenantId: string,\n    data: Record<string, unknown> = {},\n  ): Promise<void> {\n    const payload: WebhookPayload = {\n      event,\n      jobId,\n      projectId,\n      tenantId,\n      timestamp: new Date().toISOString(),\n      data,\n    };\n    const body = JSON.stringify(payload);\n    const headers: Record<string, string> = {\n      'Content-Type': 'application/json',\n      'User-Agent': 'ZeTa-Crawler/1.0',\n      ...this.opts.headers,\n    };\n    if (this.opts.secret) {\n      headers['X-ZeTa-Signature'] = `sha256=${await hmacSign(this.opts.secret, body)}`;\n    }\n\n    let lastErr: any;\n    for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {\n      try {\n        const res = await fetch(this.opts.url, {\n          method: 'POST',\n          headers,\n          body,\n          signal: AbortSignal.timeout(this.opts.timeoutMs),\n        });\n        if (res.ok) return;\n        lastErr = new Error(`Webhook HTTP ${res.status}`);\n      } catch (e) {\n        lastErr = e;\n      }\n      if (attempt < this.opts.maxRetries) {\n        await new Promise(r => setTimeout(r, 1_000 * Math.pow(2, attempt)));\n      }\n    }\n    console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);\n  }\n\n  /** Slack-compatible message format */\n  static slackPayload(event: CrawlEventType, data: Record<string, unknown>): Record<string, unknown> {\n    const emoji = event === 'crawl.completed' ? '\u2705' : event === 'crawl.failed' ? '\u274C' : '\uD83D\uDD04';\n    return {\n      text: `${emoji} *ZeTa Crawl Event: ${event}*`,\n      blocks: [\n        { type: 'section', text: { type: 'mrkdwn', text: `${emoji} *${event}*\\n${Object.entries(data).map(([k, v]) => `\u2022 *${k}*: ${v}`).join('\\n')}` } },\n      ],\n    };\n  }\n}\n"],
+  "mappings": "AA2BA,eAAe,SAAS,QAAgB,SAAkC;AACxE,QAAM,MAAM,MAAM,OAAO,OAAO;AAAA,IAC9B;AAAA,IACA,IAAI,YAAY,EAAE,OAAO,MAAM;AAAA,IAC/B,EAAE,MAAM,QAAQ,MAAM,UAAU;AAAA,IAChC;AAAA,IACA,CAAC,MAAM;AAAA,EACT;AACA,QAAM,MAAM,MAAM,OAAO,OAAO,KAAK,QAAQ,KAAK,IAAI,YAAY,EAAE,OAAO,OAAO,CAAC;AACnF,SAAO,MAAM,KAAK,IAAI,WAAW,GAAG,CAAC,EAAE,IAAI,OAAK,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,CAAC,EAAE,KAAK,EAAE;AAC1F;AAEO,MAAM,gBAAgB;AAAA,EACV;AAAA,EAEjB,YAAY,MAA2B;AACrC,SAAK,OAAO;AAAA,MACV,KAAK,KAAK;AAAA,MACV,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK,WAAW,CAAC;AAAA,MAC1B,WAAW,KAAK,aAAa;AAAA,MAC7B,YAAY,KAAK,cAAc;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,OACA,WACA,UACA,OAAgC,CAAC,GAClB;AACf,UAAM,UAA0B;AAAA,MAC9B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC;AAAA,IACF;AACA,UAAM,OAAO,KAAK,UAAU,OAAO;AACnC,UAAM,UAAkC;AAAA,MACtC,gBAAgB;AAAA,MAChB,cAAc;AAAA,MACd,GAAG,KAAK,KAAK;AAAA,IACf;AACA,QAAI,KAAK,KAAK,QAAQ;AACpB,cAAQ,kBAAkB,IAAI,UAAU,MAAM,SAAS,KAAK,KAAK,QAAQ,IAAI,CAAC;AAAA,IAChF;AAEA,QAAI;AACJ,aAAS,UAAU,GAAG,WAAW,KAAK,KAAK,YAAY,WAAW;AAChE,UAAI;AACF,cAAM,MAAM,MAAM,MAAM,KAAK,KAAK,KAAK;AAAA,UACrC,QAAQ;AAAA,UACR;AAAA,UACA;AAAA,UACA,QAAQ,YAAY,QAAQ,KAAK,KAAK,SAAS;AAAA,QACjD,CAAC;AACD,YAAI,IAAI,GAAI;AACZ,kBAAU,IAAI,MAAM,gBAAgB,IAAI,MAAM,EAAE;AAAA,MAClD,SAAS,GAAG;AACV,kBAAU;AAAA,MACZ;AACA,UAAI,UAAU,KAAK,KAAK,YAAY;AAClC,cAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,MAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,CAAC;AAAA,MACpE;AAAA,IACF;AACA,YAAQ,MAAM,+BAA+B,KAAK,UAAU,KAAK,KAAK,aAAa,CAAC,cAAc,SAAS,OAAO;AAAA,EACpH;AAAA;AAAA,EAGA,OAAO,aAAa,OAAuB,MAAwD;AACjG,UAAM,QAAQ,UAAU,oBAAoB,WAAM,UAAU,iBAAiB,WAAM;AACnF,WAAO;AAAA,MACL,MAAM,GAAG,KAAK,uBAAuB,KAAK;AAAA,MAC1C,QAAQ;AAAA,QACN,EAAE,MAAM,WAAW,MAAM,EAAE,MAAM,UAAU,MAAM,GAAG,KAAK,KAAK,KAAK;AAAA,EAAM,OAAO,QAAQ,IAAI,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,WAAM,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,CAAC,GAAG,EAAE;AAAA,MACjJ;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/websocket-detector.d.ts

@@ -0,0 +1,24 @@
+/**
+ * WebSocket detection — intercepts WS connections opened during page crawl.
+ * Records connection URLs, message schemas, and message counts.
+ * This lets ZeTa discover real-time API contracts (chat, notifications, presence)
+ * that would be invisible to HTTP-only crawlers.
+ */
+export interface WsConnection {
+    url: string;
+    framesSent: number;
+    framesReceived: number;
+    /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */
+    samples: Array<{
+        direction: 'sent' | 'received';
+        payload: string;
+        ts: string;
+    }>;
+    /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */
+    protocol: string;
+    openedAt: string;
+}
+export declare function attachWebSocketDetector(cdpSession: any): {
+    getConnections(): WsConnection[];
+    detach(): void;
+};

package/dist/websocket-detector.js

@@ -0,0 +1,60 @@
+function inferProtocol(sample) {
+  try {
+    const obj = JSON.parse(sample);
+    if (obj?.jsonrpc) return "json-rpc";
+    if (obj?.type === "connection_init" || obj?.type === "subscribe") return "graphql-ws";
+    if (typeof obj?.sid === "string" || sample.startsWith("0{")) return "socket.io";
+    return "json";
+  } catch {
+    return "raw";
+  }
+}
+function attachWebSocketDetector(cdpSession) {
+  const connections = /* @__PURE__ */ new Map();
+  const onCreated = (evt) => {
+    const { requestId, url, timestamp } = evt ?? {};
+    if (!requestId || !url) return;
+    connections.set(requestId, {
+      url,
+      framesSent: 0,
+      framesReceived: 0,
+      samples: [],
+      protocol: "unknown",
+      openedAt: new Date(timestamp * 1e3).toISOString()
+    });
+  };
+  const onFrame = (evt, direction) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId) return;
+    const conn = connections.get(requestId);
+    if (!conn) return;
+    if (direction === "sent") conn.framesSent++;
+    else conn.framesReceived++;
+    if (conn.samples.length < 10 && response?.payloadData) {
+      const payload = String(response.payloadData).slice(0, 500);
+      conn.samples.push({ direction, payload, ts: (/* @__PURE__ */ new Date()).toISOString() });
+      if (conn.protocol === "unknown") conn.protocol = inferProtocol(payload);
+    }
+  };
+  const onSent = (evt) => onFrame(evt, "sent");
+  const onReceived = (evt) => onFrame(evt, "received");
+  if (cdpSession?.on) {
+    cdpSession.on("Network.webSocketCreated", onCreated);
+    cdpSession.on("Network.webSocketFrameSent", onSent);
+    cdpSession.on("Network.webSocketFrameReceived", onReceived);
+  }
+  return {
+    getConnections: () => [...connections.values()],
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.webSocketCreated", onCreated);
+        cdpSession.off("Network.webSocketFrameSent", onSent);
+        cdpSession.off("Network.webSocketFrameReceived", onReceived);
+      }
+    }
+  };
+}
+export {
+  attachWebSocketDetector
+};
+//# sourceMappingURL=websocket-detector.js.map

package/dist/websocket-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/websocket-detector.ts"],
+  "sourcesContent": ["/**\n * WebSocket detection \u2014 intercepts WS connections opened during page crawl.\n * Records connection URLs, message schemas, and message counts.\n * This lets ZeTa discover real-time API contracts (chat, notifications, presence)\n * that would be invisible to HTTP-only crawlers.\n */\n\nexport interface WsConnection {\n  url: string;\n  framesSent: number;\n  framesReceived: number;\n  /** Sampled message bodies (first 10 per connection, truncated to 500 chars) */\n  samples: Array<{ direction: 'sent' | 'received'; payload: string; ts: string }>;\n  /** Inferred protocol: 'json-rpc' | 'socket.io' | 'graphql-ws' | 'raw' */\n  protocol: string;\n  openedAt: string;\n}\n\nfunction inferProtocol(sample: string): string {\n  try {\n    const obj = JSON.parse(sample);\n    if (obj?.jsonrpc) return 'json-rpc';\n    if (obj?.type === 'connection_init' || obj?.type === 'subscribe') return 'graphql-ws';\n    if (typeof obj?.sid === 'string' || sample.startsWith('0{')) return 'socket.io';\n    return 'json';\n  } catch {\n    return 'raw';\n  }\n}\n\nexport function attachWebSocketDetector(cdpSession: any): {\n  getConnections(): WsConnection[];\n  detach(): void;\n} {\n  const connections = new Map<string, WsConnection>();\n\n  const onCreated = (evt: any) => {\n    const { requestId, url, timestamp } = evt ?? {};\n    if (!requestId || !url) return;\n    connections.set(requestId, {\n      url,\n      framesSent: 0,\n      framesReceived: 0,\n      samples: [],\n      protocol: 'unknown',\n      openedAt: new Date(timestamp * 1000).toISOString(),\n    });\n  };\n\n  const onFrame = (evt: any, direction: 'sent' | 'received') => {\n    const { requestId, response } = evt ?? {};\n    if (!requestId) return;\n    const conn = connections.get(requestId);\n    if (!conn) return;\n    if (direction === 'sent') conn.framesSent++;\n    else conn.framesReceived++;\n    if (conn.samples.length < 10 && response?.payloadData) {\n      const payload = String(response.payloadData).slice(0, 500);\n      conn.samples.push({ direction, payload, ts: new Date().toISOString() });\n      if (conn.protocol === 'unknown') conn.protocol = inferProtocol(payload);\n    }\n  };\n\n  const onSent = (evt: any) => onFrame(evt, 'sent');\n  const onReceived = (evt: any) => onFrame(evt, 'received');\n\n  if (cdpSession?.on) {\n    cdpSession.on('Network.webSocketCreated', onCreated);\n    cdpSession.on('Network.webSocketFrameSent', onSent);\n    cdpSession.on('Network.webSocketFrameReceived', onReceived);\n  }\n\n  return {\n    getConnections: () => [...connections.values()],\n    detach() {\n      if (cdpSession?.off) {\n        cdpSession.off('Network.webSocketCreated', onCreated);\n        cdpSession.off('Network.webSocketFrameSent', onSent);\n        cdpSession.off('Network.webSocketFrameReceived', onReceived);\n      }\n    },\n  };\n}\n"],
+  "mappings": "AAkBA,SAAS,cAAc,QAAwB;AAC7C,MAAI;AACF,UAAM,MAAM,KAAK,MAAM,MAAM;AAC7B,QAAI,KAAK,QAAS,QAAO;AACzB,QAAI,KAAK,SAAS,qBAAqB,KAAK,SAAS,YAAa,QAAO;AACzE,QAAI,OAAO,KAAK,QAAQ,YAAY,OAAO,WAAW,IAAI,EAAG,QAAO;AACpE,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,wBAAwB,YAGtC;AACA,QAAM,cAAc,oBAAI,IAA0B;AAElD,QAAM,YAAY,CAAC,QAAa;AAC9B,UAAM,EAAE,WAAW,KAAK,UAAU,IAAI,OAAO,CAAC;AAC9C,QAAI,CAAC,aAAa,CAAC,IAAK;AACxB,gBAAY,IAAI,WAAW;AAAA,MACzB;AAAA,MACA,YAAY;AAAA,MACZ,gBAAgB;AAAA,MAChB,SAAS,CAAC;AAAA,MACV,UAAU;AAAA,MACV,UAAU,IAAI,KAAK,YAAY,GAAI,EAAE,YAAY;AAAA,IACnD,CAAC;AAAA,EACH;AAEA,QAAM,UAAU,CAAC,KAAU,cAAmC;AAC5D,UAAM,EAAE,WAAW,SAAS,IAAI,OAAO,CAAC;AACxC,QAAI,CAAC,UAAW;AAChB,UAAM,OAAO,YAAY,IAAI,SAAS;AACtC,QAAI,CAAC,KAAM;AACX,QAAI,cAAc,OAAQ,MAAK;AAAA,QAC1B,MAAK;AACV,QAAI,KAAK,QAAQ,SAAS,MAAM,UAAU,aAAa;AACrD,YAAM,UAAU,OAAO,SAAS,WAAW,EAAE,MAAM,GAAG,GAAG;AACzD,WAAK,QAAQ,KAAK,EAAE,WAAW,SAAS,KAAI,oBAAI,KAAK,GAAE,YAAY,EAAE,CAAC;AACtE,UAAI,KAAK,aAAa,UAAW,MAAK,WAAW,cAAc,OAAO;AAAA,IACxE;AAAA,EACF;AAEA,QAAM,SAAS,CAAC,QAAa,QAAQ,KAAK,MAAM;AAChD,QAAM,aAAa,CAAC,QAAa,QAAQ,KAAK,UAAU;AAExD,MAAI,YAAY,IAAI;AAClB,eAAW,GAAG,4BAA4B,SAAS;AACnD,eAAW,GAAG,8BAA8B,MAAM;AAClD,eAAW,GAAG,kCAAkC,UAAU;AAAA,EAC5D;AAEA,SAAO;AAAA,IACL,gBAAgB,MAAM,CAAC,GAAG,YAAY,OAAO,CAAC;AAAA,IAC9C,SAAS;AACP,UAAI,YAAY,KAAK;AACnB,mBAAW,IAAI,4BAA4B,SAAS;AACpD,mBAAW,IAAI,8BAA8B,MAAM;AACnD,mBAAW,IAAI,kCAAkC,UAAU;AAAA,MAC7D;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/worker.js

@@ -0,0 +1,33 @@
+import { startJobWorkerLoop, processDataRetentionPurge } from "./job-queue.js";
+import { logger } from "./logger.js";
+async function main() {
+  logger.info("[worker] starting standalone crawler worker process");
+  if (process.env.JOB_QUEUE_BACKEND === "sqs") {
+    const { startSqsWorker } = await import("./sqs-queue.js");
+    startSqsWorker().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] SQS worker crashed");
+      process.exit(1);
+    });
+  }
+  startJobWorkerLoop();
+  const runPurge = () => {
+    processDataRetentionPurge().catch((err) => {
+      logger.error({ err: String(err) }, "[worker] data-retention purge failed");
+    });
+  };
+  runPurge();
+  setInterval(runPurge, 24 * 60 * 60 * 1e3);
+}
+process.on("SIGTERM", () => {
+  logger.info("[worker] SIGTERM received, exiting");
+  process.exit(0);
+});
+process.on("SIGINT", () => {
+  logger.info("[worker] SIGINT received, exiting");
+  process.exit(0);
+});
+main().catch((err) => {
+  logger.error({ err: String(err) }, "[worker] fatal startup error");
+  process.exit(1);
+});
+//# sourceMappingURL=worker.js.map