@detiq/crawler 1.1.7 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/accessibility-auditor.js +74 -54
  2. package/dist/ai-login.js +249 -227
  3. package/dist/auth-token-rotator.js +67 -57
  4. package/dist/auth.js +336 -299
  5. package/dist/browser-extension.js +142 -104
  6. package/dist/captcha-solver.js +125 -98
  7. package/dist/color-contrast.js +89 -77
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +71 -54
  10. package/dist/cookie-consent.js +83 -68
  11. package/dist/cookie-jar.js +60 -53
  12. package/dist/crawler.js +5345 -4486
  13. package/dist/custom-js-injector.js +61 -45
  14. package/dist/dark-mode-capture.js +81 -67
  15. package/dist/dead-link-detector.js +55 -47
  16. package/dist/element-hash.js +12 -8
  17. package/dist/email-otp.js +195 -154
  18. package/dist/engine-router.js +98 -87
  19. package/dist/export/bulk-export.js +55 -70
  20. package/dist/export/vector-export.js +105 -77
  21. package/dist/extractors/advanced.d.ts +10 -0
  22. package/dist/extractors/advanced.js +266 -249
  23. package/dist/extractors/artifact-envelope.js +58 -59
  24. package/dist/failure-analysis-agent.js +33 -37
  25. package/dist/feature-flag-detector.js +66 -51
  26. package/dist/form-detector.js +75 -55
  27. package/dist/frontier.js +101 -76
  28. package/dist/gap-pass.js +89 -68
  29. package/dist/graphql-detector.js +72 -62
  30. package/dist/http-engine.js +330 -282
  31. package/dist/incremental-diff.js +87 -54
  32. package/dist/index.d.ts +2 -0
  33. package/dist/index.js +28 -63
  34. package/dist/index.js.map +2 -2
  35. package/dist/input-fuzzer.js +97 -93
  36. package/dist/interactive-probe.d.ts +36 -0
  37. package/dist/interactive-probe.js +827 -715
  38. package/dist/job-queue.js +2169 -2151
  39. package/dist/js-coverage.js +56 -52
  40. package/dist/js-router-detector.js +117 -86
  41. package/dist/language-detector.js +63 -51
  42. package/dist/link-classifier.js +50 -32
  43. package/dist/live-view.js +108 -91
  44. package/dist/locator-extractor.d.ts +37 -0
  45. package/dist/locator-extractor.js +199 -0
  46. package/dist/locator-extractor.js.map +7 -0
  47. package/dist/logger.js +7 -9
  48. package/dist/markdown-extract.js +61 -48
  49. package/dist/native-crawler.js +946 -780
  50. package/dist/network-observer.js +156 -135
  51. package/dist/network-throttle.js +40 -29
  52. package/dist/openapi-generator.js +92 -77
  53. package/dist/pdf-extractor.js +90 -72
  54. package/dist/performance-metrics.js +102 -80
  55. package/dist/phash.js +144 -125
  56. package/dist/pii-scrubber.js +68 -65
  57. package/dist/playwright-recorder.js +58 -70
  58. package/dist/policies/crawl-safety.js +34 -31
  59. package/dist/profiles/domain-profile.js +125 -130
  60. package/dist/proxy.js +172 -146
  61. package/dist/rate-limiter.js +65 -58
  62. package/dist/redirect-tracker.js +64 -60
  63. package/dist/redis-frontier.js +136 -116
  64. package/dist/redis-queue.js +144 -157
  65. package/dist/response-cache.js +140 -127
  66. package/dist/responsive-detector.js +46 -42
  67. package/dist/retry-policy.js +50 -46
  68. package/dist/robots-txt.js +83 -69
  69. package/dist/screen-reader.js +70 -45
  70. package/dist/screenshot-masker.js +77 -59
  71. package/dist/seo-checker.js +107 -88
  72. package/dist/session-capture.js +212 -192
  73. package/dist/session-export.js +133 -107
  74. package/dist/sitemap-writer.js +41 -38
  75. package/dist/sitemap.js +202 -141
  76. package/dist/skyvern.js +148 -123
  77. package/dist/sqs-queue.js +42 -42
  78. package/dist/sso-token.js +90 -74
  79. package/dist/stealth/human-behavior.js +173 -156
  80. package/dist/telemetry.js +82 -68
  81. package/dist/test-executor.js +335 -309
  82. package/dist/tls-fingerprint.js +88 -77
  83. package/dist/url-pattern-completer.js +90 -71
  84. package/dist/visual-diff-verdict.js +38 -23
  85. package/dist/webhook-notifier.js +69 -71
  86. package/dist/websocket-detector.js +68 -56
  87. package/dist/worker.js +38 -29
  88. package/package.json +1 -1
  89. package/src/index.ts +2 -0
  90. package/src/locator-extractor.ts +226 -0
  91. package/types/index.d.ts +2 -0
  92. package/types/locator-extractor.d.ts +37 -0
@@ -1,81 +1,92 @@
1
- import { Agent as HttpsAgent } from "node:https";
2
- import { connect as tlsConnect } from "node:tls";
3
- const CHROME_CIPHERS = [
4
- "TLS_AES_128_GCM_SHA256",
5
- "TLS_AES_256_GCM_SHA384",
6
- "TLS_CHACHA20_POLY1305_SHA256",
7
- "ECDHE-ECDSA-AES128-GCM-SHA256",
8
- "ECDHE-RSA-AES128-GCM-SHA256",
9
- "ECDHE-ECDSA-AES256-GCM-SHA384",
10
- "ECDHE-RSA-AES256-GCM-SHA384",
11
- "ECDHE-ECDSA-CHACHA20-POLY1305",
12
- "ECDHE-RSA-CHACHA20-POLY1305",
13
- "ECDHE-RSA-AES128-SHA",
14
- "ECDHE-RSA-AES256-SHA",
15
- "AES128-GCM-SHA256",
16
- "AES256-GCM-SHA384",
17
- "AES128-SHA",
18
- "AES256-SHA"
19
- ].join(":");
20
- const CHROME_CURVES = "X25519:P-256:P-384";
21
- function createChromeTlsAgent(opts = {}) {
22
- return new HttpsAgent({
23
- ciphers: CHROME_CIPHERS,
24
- ecdhCurve: CHROME_CURVES,
25
- minVersion: "TLSv1.2",
26
- maxVersion: "TLSv1.3",
27
- rejectUnauthorized: opts.rejectUnauthorized ?? true,
28
- keepAlive: opts.keepAlive ?? true,
29
- keepAliveMsecs: opts.keepAliveMsecs ?? 1e4,
30
- maxSockets: opts.maxSockets ?? 50
31
- });
1
+ /**
2
+ * TLS fingerprint spoofing — Chrome JA3 cipher suite ordering.
3
+ *
4
+ * Node.js TLS doesn't expose JA3 hash control directly, but ordering
5
+ * cipher suites to match Chrome's TLS ClientHello reduces fingerprint
6
+ * detectability vs the default OpenSSL order.
7
+ *
8
+ * For full JA3 bypass, pair with the browser-stealth engine (Camoufox).
9
+ */
10
+ import { Agent as HttpsAgent } from 'node:https';
11
+ import { connect as tlsConnect } from 'node:tls';
12
+ // Chrome 120+ TLS 1.3 + 1.2 cipher suite order
13
+ export const CHROME_CIPHERS = [
14
+ 'TLS_AES_128_GCM_SHA256',
15
+ 'TLS_AES_256_GCM_SHA384',
16
+ 'TLS_CHACHA20_POLY1305_SHA256',
17
+ 'ECDHE-ECDSA-AES128-GCM-SHA256',
18
+ 'ECDHE-RSA-AES128-GCM-SHA256',
19
+ 'ECDHE-ECDSA-AES256-GCM-SHA384',
20
+ 'ECDHE-RSA-AES256-GCM-SHA384',
21
+ 'ECDHE-ECDSA-CHACHA20-POLY1305',
22
+ 'ECDHE-RSA-CHACHA20-POLY1305',
23
+ 'ECDHE-RSA-AES128-SHA',
24
+ 'ECDHE-RSA-AES256-SHA',
25
+ 'AES128-GCM-SHA256',
26
+ 'AES256-GCM-SHA384',
27
+ 'AES128-SHA',
28
+ 'AES256-SHA',
29
+ ].join(':');
30
+ // Chrome's ECDH curve preference order
31
+ export const CHROME_CURVES = 'X25519:P-256:P-384';
32
+ /**
33
+ * Returns an HTTPS agent configured with Chrome-like TLS settings.
34
+ * Drop-in replacement for the default agent in fetch/got/axios.
35
+ */
36
+ export function createChromeTlsAgent(opts = {}) {
37
+ return new HttpsAgent({
38
+ ciphers: CHROME_CIPHERS,
39
+ ecdhCurve: CHROME_CURVES,
40
+ minVersion: 'TLSv1.2',
41
+ maxVersion: 'TLSv1.3',
42
+ rejectUnauthorized: opts.rejectUnauthorized ?? true,
43
+ keepAlive: opts.keepAlive ?? true,
44
+ keepAliveMsecs: opts.keepAliveMsecs ?? 10_000,
45
+ maxSockets: opts.maxSockets ?? 50,
46
+ });
32
47
  }
33
- async function probeTlsFingerprint(host, port = 443) {
34
- return new Promise((resolve, reject) => {
35
- const socket = tlsConnect({
36
- host,
37
- port,
38
- servername: host,
39
- ciphers: CHROME_CIPHERS,
40
- ecdhCurve: CHROME_CURVES,
41
- minVersion: "TLSv1.2",
42
- rejectUnauthorized: false
43
- }, () => {
44
- const cert = socket.getPeerCertificate();
45
- resolve({
46
- protocol: socket.getProtocol() ?? "unknown",
47
- cipher: socket.getCipher()?.name ?? "unknown",
48
- authorized: socket.authorized,
49
- subjectCN: cert?.subject?.CN?.toString() ?? null
50
- });
51
- socket.destroy();
48
+ /**
49
+ * Probe a host and return its JA3-relevant TLS parameters.
50
+ * Useful for verifying that the fingerprint changed vs default.
51
+ */
52
+ export async function probeTlsFingerprint(host, port = 443) {
53
+ return new Promise((resolve, reject) => {
54
+ const socket = tlsConnect({
55
+ host,
56
+ port,
57
+ servername: host,
58
+ ciphers: CHROME_CIPHERS,
59
+ ecdhCurve: CHROME_CURVES,
60
+ minVersion: 'TLSv1.2',
61
+ rejectUnauthorized: false,
62
+ }, () => {
63
+ const cert = socket.getPeerCertificate();
64
+ resolve({
65
+ protocol: socket.getProtocol() ?? 'unknown',
66
+ cipher: socket.getCipher()?.name ?? 'unknown',
67
+ authorized: socket.authorized,
68
+ subjectCN: cert?.subject?.CN?.toString() ?? null,
69
+ });
70
+ socket.destroy();
71
+ });
72
+ socket.on('error', reject);
73
+ setTimeout(() => { socket.destroy(); reject(new Error('TLS probe timeout')); }, 5_000);
52
74
  });
53
- socket.on("error", reject);
54
- setTimeout(() => {
55
- socket.destroy();
56
- reject(new Error("TLS probe timeout"));
57
- }, 5e3);
58
- });
59
75
  }
60
- const chromeTlsAgent = createChromeTlsAgent();
61
- const CHROME_TLS_HEADERS = {
62
- "Accept-Encoding": "gzip, deflate, br, zstd",
63
- "Accept-Language": "en-US,en;q=0.9",
64
- "sec-ch-ua": '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
65
- "sec-ch-ua-mobile": "?0",
66
- "sec-ch-ua-platform": '"Windows"',
67
- "Sec-Fetch-Dest": "document",
68
- "Sec-Fetch-Mode": "navigate",
69
- "Sec-Fetch-Site": "none",
70
- "Sec-Fetch-User": "?1",
71
- "Upgrade-Insecure-Requests": "1"
72
- };
73
- export {
74
- CHROME_CIPHERS,
75
- CHROME_CURVES,
76
- CHROME_TLS_HEADERS,
77
- chromeTlsAgent,
78
- createChromeTlsAgent,
79
- probeTlsFingerprint
76
+ /**
77
+ * Node.js fetch dispatcher options with Chrome TLS agent.
78
+ * Pass as the `dispatcher` option to undici or as `agent` to node-fetch.
79
+ */
80
+ export const chromeTlsAgent = createChromeTlsAgent();
81
+ export const CHROME_TLS_HEADERS = {
82
+ 'Accept-Encoding': 'gzip, deflate, br, zstd',
83
+ 'Accept-Language': 'en-US,en;q=0.9',
84
+ 'sec-ch-ua': '"Google Chrome";v="120", "Chromium";v="120", "Not-A.Brand";v="99"',
85
+ 'sec-ch-ua-mobile': '?0',
86
+ 'sec-ch-ua-platform': '"Windows"',
87
+ 'Sec-Fetch-Dest': 'document',
88
+ 'Sec-Fetch-Mode': 'navigate',
89
+ 'Sec-Fetch-Site': 'none',
90
+ 'Sec-Fetch-User': '?1',
91
+ 'Upgrade-Insecure-Requests': '1',
80
92
  };
81
- //# sourceMappingURL=tls-fingerprint.js.map
@@ -1,88 +1,107 @@
1
- import { logger as rootLogger } from "./logger.js";
2
- const logger = rootLogger.child({ module: "url-pattern-completer" });
3
- async function runUrlPatternCompletion(discoveredUrls, appUrl, llmCfg) {
4
- if (!llmCfg?.apiKey || discoveredUrls.length < 2) return [];
5
- const origin = (() => {
6
- try {
7
- return new URL(appUrl).origin;
8
- } catch {
9
- return "";
1
+ /**
2
+ * P1: URL Pattern LLM Completion
3
+ *
4
+ * After BFS crawl completes, runs a single LLM call to predict URLs that exist
5
+ * but were never linked from within the app. Validates each candidate with
6
+ * HTTP HEAD before emitting as crawl evidence (URL_CANDIDATES).
7
+ *
8
+ * Cost: ~1 LLM call + N HEAD requests. Takes ~5-30 seconds depending on
9
+ * candidate count. Does NOT launch a browser.
10
+ */
11
+ import { logger as rootLogger } from './logger.js';
12
+ const logger = rootLogger.child({ module: 'url-pattern-completer' });
13
+ /**
14
+ * Returns validated URL candidates (HEAD=200) predicted by LLM from
15
+ * the pattern of already-discovered URLs.
16
+ */
17
+ export async function runUrlPatternCompletion(discoveredUrls, appUrl, llmCfg) {
18
+ if (!llmCfg?.apiKey || discoveredUrls.length < 2)
19
+ return [];
20
+ const origin = (() => { try {
21
+ return new URL(appUrl).origin;
10
22
  }
11
- })();
12
- if (!origin) return [];
13
- const ASSET_EXT = /\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;
14
- const internalUrls = discoveredUrls.filter((u) => {
15
- try {
16
- return new URL(u).origin === origin && !ASSET_EXT.test(u);
17
- } catch {
18
- return false;
23
+ catch {
24
+ return '';
25
+ } })();
26
+ if (!origin)
27
+ return [];
28
+ // Only send internal URLs (same origin, no assets)
29
+ const ASSET_EXT = /\.(png|jpg|jpeg|gif|svg|ico|webp|woff|woff2|ttf|eot|css|js|map|json|xml|txt|pdf)$/i;
30
+ const internalUrls = discoveredUrls
31
+ .filter(u => { try {
32
+ return new URL(u).origin === origin && !ASSET_EXT.test(u);
19
33
  }
20
- }).slice(0, 60);
21
- if (internalUrls.length < 2) return [];
22
- const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
34
+ catch {
35
+ return false;
36
+ } })
37
+ .slice(0, 60); // cap context size
38
+ if (internalUrls.length < 2)
39
+ return [];
40
+ const prompt = `You are analyzing a web application's URL structure to find screens that exist but weren't discovered during automated crawling.
23
41
 
24
42
  Discovered URLs:
25
- ${internalUrls.map((u) => {
26
- try {
27
- return new URL(u).pathname + new URL(u).search;
28
- } catch {
29
- return u;
43
+ ${internalUrls.map(u => { try {
44
+ return new URL(u).pathname + new URL(u).search;
30
45
  }
31
- }).join("\n")}
46
+ catch {
47
+ return u;
48
+ } }).join('\n')}
32
49
 
33
- Based on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found \u2014 typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.
50
+ Based on these URL patterns, suggest up to 15 additional URLs that likely exist in this application but weren't found — typically settings subpages, admin panels, profile variants, paginated views, or feature-gated routes.
34
51
 
35
52
  Rules:
36
53
  - Only suggest URLs on the same domain
37
54
  - Use relative paths (starting with /)
38
55
  - Skip generic guesses (/login, /register, /home) unless the pattern strongly implies them
39
56
  - Focus on patterns YOU SEE in the discovered URLs (e.g. if /settings/profile exists, try /settings/security, /settings/billing)
40
- - Confidence 0.0\u20131.0: how certain you are the page exists
57
+ - Confidence 0.0–1.0: how certain you are the page exists
41
58
 
42
59
  Respond with ONLY valid JSON, no explanation:
43
60
  {"candidates":[{"path":"/settings/billing","confidence":0.85,"reason":"settings subpage pattern from /settings/profile"},{"path":"/admin/users","confidence":0.6,"reason":"admin prefix implied by /admin/dashboard"}]}`;
44
- try {
45
- const Anthropic = (await import("@anthropic-ai/sdk")).default;
46
- const client = new Anthropic({ apiKey: llmCfg.apiKey });
47
- const response = await client.messages.create({
48
- model: llmCfg.model ?? "claude-haiku-4-5-20251001",
49
- max_tokens: 1024,
50
- temperature: 0,
51
- // @ts-ignore: cache_control supported in beta
52
- system: [{ type: "text", text: "You are a URL pattern analyst. Always respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
53
- messages: [{ role: "user", content: prompt }]
54
- });
55
- const text = response.content[0]?.text ?? "";
56
- const jsonMatch = text.match(/\{[\s\S]*\}/);
57
- if (!jsonMatch) return [];
58
- const parsed = JSON.parse(jsonMatch[0]);
59
- const rawCandidates = parsed.candidates ?? [];
60
- if (!Array.isArray(rawCandidates) || rawCandidates.length === 0) return [];
61
- const results = [];
62
- for (const c of rawCandidates.slice(0, 20)) {
63
- if (!c.path || typeof c.path !== "string") continue;
64
- const fullUrl = c.path.startsWith("http") ? c.path : `${origin}${c.path.startsWith("/") ? "" : "/"}${c.path}`;
65
- try {
66
- const controller = new AbortController();
67
- const timeout = setTimeout(() => controller.abort(), 6e3);
68
- const res = await fetch(fullUrl, { method: "HEAD", signal: controller.signal, redirect: "follow" }).catch(() => null);
69
- clearTimeout(timeout);
70
- const status = res?.ok ? "hit" : "miss";
71
- results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? "", status });
72
- logger.info({ url: fullUrl, httpStatus: res?.status ?? "timeout", status }, "[url-completer] candidate checked");
73
- } catch {
74
- results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? "", status: "error" });
75
- }
61
+ try {
62
+ const Anthropic = (await import('@anthropic-ai/sdk')).default;
63
+ const client = new Anthropic({ apiKey: llmCfg.apiKey });
64
+ const response = await client.messages.create({
65
+ model: llmCfg.model ?? 'claude-haiku-4-5-20251001',
66
+ max_tokens: 1024,
67
+ temperature: 0,
68
+ // @ts-ignore: cache_control supported in beta
69
+ system: [{ type: 'text', text: 'You are a URL pattern analyst. Always respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],
70
+ messages: [{ role: 'user', content: prompt }],
71
+ });
72
+ const text = response.content[0]?.text ?? '';
73
+ const jsonMatch = text.match(/\{[\s\S]*\}/);
74
+ if (!jsonMatch)
75
+ return [];
76
+ const parsed = JSON.parse(jsonMatch[0]);
77
+ const rawCandidates = parsed.candidates ?? [];
78
+ if (!Array.isArray(rawCandidates) || rawCandidates.length === 0)
79
+ return [];
80
+ // HEAD-validate each candidate
81
+ const results = [];
82
+ for (const c of rawCandidates.slice(0, 20)) {
83
+ if (!c.path || typeof c.path !== 'string')
84
+ continue;
85
+ const fullUrl = c.path.startsWith('http') ? c.path : `${origin}${c.path.startsWith('/') ? '' : '/'}${c.path}`;
86
+ try {
87
+ const controller = new AbortController();
88
+ const timeout = setTimeout(() => controller.abort(), 6_000);
89
+ const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);
90
+ clearTimeout(timeout);
91
+ const status = res?.ok ? 'hit' : 'miss';
92
+ results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status });
93
+ logger.info({ url: fullUrl, httpStatus: res?.status ?? 'timeout', status }, '[url-completer] candidate checked');
94
+ }
95
+ catch {
96
+ results.push({ url: fullUrl, confidence: c.confidence ?? 0.5, reason: c.reason ?? '', status: 'error' });
97
+ }
98
+ }
99
+ const hits = results.filter(r => r.status === 'hit');
100
+ logger.info({ total: results.length, hits: hits.length }, '[url-completer] pattern completion done');
101
+ return results;
102
+ }
103
+ catch (err) {
104
+ logger.warn({ err: String(err) }, '[url-completer] LLM call failed — skipping pattern completion');
105
+ return [];
76
106
  }
77
- const hits = results.filter((r) => r.status === "hit");
78
- logger.info({ total: results.length, hits: hits.length }, "[url-completer] pattern completion done");
79
- return results;
80
- } catch (err) {
81
- logger.warn({ err: String(err) }, "[url-completer] LLM call failed \u2014 skipping pattern completion");
82
- return [];
83
- }
84
107
  }
85
- export {
86
- runUrlPatternCompletion
87
- };
88
- //# sourceMappingURL=url-pattern-completer.js.map
@@ -1,25 +1,40 @@
1
- function pHashVerdict(pHashDistance, changeLevel, changes) {
2
- const changeType = changeLevel === "IDENTICAL" ? "NONE" : changeLevel === "MINOR" ? "MINOR" : changeLevel === "MODERATE" ? "SIGNIFICANT" : "CRITICAL";
3
- const recommendation = pHashDistance === 0 ? "APPROVE" : pHashDistance < 5 ? "APPROVE" : pHashDistance < 20 ? "REVIEW" : "FLAG";
4
- return { changeType, recommendation, changes };
1
+ /**
2
+ * Pure visual-diff verdict logic, split out of crawler.ts so it can be unit-tested
3
+ * without pulling in crawler.ts's full import graph (Playwright, Stagehand, and
4
+ * transitively @detiq/database, which instantiates a PrismaClient at module load and
5
+ * so requires a live DATABASE_URL just to import the module).
6
+ */
7
+ /** Cheap, always-on first-pass verdict from perceptual hash distance + layout/element diffs. */
8
+ export function pHashVerdict(pHashDistance, changeLevel, changes) {
9
+ const changeType = changeLevel === 'IDENTICAL' ? 'NONE'
10
+ : changeLevel === 'MINOR' ? 'MINOR'
11
+ : changeLevel === 'MODERATE' ? 'SIGNIFICANT'
12
+ : 'CRITICAL';
13
+ const recommendation = pHashDistance === 0 ? 'APPROVE'
14
+ : pHashDistance < 5 ? 'APPROVE'
15
+ : pHashDistance < 20 ? 'REVIEW'
16
+ : 'FLAG';
17
+ return { changeType, recommendation, changes };
5
18
  }
6
- function mergeAiVisualVerdict(pHashResult, aiResult, screenName) {
7
- if (!aiResult.changeType) return pHashResult;
8
- const changeType = aiResult.changeType;
9
- const recommendation = aiResult.recommendation;
10
- const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
11
- const aiSummary = aiResult.summary ?? void 0;
12
- const bugReportDraft = changeType === "CRITICAL" || changeType === "SIGNIFICANT" ? {
13
- title: `Visual regression on "${screenName}"`,
14
- severity: changeType === "CRITICAL" ? "High" : "Medium",
15
- steps: ["Navigate to the screen", "Compare current view to baseline screenshot"],
16
- expected: "Screen appearance should match the stored baseline.",
17
- actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`
18
- } : void 0;
19
- return { changeType, recommendation, changes, bugReportDraft, aiSummary };
19
+ /** Replaces a pHash-only verdict with the AI's semantic judgment once it's available,
20
+ * and drafts a bug report for anything still CRITICAL/SIGNIFICANT after AI review
21
+ * (pHash flags plenty of anti-aliasing/font-rendering noise the AI correctly discards
22
+ * as NONE/MINOR — only what survives AI review is worth a draft). */
23
+ export function mergeAiVisualVerdict(pHashResult, aiResult, screenName) {
24
+ if (!aiResult.changeType)
25
+ return pHashResult;
26
+ const changeType = aiResult.changeType;
27
+ const recommendation = aiResult.recommendation;
28
+ const changes = Array.isArray(aiResult.changes) && aiResult.changes.length > 0 ? aiResult.changes : pHashResult.changes;
29
+ const aiSummary = aiResult.summary ?? undefined;
30
+ const bugReportDraft = (changeType === 'CRITICAL' || changeType === 'SIGNIFICANT')
31
+ ? {
32
+ title: `Visual regression on "${screenName}"`,
33
+ severity: changeType === 'CRITICAL' ? 'High' : 'Medium',
34
+ steps: ['Navigate to the screen', 'Compare current view to baseline screenshot'],
35
+ expected: 'Screen appearance should match the stored baseline.',
36
+ actual: aiSummary || `Screen has changed (diff score: ${aiResult.diffScore.toFixed(1)}%)`,
37
+ }
38
+ : undefined;
39
+ return { changeType, recommendation, changes, bugReportDraft, aiSummary };
20
40
  }
21
- export {
22
- mergeAiVisualVerdict,
23
- pHashVerdict
24
- };
25
- //# sourceMappingURL=visual-diff-verdict.js.map
@@ -1,76 +1,74 @@
1
+ /**
2
+ * Webhook notifier — fire HTTP POST callbacks on crawl lifecycle events.
3
+ *
4
+ * Supports: crawl.started, crawl.progress, crawl.completed, crawl.failed
5
+ * Payload format compatible with Slack, Discord, and generic REST webhooks.
6
+ * Retries 3x on failure with exponential backoff.
7
+ */
1
8
  async function hmacSign(secret, payload) {
2
- const key = await crypto.subtle.importKey(
3
- "raw",
4
- new TextEncoder().encode(secret),
5
- { name: "HMAC", hash: "SHA-256" },
6
- false,
7
- ["sign"]
8
- );
9
- const sig = await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(payload));
10
- return Array.from(new Uint8Array(sig)).map((b) => b.toString(16).padStart(2, "0")).join("");
9
+ const key = await crypto.subtle.importKey('raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
10
+ const sig = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(payload));
11
+ return Array.from(new Uint8Array(sig)).map(b => b.toString(16).padStart(2, '0')).join('');
11
12
  }
12
- class WebhookNotifier {
13
- opts;
14
- constructor(opts) {
15
- this.opts = {
16
- url: opts.url,
17
- secret: opts.secret,
18
- headers: opts.headers ?? {},
19
- timeoutMs: opts.timeoutMs ?? 1e4,
20
- maxRetries: opts.maxRetries ?? 3
21
- };
22
- }
23
- async notify(event, jobId, projectId, tenantId, data = {}) {
24
- const payload = {
25
- event,
26
- jobId,
27
- projectId,
28
- tenantId,
29
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
30
- data
31
- };
32
- const body = JSON.stringify(payload);
33
- const headers = {
34
- "Content-Type": "application/json",
35
- "User-Agent": "ZeTa-Crawler/1.0",
36
- ...this.opts.headers
37
- };
38
- if (this.opts.secret) {
39
- headers["X-ZeTa-Signature"] = `sha256=${await hmacSign(this.opts.secret, body)}`;
13
+ export class WebhookNotifier {
14
+ opts;
15
+ constructor(opts) {
16
+ this.opts = {
17
+ url: opts.url,
18
+ secret: opts.secret,
19
+ headers: opts.headers ?? {},
20
+ timeoutMs: opts.timeoutMs ?? 10_000,
21
+ maxRetries: opts.maxRetries ?? 3,
22
+ };
40
23
  }
41
- let lastErr;
42
- for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
43
- try {
44
- const res = await fetch(this.opts.url, {
45
- method: "POST",
46
- headers,
47
- body,
48
- signal: AbortSignal.timeout(this.opts.timeoutMs)
49
- });
50
- if (res.ok) return;
51
- lastErr = new Error(`Webhook HTTP ${res.status}`);
52
- } catch (e) {
53
- lastErr = e;
54
- }
55
- if (attempt < this.opts.maxRetries) {
56
- await new Promise((r) => setTimeout(r, 1e3 * Math.pow(2, attempt)));
57
- }
24
+ async notify(event, jobId, projectId, tenantId, data = {}) {
25
+ const payload = {
26
+ event,
27
+ jobId,
28
+ projectId,
29
+ tenantId,
30
+ timestamp: new Date().toISOString(),
31
+ data,
32
+ };
33
+ const body = JSON.stringify(payload);
34
+ const headers = {
35
+ 'Content-Type': 'application/json',
36
+ 'User-Agent': 'ZeTa-Crawler/1.0',
37
+ ...this.opts.headers,
38
+ };
39
+ if (this.opts.secret) {
40
+ headers['X-ZeTa-Signature'] = `sha256=${await hmacSign(this.opts.secret, body)}`;
41
+ }
42
+ let lastErr;
43
+ for (let attempt = 0; attempt <= this.opts.maxRetries; attempt++) {
44
+ try {
45
+ const res = await fetch(this.opts.url, {
46
+ method: 'POST',
47
+ headers,
48
+ body,
49
+ signal: AbortSignal.timeout(this.opts.timeoutMs),
50
+ });
51
+ if (res.ok)
52
+ return;
53
+ lastErr = new Error(`Webhook HTTP ${res.status}`);
54
+ }
55
+ catch (e) {
56
+ lastErr = e;
57
+ }
58
+ if (attempt < this.opts.maxRetries) {
59
+ await new Promise(r => setTimeout(r, 1_000 * Math.pow(2, attempt)));
60
+ }
61
+ }
62
+ console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
63
+ }
64
+ /** Slack-compatible message format */
65
+ static slackPayload(event, data) {
66
+ const emoji = event === 'crawl.completed' ? '✅' : event === 'crawl.failed' ? '❌' : '🔄';
67
+ return {
68
+ text: `${emoji} *ZeTa Crawl Event: ${event}*`,
69
+ blocks: [
70
+ { type: 'section', text: { type: 'mrkdwn', text: `${emoji} *${event}*\n${Object.entries(data).map(([k, v]) => `• *${k}*: ${v}`).join('\n')}` } },
71
+ ],
72
+ };
58
73
  }
59
- console.error(`[webhook] Failed to deliver ${event} after ${this.opts.maxRetries + 1} attempts:`, lastErr?.message);
60
- }
61
- /** Slack-compatible message format */
62
- static slackPayload(event, data) {
63
- const emoji = event === "crawl.completed" ? "\u2705" : event === "crawl.failed" ? "\u274C" : "\u{1F504}";
64
- return {
65
- text: `${emoji} *ZeTa Crawl Event: ${event}*`,
66
- blocks: [
67
- { type: "section", text: { type: "mrkdwn", text: `${emoji} *${event}*
68
- ${Object.entries(data).map(([k, v]) => `\u2022 *${k}*: ${v}`).join("\n")}` } }
69
- ]
70
- };
71
- }
72
74
  }
73
- export {
74
- WebhookNotifier
75
- };
76
- //# sourceMappingURL=webhook-notifier.js.map