@detiq/crawler 1.1.7 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/accessibility-auditor.js +74 -54
  2. package/dist/ai-login.js +249 -227
  3. package/dist/auth-token-rotator.js +67 -57
  4. package/dist/auth.js +336 -299
  5. package/dist/browser-extension.js +142 -104
  6. package/dist/captcha-solver.js +125 -98
  7. package/dist/color-contrast.js +89 -77
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +71 -54
  10. package/dist/cookie-consent.js +83 -68
  11. package/dist/cookie-jar.js +60 -53
  12. package/dist/crawler.js +5345 -4486
  13. package/dist/custom-js-injector.js +61 -45
  14. package/dist/dark-mode-capture.js +81 -67
  15. package/dist/dead-link-detector.js +55 -47
  16. package/dist/element-hash.js +12 -8
  17. package/dist/email-otp.js +195 -154
  18. package/dist/engine-router.js +98 -87
  19. package/dist/export/bulk-export.js +55 -70
  20. package/dist/export/vector-export.js +105 -77
  21. package/dist/extractors/advanced.d.ts +10 -0
  22. package/dist/extractors/advanced.js +266 -249
  23. package/dist/extractors/artifact-envelope.js +58 -59
  24. package/dist/failure-analysis-agent.js +33 -37
  25. package/dist/feature-flag-detector.js +66 -51
  26. package/dist/form-detector.js +75 -55
  27. package/dist/frontier.js +101 -76
  28. package/dist/gap-pass.js +89 -68
  29. package/dist/graphql-detector.js +72 -62
  30. package/dist/http-engine.js +330 -282
  31. package/dist/incremental-diff.js +87 -54
  32. package/dist/index.d.ts +2 -0
  33. package/dist/index.js +28 -63
  34. package/dist/index.js.map +2 -2
  35. package/dist/input-fuzzer.js +97 -93
  36. package/dist/interactive-probe.d.ts +36 -0
  37. package/dist/interactive-probe.js +827 -715
  38. package/dist/job-queue.js +2169 -2151
  39. package/dist/js-coverage.js +56 -52
  40. package/dist/js-router-detector.js +117 -86
  41. package/dist/language-detector.js +63 -51
  42. package/dist/link-classifier.js +50 -32
  43. package/dist/live-view.js +108 -91
  44. package/dist/locator-extractor.d.ts +37 -0
  45. package/dist/locator-extractor.js +199 -0
  46. package/dist/locator-extractor.js.map +7 -0
  47. package/dist/logger.js +7 -9
  48. package/dist/markdown-extract.js +61 -48
  49. package/dist/native-crawler.js +946 -780
  50. package/dist/network-observer.js +156 -135
  51. package/dist/network-throttle.js +40 -29
  52. package/dist/openapi-generator.js +92 -77
  53. package/dist/pdf-extractor.js +90 -72
  54. package/dist/performance-metrics.js +102 -80
  55. package/dist/phash.js +144 -125
  56. package/dist/pii-scrubber.js +68 -65
  57. package/dist/playwright-recorder.js +58 -70
  58. package/dist/policies/crawl-safety.js +34 -31
  59. package/dist/profiles/domain-profile.js +125 -130
  60. package/dist/proxy.js +172 -146
  61. package/dist/rate-limiter.js +65 -58
  62. package/dist/redirect-tracker.js +64 -60
  63. package/dist/redis-frontier.js +136 -116
  64. package/dist/redis-queue.js +144 -157
  65. package/dist/response-cache.js +140 -127
  66. package/dist/responsive-detector.js +46 -42
  67. package/dist/retry-policy.js +50 -46
  68. package/dist/robots-txt.js +83 -69
  69. package/dist/screen-reader.js +70 -45
  70. package/dist/screenshot-masker.js +77 -59
  71. package/dist/seo-checker.js +107 -88
  72. package/dist/session-capture.js +212 -192
  73. package/dist/session-export.js +133 -107
  74. package/dist/sitemap-writer.js +41 -38
  75. package/dist/sitemap.js +202 -141
  76. package/dist/skyvern.js +148 -123
  77. package/dist/sqs-queue.js +42 -42
  78. package/dist/sso-token.js +90 -74
  79. package/dist/stealth/human-behavior.js +173 -156
  80. package/dist/telemetry.js +82 -68
  81. package/dist/test-executor.js +335 -309
  82. package/dist/tls-fingerprint.js +88 -77
  83. package/dist/url-pattern-completer.js +90 -71
  84. package/dist/visual-diff-verdict.js +38 -23
  85. package/dist/webhook-notifier.js +69 -71
  86. package/dist/websocket-detector.js +68 -56
  87. package/dist/worker.js +38 -29
  88. package/package.json +1 -1
  89. package/src/index.ts +2 -0
  90. package/src/locator-extractor.ts +226 -0
  91. package/types/index.d.ts +2 -0
  92. package/types/locator-extractor.d.ts +37 -0
@@ -1,81 +1,99 @@
1
+ /**
2
+ * PDF extraction — discovers and extracts text from PDF URLs found during crawl.
3
+ *
4
+ * When the HTTP engine or browser engine discovers a link to a .pdf file,
5
+ * this module fetches the PDF, extracts its text content, and returns it
6
+ * as a screen-compatible markdown string for test generation.
7
+ *
8
+ * Uses pdf-parse (already installed in API package, accessible via dynamic import).
9
+ */
1
10
  const PDF_EXT = /\.pdf(\?.*)?$/i;
2
- const PDF_MIME = "application/pdf";
3
- function isPdfUrl(url, contentType) {
4
- return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);
11
+ const PDF_MIME = 'application/pdf';
12
+ export function isPdfUrl(url, contentType) {
13
+ return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);
5
14
  }
6
- async function extractPdfFromUrl(url, opts = {}) {
7
- const { userAgent = "ZeTa-Crawler/1.0", timeoutMs = 3e4, maxSizeBytes = 50 * 1024 * 1024 } = opts;
8
- let buffer;
9
- let sizeBytes;
10
- try {
11
- const response = await fetch(url, {
12
- headers: { "User-Agent": userAgent, "Accept": "application/pdf,*/*" },
13
- signal: AbortSignal.timeout(timeoutMs)
14
- });
15
- if (!response.ok) return null;
16
- const contentType = response.headers.get("content-type") ?? "";
17
- if (!contentType.includes("pdf") && !PDF_EXT.test(url)) return null;
18
- const arrayBuffer = await response.arrayBuffer();
19
- sizeBytes = arrayBuffer.byteLength;
20
- if (sizeBytes > maxSizeBytes) {
21
- console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
22
- return null;
15
+ export async function extractPdfFromUrl(url, opts = {}) {
16
+ const { userAgent = 'ZeTa-Crawler/1.0', timeoutMs = 30_000, maxSizeBytes = 50 * 1024 * 1024 } = opts;
17
+ let buffer;
18
+ let sizeBytes;
19
+ try {
20
+ const response = await fetch(url, {
21
+ headers: { 'User-Agent': userAgent, 'Accept': 'application/pdf,*/*' },
22
+ signal: AbortSignal.timeout(timeoutMs),
23
+ });
24
+ if (!response.ok)
25
+ return null;
26
+ const contentType = response.headers.get('content-type') ?? '';
27
+ if (!contentType.includes('pdf') && !PDF_EXT.test(url))
28
+ return null;
29
+ const arrayBuffer = await response.arrayBuffer();
30
+ sizeBytes = arrayBuffer.byteLength;
31
+ if (sizeBytes > maxSizeBytes) {
32
+ console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
33
+ return null;
34
+ }
35
+ buffer = Buffer.from(arrayBuffer);
36
+ }
37
+ catch (e) {
38
+ console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
39
+ return null;
40
+ }
41
+ try {
42
+ // pdf-parse is installed in the API package — use dynamic import
43
+ // so this file compiles without adding pdf-parse to crawler's package.json
44
+ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
45
+ // @ts-ignore – no types for pdf-parse/lib/pdf-parse.js
46
+ const pdfParse = await import('pdf-parse/lib/pdf-parse.js').catch(() => import('pdf-parse'));
47
+ const parse = pdfParse.default ?? pdfParse;
48
+ const data = await parse(buffer, { max: 0 }); // max: 0 = all pages
49
+ const title = data.info?.Title || extractTitleFromUrl(url);
50
+ const markdown = pdfTextToMarkdown(data.text ?? '', title);
51
+ return {
52
+ url,
53
+ title,
54
+ markdown,
55
+ pageCount: data.numpages ?? 0,
56
+ info: {
57
+ author: data.info?.Author ?? '',
58
+ subject: data.info?.Subject ?? '',
59
+ keywords: data.info?.Keywords ?? '',
60
+ creator: data.info?.Creator ?? '',
61
+ },
62
+ sizeBytes,
63
+ };
64
+ }
65
+ catch (e) {
66
+ console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
67
+ return null;
23
68
  }
24
- buffer = Buffer.from(arrayBuffer);
25
- } catch (e) {
26
- console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
27
- return null;
28
- }
29
- try {
30
- const pdfParse = await import("pdf-parse/lib/pdf-parse.js").catch(
31
- () => import("pdf-parse")
32
- );
33
- const parse = pdfParse.default ?? pdfParse;
34
- const data = await parse(buffer, { max: 0 });
35
- const title = data.info?.Title || extractTitleFromUrl(url);
36
- const markdown = pdfTextToMarkdown(data.text ?? "", title);
37
- return {
38
- url,
39
- title,
40
- markdown,
41
- pageCount: data.numpages ?? 0,
42
- info: {
43
- author: data.info?.Author ?? "",
44
- subject: data.info?.Subject ?? "",
45
- keywords: data.info?.Keywords ?? "",
46
- creator: data.info?.Creator ?? ""
47
- },
48
- sizeBytes
49
- };
50
- } catch (e) {
51
- console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
52
- return null;
53
- }
54
69
  }
55
70
  function extractTitleFromUrl(url) {
56
- try {
57
- const { pathname } = new URL(url);
58
- const filename = pathname.split("/").pop() ?? "document";
59
- return filename.replace(/\.pdf$/i, "").replace(/[-_]/g, " ").trim();
60
- } catch {
61
- return "PDF Document";
62
- }
71
+ try {
72
+ const { pathname } = new URL(url);
73
+ const filename = pathname.split('/').pop() ?? 'document';
74
+ return filename.replace(/\.pdf$/i, '').replace(/[-_]/g, ' ').trim();
75
+ }
76
+ catch {
77
+ return 'PDF Document';
78
+ }
63
79
  }
64
80
  function pdfTextToMarkdown(text, title) {
65
- if (!text.trim()) return `# ${title}
66
-
67
- *No extractable text content.*`;
68
- const cleaned = text.replace(/\f/g, "\n\n---\n\n").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim();
69
- return `# ${title}
70
-
71
- ${cleaned}`;
81
+ if (!text.trim())
82
+ return `# ${title}\n\n*No extractable text content.*`;
83
+ // Clean up common PDF extraction artifacts
84
+ const cleaned = text
85
+ .replace(/\f/g, '\n\n---\n\n') // form feed = page break
86
+ .replace(/\r\n/g, '\n')
87
+ .replace(/\r/g, '\n')
88
+ .replace(/\n{3,}/g, '\n\n')
89
+ .replace(/[ \t]{2,}/g, ' ')
90
+ .trim();
91
+ return `# ${title}\n\n${cleaned}`;
72
92
  }
73
- function filterPdfUrls(urls) {
74
- return urls.filter((u) => PDF_EXT.test(u));
93
+ /**
94
+ * Filter a list of discovered URLs to find PDF links.
95
+ * Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
96
+ */
97
+ export function filterPdfUrls(urls) {
98
+ return urls.filter(u => PDF_EXT.test(u));
75
99
  }
76
- export {
77
- extractPdfFromUrl,
78
- filterPdfUrls,
79
- isPdfUrl
80
- };
81
- //# sourceMappingURL=pdf-extractor.js.map
@@ -1,83 +1,105 @@
1
- async function collectPerfMetrics(page, url) {
2
- const measuredAt = (/* @__PURE__ */ new Date()).toISOString();
3
- let vitals = {};
4
- try {
5
- vitals = await page.evaluate(() => {
6
- return new Promise((resolve) => {
7
- const result = {};
8
- const nav = performance.getEntriesByType("navigation")[0];
9
- if (nav) {
10
- result.ttfb = Math.round(nav.responseStart - nav.requestStart);
11
- result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
12
- result.loadEventEnd = Math.round(nav.loadEventEnd);
13
- }
14
- let lcpDone = false;
15
- let clsDone = false;
16
- const tryResolve = () => {
17
- if (lcpDone && clsDone) resolve(result);
18
- };
19
- try {
20
- new PerformanceObserver((list) => {
21
- const entries = list.getEntries();
22
- if (entries.length > 0) result.lcp = Math.round(entries[entries.length - 1].startTime);
23
- lcpDone = true;
24
- tryResolve();
25
- }).observe({ type: "largest-contentful-paint", buffered: true });
26
- } catch {
27
- lcpDone = true;
28
- }
29
- try {
30
- let cls = 0;
31
- new PerformanceObserver((list) => {
32
- for (const e of list.getEntries()) {
33
- if (!e.hadRecentInput) cls += e.value;
34
- }
35
- result.cls = Math.round(cls * 1e3) / 1e3;
36
- clsDone = true;
37
- tryResolve();
38
- }).observe({ type: "layout-shift", buffered: true });
39
- } catch {
40
- clsDone = true;
41
- }
42
- try {
43
- const fcp = performance.getEntriesByName("first-contentful-paint")[0];
44
- if (fcp) result.fcp = Math.round(fcp.startTime);
45
- } catch {
1
+ /**
2
+ * Performance metrics collection via CDP + Playwright page.evaluate().
3
+ *
4
+ * Collects Core Web Vitals and resource timing after page navigation.
5
+ * Stored alongside screen data for test generation context and regression detection.
6
+ *
7
+ * Metrics:
8
+ * - LCP (Largest Contentful Paint) — via PerformanceObserver in page
9
+ * - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
10
+ * - TTFB (Time to First Byte) — from navigation timing
11
+ * - DOMContentLoaded — from navigation timing
12
+ * - Resource count and size by type
13
+ */
14
+ export async function collectPerfMetrics(page, url) {
15
+ const measuredAt = new Date().toISOString();
16
+ // Inject PerformanceObserver for LCP + CLS, wait up to 3s
17
+ let vitals = {};
18
+ try {
19
+ vitals = await page.evaluate(() => {
20
+ return new Promise((resolve) => {
21
+ const result = {};
22
+ // Navigation timing
23
+ const nav = performance.getEntriesByType('navigation')[0];
24
+ if (nav) {
25
+ result.ttfb = Math.round(nav.responseStart - nav.requestStart);
26
+ result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
27
+ result.loadEventEnd = Math.round(nav.loadEventEnd);
28
+ }
29
+ let lcpDone = false;
30
+ let clsDone = false;
31
+ const tryResolve = () => { if (lcpDone && clsDone)
32
+ resolve(result); };
33
+ // LCP
34
+ try {
35
+ new PerformanceObserver((list) => {
36
+ const entries = list.getEntries();
37
+ if (entries.length > 0)
38
+ result.lcp = Math.round(entries[entries.length - 1].startTime);
39
+ lcpDone = true;
40
+ tryResolve();
41
+ }).observe({ type: 'largest-contentful-paint', buffered: true });
42
+ }
43
+ catch {
44
+ lcpDone = true;
45
+ }
46
+ // CLS
47
+ try {
48
+ let cls = 0;
49
+ new PerformanceObserver((list) => {
50
+ for (const e of list.getEntries()) {
51
+ if (!e.hadRecentInput)
52
+ cls += e.value;
53
+ }
54
+ result.cls = Math.round(cls * 1000) / 1000;
55
+ clsDone = true;
56
+ tryResolve();
57
+ }).observe({ type: 'layout-shift', buffered: true });
58
+ }
59
+ catch {
60
+ clsDone = true;
61
+ }
62
+ // FCP
63
+ try {
64
+ const fcp = performance.getEntriesByName('first-contentful-paint')[0];
65
+ if (fcp)
66
+ result.fcp = Math.round(fcp.startTime);
67
+ }
68
+ catch { }
69
+ // Fallback resolve after 3s
70
+ setTimeout(() => resolve(result), 3_000);
71
+ });
72
+ }).catch(() => ({}));
73
+ }
74
+ catch { /* page closed or navigate already */ }
75
+ // Resource timing
76
+ let resources = {
77
+ scripts: { count: 0, totalBytes: 0 },
78
+ styles: { count: 0, totalBytes: 0 },
79
+ images: { count: 0, totalBytes: 0 },
80
+ fonts: { count: 0, totalBytes: 0 },
81
+ other: { count: 0, totalBytes: 0 },
82
+ };
83
+ let totalTransferBytes = 0;
84
+ let requestCount = 0;
85
+ try {
86
+ const entries = await page.evaluate(() => performance.getEntriesByType('resource').map((e) => ({
87
+ name: e.name,
88
+ initiatorType: e.initiatorType,
89
+ transferSize: e.transferSize ?? 0,
90
+ }))).catch(() => []);
91
+ for (const e of entries) {
92
+ requestCount++;
93
+ totalTransferBytes += e.transferSize;
94
+ const cat = e.initiatorType === 'script' ? 'scripts'
95
+ : e.initiatorType === 'css' || e.name?.endsWith('.css') ? 'styles'
96
+ : e.initiatorType === 'img' ? 'images'
97
+ : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? 'fonts'
98
+ : 'other';
99
+ resources[cat].count++;
100
+ resources[cat].totalBytes += e.transferSize;
46
101
  }
47
- setTimeout(() => resolve(result), 3e3);
48
- });
49
- }).catch(() => ({}));
50
- } catch {
51
- }
52
- let resources = {
53
- scripts: { count: 0, totalBytes: 0 },
54
- styles: { count: 0, totalBytes: 0 },
55
- images: { count: 0, totalBytes: 0 },
56
- fonts: { count: 0, totalBytes: 0 },
57
- other: { count: 0, totalBytes: 0 }
58
- };
59
- let totalTransferBytes = 0;
60
- let requestCount = 0;
61
- try {
62
- const entries = await page.evaluate(
63
- () => performance.getEntriesByType("resource").map((e) => ({
64
- name: e.name,
65
- initiatorType: e.initiatorType,
66
- transferSize: e.transferSize ?? 0
67
- }))
68
- ).catch(() => []);
69
- for (const e of entries) {
70
- requestCount++;
71
- totalTransferBytes += e.transferSize;
72
- const cat = e.initiatorType === "script" ? "scripts" : e.initiatorType === "css" || e.name?.endsWith(".css") ? "styles" : e.initiatorType === "img" ? "images" : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? "fonts" : "other";
73
- resources[cat].count++;
74
- resources[cat].totalBytes += e.transferSize;
75
102
  }
76
- } catch {
77
- }
78
- return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
103
+ catch { }
104
+ return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
79
105
  }
80
- export {
81
- collectPerfMetrics
82
- };
83
- //# sourceMappingURL=performance-metrics.js.map
package/dist/phash.js CHANGED
@@ -1,136 +1,155 @@
1
- import sharp from "sharp";
2
- async function applyMask(imagePath, ignoredRegions) {
3
- const meta = await sharp(imagePath).metadata();
4
- const W = meta.width ?? 1200;
5
- const H = meta.height ?? 800;
6
- const overlays = await Promise.all(
7
- ignoredRegions.map(async (r) => {
8
- const rW = Math.max(1, Math.round(r.width * W));
9
- const rH = Math.max(1, Math.round(r.height * H));
10
- const buf = await sharp({
11
- create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } }
12
- }).png().toBuffer();
13
- return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
14
- })
15
- );
16
- return sharp(imagePath).composite(overlays).png().toBuffer();
1
+ import sharp from 'sharp';
2
+ /**
3
+ * Composites neutral grey (128,128,128) over each ignoredRegion (fractional
4
+ * 0-1 coordinates) so masked content — timestamps, live counters, ads —
5
+ * contributes nothing to any downstream pixel comparison. Shared by every
6
+ * comparison path (whole-image hash, per-cell grid hash, AI vision) so a
7
+ * region masked once is actually ignored everywhere, not just in the
8
+ * cheapest check.
9
+ */
10
+ export async function applyMask(imagePath, ignoredRegions) {
11
+ const meta = await sharp(imagePath).metadata();
12
+ const W = meta.width ?? 1200;
13
+ const H = meta.height ?? 800;
14
+ const overlays = await Promise.all(ignoredRegions.map(async (r) => {
15
+ const rW = Math.max(1, Math.round(r.width * W));
16
+ const rH = Math.max(1, Math.round(r.height * H));
17
+ const buf = await sharp({
18
+ create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },
19
+ }).png().toBuffer();
20
+ return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
21
+ }));
22
+ return sharp(imagePath).composite(overlays).png().toBuffer();
17
23
  }
24
+ /**
25
+ * Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.
26
+ * ignoredRegions are fractional (0-1 range), converted to pixels here.
27
+ * Masked regions are filled with neutral grey (128,128,128) before hashing.
28
+ */
18
29
  async function hashWithMasks(imagePath, ignoredRegions = []) {
19
- const pipeline = ignoredRegions.length > 0 ? sharp(await applyMask(imagePath, ignoredRegions)) : sharp(imagePath);
20
- const buf = await pipeline.resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer();
21
- const pixels = Array.from(buf);
22
- const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
23
- return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
30
+ const pipeline = ignoredRegions.length > 0
31
+ ? sharp(await applyMask(imagePath, ignoredRegions))
32
+ : sharp(imagePath);
33
+ const buf = await pipeline.resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer();
34
+ const pixels = Array.from(buf);
35
+ const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
36
+ return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);
24
37
  }
25
38
  function hammingDistance(a, b) {
26
- let xor = a ^ b;
27
- let count = 0;
28
- while (xor !== 0n) {
29
- count += Number(xor & 1n);
30
- xor >>= 1n;
31
- }
32
- return count;
33
- }
34
- async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
35
- const metaA = await sharp(pathA).metadata();
36
- const W = metaA.width ?? 1200;
37
- const H = metaA.height ?? 800;
38
- const toRawRGBA = async (path) => {
39
- const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
40
- const { data } = await base.resize(W, H, { fit: "fill" }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
41
- return data;
42
- };
43
- const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
44
- const { default: pixelmatch } = await import("pixelmatch");
45
- const diffData = Buffer.alloc(W * H * 4);
46
- pixelmatch(rawA, rawB, diffData, W, H, {
47
- threshold: 0.1,
48
- alpha: 0.3,
49
- diffColor: [255, 0, 0],
50
- aaColor: [255, 255, 0]
51
- });
52
- return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
39
+ let xor = a ^ b;
40
+ let count = 0;
41
+ while (xor !== 0n) {
42
+ count += Number(xor & 1n);
43
+ xor >>= 1n;
44
+ }
45
+ return count;
53
46
  }
54
- async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
55
- const [hashA, hashB] = await Promise.all([
56
- hashWithMasks(pathA, ignoredRegions),
57
- hashWithMasks(pathB, ignoredRegions)
58
- ]);
59
- let xor = hashA ^ hashB;
60
- let dist = 0;
61
- while (xor !== 0n) {
62
- dist += Number(xor & 1n);
63
- xor >>= 1n;
64
- }
65
- const pct = parseFloat((dist / 64 * 100).toFixed(1));
66
- const changeLevel = pct === 0 ? "IDENTICAL" : pct < 10 ? "MINOR" : pct < 30 ? "MODERATE" : "SIGNIFICANT";
67
- return { pHashDistance: pct, changeLevel };
47
+ /**
48
+ * Generates a pixelmatch diff image highlighting changed pixels in red.
49
+ * Applies the same ignoredRegions mask as screenshotDiffPercent so masked
50
+ * zones don't show as "changed" in the visual output.
51
+ * Returns a PNG Buffer suitable for upload to S3.
52
+ */
53
+ export async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
54
+ const metaA = await sharp(pathA).metadata();
55
+ const W = metaA.width ?? 1200;
56
+ const H = metaA.height ?? 800;
57
+ const toRawRGBA = async (path) => {
58
+ const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
59
+ const { data } = await base.resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
60
+ return data;
61
+ };
62
+ const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
63
+ const { default: pixelmatch } = await import('pixelmatch');
64
+ const diffData = Buffer.alloc(W * H * 4);
65
+ pixelmatch(rawA, rawB, diffData, W, H, {
66
+ threshold: 0.1,
67
+ alpha: 0.3,
68
+ diffColor: [255, 0, 0],
69
+ aaColor: [255, 255, 0],
70
+ });
71
+ return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
68
72
  }
69
- async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
70
- const GRID_LABELS = [
71
- "top-left",
72
- "top-center",
73
- "top-right",
74
- "mid-left",
75
- "mid-center",
76
- "mid-right",
77
- "bottom-left",
78
- "bottom-center",
79
- "bottom-right"
80
- ];
81
- try {
82
- const [sourceA, sourceB] = ignoredRegions.length > 0 ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)]) : [pathA, pathB];
83
- const [metaA, metaB] = await Promise.all([
84
- sharp(sourceA).metadata(),
85
- sharp(sourceB).metadata()
73
+ export async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
74
+ const [hashA, hashB] = await Promise.all([
75
+ hashWithMasks(pathA, ignoredRegions),
76
+ hashWithMasks(pathB, ignoredRegions),
86
77
  ]);
87
- const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
88
- const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
89
- const cW = Math.floor(W / 3);
90
- const cH = Math.floor(H / 3);
91
- const changes = [];
92
- for (let row = 0; row < 3; row++) {
93
- for (let col = 0; col < 3; col++) {
94
- const region = { left: col * cW, top: row * cH, width: cW, height: cH };
95
- const label = GRID_LABELS[row * 3 + col];
96
- const [hashA, hashB] = await Promise.all([
97
- sharp(sourceA).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
98
- const pixels = Array.from(buf);
99
- const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
100
- return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
101
- }),
102
- sharp(sourceB).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
103
- const pixels = Array.from(buf);
104
- const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
105
- return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
106
- })
78
+ let xor = hashA ^ hashB;
79
+ let dist = 0;
80
+ while (xor !== 0n) {
81
+ dist += Number(xor & 1n);
82
+ xor >>= 1n;
83
+ }
84
+ const pct = parseFloat(((dist / 64) * 100).toFixed(1));
85
+ const changeLevel = pct === 0 ? 'IDENTICAL' : pct < 10 ? 'MINOR' : pct < 30 ? 'MODERATE' : 'SIGNIFICANT';
86
+ return { pHashDistance: pct, changeLevel };
87
+ }
88
+ /**
89
+ * Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
90
+ * pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
91
+ * Uses sharp's `extract` to crop each cell before hashing.
92
+ */
93
+ export async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
94
+ const GRID_LABELS = [
95
+ 'top-left', 'top-center', 'top-right',
96
+ 'mid-left', 'mid-center', 'mid-right',
97
+ 'bottom-left', 'bottom-center', 'bottom-right',
98
+ ];
99
+ try {
100
+ // Mask before cropping into cells — a masked region (e.g. a timestamp)
101
+ // then contributes nothing to any cell's hash, instead of only being
102
+ // ignored by the separate whole-image hash in screenshotDiffPercent().
103
+ const [sourceA, sourceB] = ignoredRegions.length > 0
104
+ ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)])
105
+ : [pathA, pathB];
106
+ const [metaA, metaB] = await Promise.all([
107
+ sharp(sourceA).metadata(),
108
+ sharp(sourceB).metadata(),
107
109
  ]);
108
- let xor = hashA ^ hashB;
109
- let dist = 0;
110
- while (xor !== 0n) {
111
- dist += Number(xor & 1n);
112
- xor >>= 1n;
113
- }
114
- const pct = dist / 64 * 100;
115
- if (pct >= 5) {
116
- const severity = pct >= 20 ? "HIGH" : pct >= 10 ? "MEDIUM" : "LOW";
117
- changes.push({
118
- location: label,
119
- severity,
120
- description: `${Math.round(pct)}% visual change in ${label.replace("-", " ")} region`
121
- });
110
+ const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
111
+ const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
112
+ const cW = Math.floor(W / 3);
113
+ const cH = Math.floor(H / 3);
114
+ const changes = [];
115
+ for (let row = 0; row < 3; row++) {
116
+ for (let col = 0; col < 3; col++) {
117
+ const region = { left: col * cW, top: row * cH, width: cW, height: cH };
118
+ const label = GRID_LABELS[row * 3 + col];
119
+ const [hashA, hashB] = await Promise.all([
120
+ sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
121
+ .then((buf) => {
122
+ const pixels = Array.from(buf);
123
+ const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
124
+ return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
125
+ }),
126
+ sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
127
+ .then((buf) => {
128
+ const pixels = Array.from(buf);
129
+ const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
130
+ return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
131
+ }),
132
+ ]);
133
+ let xor = hashA ^ hashB;
134
+ let dist = 0;
135
+ while (xor !== 0n) {
136
+ dist += Number(xor & 1n);
137
+ xor >>= 1n;
138
+ }
139
+ const pct = (dist / 64) * 100;
140
+ if (pct >= 5) {
141
+ const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';
142
+ changes.push({
143
+ location: label,
144
+ severity,
145
+ description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,
146
+ });
147
+ }
148
+ }
122
149
  }
123
- }
150
+ return changes;
151
+ }
152
+ catch {
153
+ return [];
124
154
  }
125
- return changes;
126
- } catch {
127
- return [];
128
- }
129
155
  }
130
- export {
131
- applyMask,
132
- generatePixelmatchDiff,
133
- gridLayoutDiff,
134
- screenshotDiffPercent
135
- };
136
- //# sourceMappingURL=phash.js.map