@detiq/crawler 1.1.7 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accessibility-auditor.js +74 -54
- package/dist/ai-login.js +249 -227
- package/dist/auth-token-rotator.js +67 -57
- package/dist/auth.js +336 -299
- package/dist/browser-extension.js +142 -104
- package/dist/captcha-solver.js +125 -98
- package/dist/color-contrast.js +89 -77
- package/dist/content-chunker.js +126 -126
- package/dist/content-dedup.js +71 -54
- package/dist/cookie-consent.js +83 -68
- package/dist/cookie-jar.js +60 -53
- package/dist/crawler.js +5345 -4486
- package/dist/custom-js-injector.js +61 -45
- package/dist/dark-mode-capture.js +81 -67
- package/dist/dead-link-detector.js +55 -47
- package/dist/element-hash.js +12 -8
- package/dist/email-otp.js +195 -154
- package/dist/engine-router.js +98 -87
- package/dist/export/bulk-export.js +55 -70
- package/dist/export/vector-export.js +105 -77
- package/dist/extractors/advanced.d.ts +10 -0
- package/dist/extractors/advanced.js +266 -249
- package/dist/extractors/artifact-envelope.js +58 -59
- package/dist/failure-analysis-agent.js +33 -37
- package/dist/feature-flag-detector.js +66 -51
- package/dist/form-detector.js +75 -55
- package/dist/frontier.js +101 -76
- package/dist/gap-pass.js +89 -68
- package/dist/graphql-detector.js +72 -62
- package/dist/http-engine.js +330 -282
- package/dist/incremental-diff.js +87 -54
- package/dist/index.d.ts +2 -0
- package/dist/index.js +28 -63
- package/dist/index.js.map +2 -2
- package/dist/input-fuzzer.js +97 -93
- package/dist/interactive-probe.d.ts +36 -0
- package/dist/interactive-probe.js +827 -715
- package/dist/job-queue.js +2169 -2151
- package/dist/js-coverage.js +56 -52
- package/dist/js-router-detector.js +117 -86
- package/dist/language-detector.js +63 -51
- package/dist/link-classifier.js +50 -32
- package/dist/live-view.js +108 -91
- package/dist/locator-extractor.d.ts +37 -0
- package/dist/locator-extractor.js +199 -0
- package/dist/locator-extractor.js.map +7 -0
- package/dist/logger.js +7 -9
- package/dist/markdown-extract.js +61 -48
- package/dist/native-crawler.js +946 -780
- package/dist/network-observer.js +156 -135
- package/dist/network-throttle.js +40 -29
- package/dist/openapi-generator.js +92 -77
- package/dist/pdf-extractor.js +90 -72
- package/dist/performance-metrics.js +102 -80
- package/dist/phash.js +144 -125
- package/dist/pii-scrubber.js +68 -65
- package/dist/playwright-recorder.js +58 -70
- package/dist/policies/crawl-safety.js +34 -31
- package/dist/profiles/domain-profile.js +125 -130
- package/dist/proxy.js +172 -146
- package/dist/rate-limiter.js +65 -58
- package/dist/redirect-tracker.js +64 -60
- package/dist/redis-frontier.js +136 -116
- package/dist/redis-queue.js +144 -157
- package/dist/response-cache.js +140 -127
- package/dist/responsive-detector.js +46 -42
- package/dist/retry-policy.js +50 -46
- package/dist/robots-txt.js +83 -69
- package/dist/screen-reader.js +70 -45
- package/dist/screenshot-masker.js +77 -59
- package/dist/seo-checker.js +107 -88
- package/dist/session-capture.js +212 -192
- package/dist/session-export.js +133 -107
- package/dist/sitemap-writer.js +41 -38
- package/dist/sitemap.js +202 -141
- package/dist/skyvern.js +148 -123
- package/dist/sqs-queue.js +42 -42
- package/dist/sso-token.js +90 -74
- package/dist/stealth/human-behavior.js +173 -156
- package/dist/telemetry.js +82 -68
- package/dist/test-executor.js +335 -309
- package/dist/tls-fingerprint.js +88 -77
- package/dist/url-pattern-completer.js +90 -71
- package/dist/visual-diff-verdict.js +38 -23
- package/dist/webhook-notifier.js +69 -71
- package/dist/websocket-detector.js +68 -56
- package/dist/worker.js +38 -29
- package/package.json +1 -1
- package/src/index.ts +2 -0
- package/src/locator-extractor.ts +226 -0
- package/types/index.d.ts +2 -0
- package/types/locator-extractor.d.ts +37 -0
package/dist/pdf-extractor.js
CHANGED
|
@@ -1,81 +1,99 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PDF extraction — discovers and extracts text from PDF URLs found during crawl.
|
|
3
|
+
*
|
|
4
|
+
* When the HTTP engine or browser engine discovers a link to a .pdf file,
|
|
5
|
+
* this module fetches the PDF, extracts its text content, and returns it
|
|
6
|
+
* as a screen-compatible markdown string for test generation.
|
|
7
|
+
*
|
|
8
|
+
* Uses pdf-parse (already installed in API package, accessible via dynamic import).
|
|
9
|
+
*/
|
|
1
10
|
const PDF_EXT = /\.pdf(\?.*)?$/i;
|
|
2
|
-
const PDF_MIME =
|
|
3
|
-
function isPdfUrl(url, contentType) {
|
|
4
|
-
|
|
11
|
+
const PDF_MIME = 'application/pdf';
|
|
12
|
+
export function isPdfUrl(url, contentType) {
|
|
13
|
+
return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);
|
|
5
14
|
}
|
|
6
|
-
async function extractPdfFromUrl(url, opts = {}) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
15
|
+
export async function extractPdfFromUrl(url, opts = {}) {
|
|
16
|
+
const { userAgent = 'ZeTa-Crawler/1.0', timeoutMs = 30_000, maxSizeBytes = 50 * 1024 * 1024 } = opts;
|
|
17
|
+
let buffer;
|
|
18
|
+
let sizeBytes;
|
|
19
|
+
try {
|
|
20
|
+
const response = await fetch(url, {
|
|
21
|
+
headers: { 'User-Agent': userAgent, 'Accept': 'application/pdf,*/*' },
|
|
22
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
23
|
+
});
|
|
24
|
+
if (!response.ok)
|
|
25
|
+
return null;
|
|
26
|
+
const contentType = response.headers.get('content-type') ?? '';
|
|
27
|
+
if (!contentType.includes('pdf') && !PDF_EXT.test(url))
|
|
28
|
+
return null;
|
|
29
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
30
|
+
sizeBytes = arrayBuffer.byteLength;
|
|
31
|
+
if (sizeBytes > maxSizeBytes) {
|
|
32
|
+
console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
buffer = Buffer.from(arrayBuffer);
|
|
36
|
+
}
|
|
37
|
+
catch (e) {
|
|
38
|
+
console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
try {
|
|
42
|
+
// pdf-parse is installed in the API package — use dynamic import
|
|
43
|
+
// so this file compiles without adding pdf-parse to crawler's package.json
|
|
44
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
45
|
+
// @ts-ignore – no types for pdf-parse/lib/pdf-parse.js
|
|
46
|
+
const pdfParse = await import('pdf-parse/lib/pdf-parse.js').catch(() => import('pdf-parse'));
|
|
47
|
+
const parse = pdfParse.default ?? pdfParse;
|
|
48
|
+
const data = await parse(buffer, { max: 0 }); // max: 0 = all pages
|
|
49
|
+
const title = data.info?.Title || extractTitleFromUrl(url);
|
|
50
|
+
const markdown = pdfTextToMarkdown(data.text ?? '', title);
|
|
51
|
+
return {
|
|
52
|
+
url,
|
|
53
|
+
title,
|
|
54
|
+
markdown,
|
|
55
|
+
pageCount: data.numpages ?? 0,
|
|
56
|
+
info: {
|
|
57
|
+
author: data.info?.Author ?? '',
|
|
58
|
+
subject: data.info?.Subject ?? '',
|
|
59
|
+
keywords: data.info?.Keywords ?? '',
|
|
60
|
+
creator: data.info?.Creator ?? '',
|
|
61
|
+
},
|
|
62
|
+
sizeBytes,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
catch (e) {
|
|
66
|
+
console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
|
|
67
|
+
return null;
|
|
23
68
|
}
|
|
24
|
-
buffer = Buffer.from(arrayBuffer);
|
|
25
|
-
} catch (e) {
|
|
26
|
-
console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
|
|
27
|
-
return null;
|
|
28
|
-
}
|
|
29
|
-
try {
|
|
30
|
-
const pdfParse = await import("pdf-parse/lib/pdf-parse.js").catch(
|
|
31
|
-
() => import("pdf-parse")
|
|
32
|
-
);
|
|
33
|
-
const parse = pdfParse.default ?? pdfParse;
|
|
34
|
-
const data = await parse(buffer, { max: 0 });
|
|
35
|
-
const title = data.info?.Title || extractTitleFromUrl(url);
|
|
36
|
-
const markdown = pdfTextToMarkdown(data.text ?? "", title);
|
|
37
|
-
return {
|
|
38
|
-
url,
|
|
39
|
-
title,
|
|
40
|
-
markdown,
|
|
41
|
-
pageCount: data.numpages ?? 0,
|
|
42
|
-
info: {
|
|
43
|
-
author: data.info?.Author ?? "",
|
|
44
|
-
subject: data.info?.Subject ?? "",
|
|
45
|
-
keywords: data.info?.Keywords ?? "",
|
|
46
|
-
creator: data.info?.Creator ?? ""
|
|
47
|
-
},
|
|
48
|
-
sizeBytes
|
|
49
|
-
};
|
|
50
|
-
} catch (e) {
|
|
51
|
-
console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
|
|
52
|
-
return null;
|
|
53
|
-
}
|
|
54
69
|
}
|
|
55
70
|
function extractTitleFromUrl(url) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
71
|
+
try {
|
|
72
|
+
const { pathname } = new URL(url);
|
|
73
|
+
const filename = pathname.split('/').pop() ?? 'document';
|
|
74
|
+
return filename.replace(/\.pdf$/i, '').replace(/[-_]/g, ' ').trim();
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return 'PDF Document';
|
|
78
|
+
}
|
|
63
79
|
}
|
|
64
80
|
function pdfTextToMarkdown(text, title) {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
81
|
+
if (!text.trim())
|
|
82
|
+
return `# ${title}\n\n*No extractable text content.*`;
|
|
83
|
+
// Clean up common PDF extraction artifacts
|
|
84
|
+
const cleaned = text
|
|
85
|
+
.replace(/\f/g, '\n\n---\n\n') // form feed = page break
|
|
86
|
+
.replace(/\r\n/g, '\n')
|
|
87
|
+
.replace(/\r/g, '\n')
|
|
88
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
89
|
+
.replace(/[ \t]{2,}/g, ' ')
|
|
90
|
+
.trim();
|
|
91
|
+
return `# ${title}\n\n${cleaned}`;
|
|
72
92
|
}
|
|
73
|
-
|
|
74
|
-
|
|
93
|
+
/**
|
|
94
|
+
* Filter a list of discovered URLs to find PDF links.
|
|
95
|
+
* Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
|
|
96
|
+
*/
|
|
97
|
+
export function filterPdfUrls(urls) {
|
|
98
|
+
return urls.filter(u => PDF_EXT.test(u));
|
|
75
99
|
}
|
|
76
|
-
export {
|
|
77
|
-
extractPdfFromUrl,
|
|
78
|
-
filterPdfUrls,
|
|
79
|
-
isPdfUrl
|
|
80
|
-
};
|
|
81
|
-
//# sourceMappingURL=pdf-extractor.js.map
|
|
@@ -1,83 +1,105 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Performance metrics collection via CDP + Playwright page.evaluate().
|
|
3
|
+
*
|
|
4
|
+
* Collects Core Web Vitals and resource timing after page navigation.
|
|
5
|
+
* Stored alongside screen data for test generation context and regression detection.
|
|
6
|
+
*
|
|
7
|
+
* Metrics:
|
|
8
|
+
* - LCP (Largest Contentful Paint) — via PerformanceObserver in page
|
|
9
|
+
* - CLS (Cumulative Layout Shift) — via PerformanceObserver in page
|
|
10
|
+
* - TTFB (Time to First Byte) — from navigation timing
|
|
11
|
+
* - DOMContentLoaded — from navigation timing
|
|
12
|
+
* - Resource count and size by type
|
|
13
|
+
*/
|
|
14
|
+
export async function collectPerfMetrics(page, url) {
|
|
15
|
+
const measuredAt = new Date().toISOString();
|
|
16
|
+
// Inject PerformanceObserver for LCP + CLS, wait up to 3s
|
|
17
|
+
let vitals = {};
|
|
18
|
+
try {
|
|
19
|
+
vitals = await page.evaluate(() => {
|
|
20
|
+
return new Promise((resolve) => {
|
|
21
|
+
const result = {};
|
|
22
|
+
// Navigation timing
|
|
23
|
+
const nav = performance.getEntriesByType('navigation')[0];
|
|
24
|
+
if (nav) {
|
|
25
|
+
result.ttfb = Math.round(nav.responseStart - nav.requestStart);
|
|
26
|
+
result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
|
|
27
|
+
result.loadEventEnd = Math.round(nav.loadEventEnd);
|
|
28
|
+
}
|
|
29
|
+
let lcpDone = false;
|
|
30
|
+
let clsDone = false;
|
|
31
|
+
const tryResolve = () => { if (lcpDone && clsDone)
|
|
32
|
+
resolve(result); };
|
|
33
|
+
// LCP
|
|
34
|
+
try {
|
|
35
|
+
new PerformanceObserver((list) => {
|
|
36
|
+
const entries = list.getEntries();
|
|
37
|
+
if (entries.length > 0)
|
|
38
|
+
result.lcp = Math.round(entries[entries.length - 1].startTime);
|
|
39
|
+
lcpDone = true;
|
|
40
|
+
tryResolve();
|
|
41
|
+
}).observe({ type: 'largest-contentful-paint', buffered: true });
|
|
42
|
+
}
|
|
43
|
+
catch {
|
|
44
|
+
lcpDone = true;
|
|
45
|
+
}
|
|
46
|
+
// CLS
|
|
47
|
+
try {
|
|
48
|
+
let cls = 0;
|
|
49
|
+
new PerformanceObserver((list) => {
|
|
50
|
+
for (const e of list.getEntries()) {
|
|
51
|
+
if (!e.hadRecentInput)
|
|
52
|
+
cls += e.value;
|
|
53
|
+
}
|
|
54
|
+
result.cls = Math.round(cls * 1000) / 1000;
|
|
55
|
+
clsDone = true;
|
|
56
|
+
tryResolve();
|
|
57
|
+
}).observe({ type: 'layout-shift', buffered: true });
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
clsDone = true;
|
|
61
|
+
}
|
|
62
|
+
// FCP
|
|
63
|
+
try {
|
|
64
|
+
const fcp = performance.getEntriesByName('first-contentful-paint')[0];
|
|
65
|
+
if (fcp)
|
|
66
|
+
result.fcp = Math.round(fcp.startTime);
|
|
67
|
+
}
|
|
68
|
+
catch { }
|
|
69
|
+
// Fallback resolve after 3s
|
|
70
|
+
setTimeout(() => resolve(result), 3_000);
|
|
71
|
+
});
|
|
72
|
+
}).catch(() => ({}));
|
|
73
|
+
}
|
|
74
|
+
catch { /* page closed or navigate already */ }
|
|
75
|
+
// Resource timing
|
|
76
|
+
let resources = {
|
|
77
|
+
scripts: { count: 0, totalBytes: 0 },
|
|
78
|
+
styles: { count: 0, totalBytes: 0 },
|
|
79
|
+
images: { count: 0, totalBytes: 0 },
|
|
80
|
+
fonts: { count: 0, totalBytes: 0 },
|
|
81
|
+
other: { count: 0, totalBytes: 0 },
|
|
82
|
+
};
|
|
83
|
+
let totalTransferBytes = 0;
|
|
84
|
+
let requestCount = 0;
|
|
85
|
+
try {
|
|
86
|
+
const entries = await page.evaluate(() => performance.getEntriesByType('resource').map((e) => ({
|
|
87
|
+
name: e.name,
|
|
88
|
+
initiatorType: e.initiatorType,
|
|
89
|
+
transferSize: e.transferSize ?? 0,
|
|
90
|
+
}))).catch(() => []);
|
|
91
|
+
for (const e of entries) {
|
|
92
|
+
requestCount++;
|
|
93
|
+
totalTransferBytes += e.transferSize;
|
|
94
|
+
const cat = e.initiatorType === 'script' ? 'scripts'
|
|
95
|
+
: e.initiatorType === 'css' || e.name?.endsWith('.css') ? 'styles'
|
|
96
|
+
: e.initiatorType === 'img' ? 'images'
|
|
97
|
+
: e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? 'fonts'
|
|
98
|
+
: 'other';
|
|
99
|
+
resources[cat].count++;
|
|
100
|
+
resources[cat].totalBytes += e.transferSize;
|
|
46
101
|
}
|
|
47
|
-
setTimeout(() => resolve(result), 3e3);
|
|
48
|
-
});
|
|
49
|
-
}).catch(() => ({}));
|
|
50
|
-
} catch {
|
|
51
|
-
}
|
|
52
|
-
let resources = {
|
|
53
|
-
scripts: { count: 0, totalBytes: 0 },
|
|
54
|
-
styles: { count: 0, totalBytes: 0 },
|
|
55
|
-
images: { count: 0, totalBytes: 0 },
|
|
56
|
-
fonts: { count: 0, totalBytes: 0 },
|
|
57
|
-
other: { count: 0, totalBytes: 0 }
|
|
58
|
-
};
|
|
59
|
-
let totalTransferBytes = 0;
|
|
60
|
-
let requestCount = 0;
|
|
61
|
-
try {
|
|
62
|
-
const entries = await page.evaluate(
|
|
63
|
-
() => performance.getEntriesByType("resource").map((e) => ({
|
|
64
|
-
name: e.name,
|
|
65
|
-
initiatorType: e.initiatorType,
|
|
66
|
-
transferSize: e.transferSize ?? 0
|
|
67
|
-
}))
|
|
68
|
-
).catch(() => []);
|
|
69
|
-
for (const e of entries) {
|
|
70
|
-
requestCount++;
|
|
71
|
-
totalTransferBytes += e.transferSize;
|
|
72
|
-
const cat = e.initiatorType === "script" ? "scripts" : e.initiatorType === "css" || e.name?.endsWith(".css") ? "styles" : e.initiatorType === "img" ? "images" : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? "fonts" : "other";
|
|
73
|
-
resources[cat].count++;
|
|
74
|
-
resources[cat].totalBytes += e.transferSize;
|
|
75
102
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
|
|
103
|
+
catch { }
|
|
104
|
+
return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
|
|
79
105
|
}
|
|
80
|
-
export {
|
|
81
|
-
collectPerfMetrics
|
|
82
|
-
};
|
|
83
|
-
//# sourceMappingURL=performance-metrics.js.map
|
package/dist/phash.js
CHANGED
|
@@ -1,136 +1,155 @@
|
|
|
1
|
-
import sharp from
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
1
|
+
import sharp from 'sharp';
|
|
2
|
+
/**
|
|
3
|
+
* Composites neutral grey (128,128,128) over each ignoredRegion (fractional
|
|
4
|
+
* 0-1 coordinates) so masked content — timestamps, live counters, ads —
|
|
5
|
+
* contributes nothing to any downstream pixel comparison. Shared by every
|
|
6
|
+
* comparison path (whole-image hash, per-cell grid hash, AI vision) so a
|
|
7
|
+
* region masked once is actually ignored everywhere, not just in the
|
|
8
|
+
* cheapest check.
|
|
9
|
+
*/
|
|
10
|
+
export async function applyMask(imagePath, ignoredRegions) {
|
|
11
|
+
const meta = await sharp(imagePath).metadata();
|
|
12
|
+
const W = meta.width ?? 1200;
|
|
13
|
+
const H = meta.height ?? 800;
|
|
14
|
+
const overlays = await Promise.all(ignoredRegions.map(async (r) => {
|
|
15
|
+
const rW = Math.max(1, Math.round(r.width * W));
|
|
16
|
+
const rH = Math.max(1, Math.round(r.height * H));
|
|
17
|
+
const buf = await sharp({
|
|
18
|
+
create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },
|
|
19
|
+
}).png().toBuffer();
|
|
20
|
+
return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
|
|
21
|
+
}));
|
|
22
|
+
return sharp(imagePath).composite(overlays).png().toBuffer();
|
|
17
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.
|
|
26
|
+
* ignoredRegions are fractional (0-1 range), converted to pixels here.
|
|
27
|
+
* Masked regions are filled with neutral grey (128,128,128) before hashing.
|
|
28
|
+
*/
|
|
18
29
|
async function hashWithMasks(imagePath, ignoredRegions = []) {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
30
|
+
const pipeline = ignoredRegions.length > 0
|
|
31
|
+
? sharp(await applyMask(imagePath, ignoredRegions))
|
|
32
|
+
: sharp(imagePath);
|
|
33
|
+
const buf = await pipeline.resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer();
|
|
34
|
+
const pixels = Array.from(buf);
|
|
35
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
36
|
+
return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);
|
|
24
37
|
}
|
|
25
38
|
function hammingDistance(a, b) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
|
|
35
|
-
const metaA = await sharp(pathA).metadata();
|
|
36
|
-
const W = metaA.width ?? 1200;
|
|
37
|
-
const H = metaA.height ?? 800;
|
|
38
|
-
const toRawRGBA = async (path) => {
|
|
39
|
-
const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
|
|
40
|
-
const { data } = await base.resize(W, H, { fit: "fill" }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
41
|
-
return data;
|
|
42
|
-
};
|
|
43
|
-
const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
|
|
44
|
-
const { default: pixelmatch } = await import("pixelmatch");
|
|
45
|
-
const diffData = Buffer.alloc(W * H * 4);
|
|
46
|
-
pixelmatch(rawA, rawB, diffData, W, H, {
|
|
47
|
-
threshold: 0.1,
|
|
48
|
-
alpha: 0.3,
|
|
49
|
-
diffColor: [255, 0, 0],
|
|
50
|
-
aaColor: [255, 255, 0]
|
|
51
|
-
});
|
|
52
|
-
return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
|
|
39
|
+
let xor = a ^ b;
|
|
40
|
+
let count = 0;
|
|
41
|
+
while (xor !== 0n) {
|
|
42
|
+
count += Number(xor & 1n);
|
|
43
|
+
xor >>= 1n;
|
|
44
|
+
}
|
|
45
|
+
return count;
|
|
53
46
|
}
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
47
|
+
/**
|
|
48
|
+
* Generates a pixelmatch diff image highlighting changed pixels in red.
|
|
49
|
+
* Applies the same ignoredRegions mask as screenshotDiffPercent so masked
|
|
50
|
+
* zones don't show as "changed" in the visual output.
|
|
51
|
+
* Returns a PNG Buffer suitable for upload to S3.
|
|
52
|
+
*/
|
|
53
|
+
export async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
|
|
54
|
+
const metaA = await sharp(pathA).metadata();
|
|
55
|
+
const W = metaA.width ?? 1200;
|
|
56
|
+
const H = metaA.height ?? 800;
|
|
57
|
+
const toRawRGBA = async (path) => {
|
|
58
|
+
const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
|
|
59
|
+
const { data } = await base.resize(W, H, { fit: 'fill' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
60
|
+
return data;
|
|
61
|
+
};
|
|
62
|
+
const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
|
|
63
|
+
const { default: pixelmatch } = await import('pixelmatch');
|
|
64
|
+
const diffData = Buffer.alloc(W * H * 4);
|
|
65
|
+
pixelmatch(rawA, rawB, diffData, W, H, {
|
|
66
|
+
threshold: 0.1,
|
|
67
|
+
alpha: 0.3,
|
|
68
|
+
diffColor: [255, 0, 0],
|
|
69
|
+
aaColor: [255, 255, 0],
|
|
70
|
+
});
|
|
71
|
+
return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
|
|
68
72
|
}
|
|
69
|
-
async function
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
"top-right",
|
|
74
|
-
"mid-left",
|
|
75
|
-
"mid-center",
|
|
76
|
-
"mid-right",
|
|
77
|
-
"bottom-left",
|
|
78
|
-
"bottom-center",
|
|
79
|
-
"bottom-right"
|
|
80
|
-
];
|
|
81
|
-
try {
|
|
82
|
-
const [sourceA, sourceB] = ignoredRegions.length > 0 ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)]) : [pathA, pathB];
|
|
83
|
-
const [metaA, metaB] = await Promise.all([
|
|
84
|
-
sharp(sourceA).metadata(),
|
|
85
|
-
sharp(sourceB).metadata()
|
|
73
|
+
export async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
|
|
74
|
+
const [hashA, hashB] = await Promise.all([
|
|
75
|
+
hashWithMasks(pathA, ignoredRegions),
|
|
76
|
+
hashWithMasks(pathB, ignoredRegions),
|
|
86
77
|
]);
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
78
|
+
let xor = hashA ^ hashB;
|
|
79
|
+
let dist = 0;
|
|
80
|
+
while (xor !== 0n) {
|
|
81
|
+
dist += Number(xor & 1n);
|
|
82
|
+
xor >>= 1n;
|
|
83
|
+
}
|
|
84
|
+
const pct = parseFloat(((dist / 64) * 100).toFixed(1));
|
|
85
|
+
const changeLevel = pct === 0 ? 'IDENTICAL' : pct < 10 ? 'MINOR' : pct < 30 ? 'MODERATE' : 'SIGNIFICANT';
|
|
86
|
+
return { pHashDistance: pct, changeLevel };
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Grid-based layout analysis: divides screenshots into a 3×3 grid and computes
|
|
90
|
+
* pHash per cell. Returns which regions changed and where, for the VisualDiff.changes field.
|
|
91
|
+
* Uses sharp's `extract` to crop each cell before hashing.
|
|
92
|
+
*/
|
|
93
|
+
export async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
|
|
94
|
+
const GRID_LABELS = [
|
|
95
|
+
'top-left', 'top-center', 'top-right',
|
|
96
|
+
'mid-left', 'mid-center', 'mid-right',
|
|
97
|
+
'bottom-left', 'bottom-center', 'bottom-right',
|
|
98
|
+
];
|
|
99
|
+
try {
|
|
100
|
+
// Mask before cropping into cells — a masked region (e.g. a timestamp)
|
|
101
|
+
// then contributes nothing to any cell's hash, instead of only being
|
|
102
|
+
// ignored by the separate whole-image hash in screenshotDiffPercent().
|
|
103
|
+
const [sourceA, sourceB] = ignoredRegions.length > 0
|
|
104
|
+
? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)])
|
|
105
|
+
: [pathA, pathB];
|
|
106
|
+
const [metaA, metaB] = await Promise.all([
|
|
107
|
+
sharp(sourceA).metadata(),
|
|
108
|
+
sharp(sourceB).metadata(),
|
|
107
109
|
]);
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
110
|
+
const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
|
|
111
|
+
const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
|
|
112
|
+
const cW = Math.floor(W / 3);
|
|
113
|
+
const cH = Math.floor(H / 3);
|
|
114
|
+
const changes = [];
|
|
115
|
+
for (let row = 0; row < 3; row++) {
|
|
116
|
+
for (let col = 0; col < 3; col++) {
|
|
117
|
+
const region = { left: col * cW, top: row * cH, width: cW, height: cH };
|
|
118
|
+
const label = GRID_LABELS[row * 3 + col];
|
|
119
|
+
const [hashA, hashB] = await Promise.all([
|
|
120
|
+
sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
|
|
121
|
+
.then((buf) => {
|
|
122
|
+
const pixels = Array.from(buf);
|
|
123
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
124
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
|
|
125
|
+
}),
|
|
126
|
+
sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
|
|
127
|
+
.then((buf) => {
|
|
128
|
+
const pixels = Array.from(buf);
|
|
129
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
130
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
|
|
131
|
+
}),
|
|
132
|
+
]);
|
|
133
|
+
let xor = hashA ^ hashB;
|
|
134
|
+
let dist = 0;
|
|
135
|
+
while (xor !== 0n) {
|
|
136
|
+
dist += Number(xor & 1n);
|
|
137
|
+
xor >>= 1n;
|
|
138
|
+
}
|
|
139
|
+
const pct = (dist / 64) * 100;
|
|
140
|
+
if (pct >= 5) {
|
|
141
|
+
const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';
|
|
142
|
+
changes.push({
|
|
143
|
+
location: label,
|
|
144
|
+
severity,
|
|
145
|
+
description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
}
|
|
122
149
|
}
|
|
123
|
-
|
|
150
|
+
return changes;
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return [];
|
|
124
154
|
}
|
|
125
|
-
return changes;
|
|
126
|
-
} catch {
|
|
127
|
-
return [];
|
|
128
|
-
}
|
|
129
155
|
}
|
|
130
|
-
export {
|
|
131
|
-
applyMask,
|
|
132
|
-
generatePixelmatchDiff,
|
|
133
|
-
gridLayoutDiff,
|
|
134
|
-
screenshotDiffPercent
|
|
135
|
-
};
|
|
136
|
-
//# sourceMappingURL=phash.js.map
|