@detiq/crawler 1.1.9 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accessibility-auditor.js +54 -74
- package/dist/ai-login.js +227 -249
- package/dist/auth-token-rotator.js +57 -67
- package/dist/auth.js +299 -336
- package/dist/browser-extension.js +104 -142
- package/dist/captcha-solver.js +98 -125
- package/dist/color-contrast.js +77 -89
- package/dist/content-chunker.js +126 -126
- package/dist/content-dedup.js +54 -71
- package/dist/cookie-consent.js +68 -83
- package/dist/cookie-jar.js +53 -60
- package/dist/crawler.js +4494 -5353
- package/dist/custom-js-injector.js +45 -61
- package/dist/dark-mode-capture.js +67 -81
- package/dist/dead-link-detector.js +47 -55
- package/dist/element-hash.js +8 -12
- package/dist/email-otp.js +154 -195
- package/dist/engine-router.js +87 -98
- package/dist/export/bulk-export.js +70 -55
- package/dist/export/vector-export.js +77 -105
- package/dist/extractors/advanced.js +249 -266
- package/dist/extractors/artifact-envelope.js +59 -58
- package/dist/failure-analysis-agent.js +37 -33
- package/dist/feature-flag-detector.js +51 -66
- package/dist/form-detector.js +55 -75
- package/dist/frontier.js +76 -101
- package/dist/gap-pass.js +68 -89
- package/dist/graphql-detector.js +62 -72
- package/dist/http-engine.js +282 -330
- package/dist/incremental-diff.js +54 -87
- package/dist/index.js +67 -28
- package/dist/input-fuzzer.js +93 -97
- package/dist/interactive-probe.js +715 -827
- package/dist/job-queue.js +2151 -2169
- package/dist/js-coverage.js +52 -56
- package/dist/js-router-detector.js +86 -117
- package/dist/language-detector.js +51 -63
- package/dist/link-classifier.js +32 -50
- package/dist/live-view.js +91 -108
- package/dist/locator-extractor.js +151 -187
- package/dist/logger.js +9 -7
- package/dist/markdown-extract.js +48 -61
- package/dist/native-crawler.js +780 -946
- package/dist/network-observer.js +135 -156
- package/dist/network-throttle.js +29 -40
- package/dist/openapi-generator.js +77 -92
- package/dist/pdf-extractor.js +72 -90
- package/dist/performance-metrics.js +80 -102
- package/dist/phash.js +125 -144
- package/dist/pii-scrubber.js +65 -68
- package/dist/playwright-recorder.js +70 -58
- package/dist/policies/crawl-safety.js +31 -34
- package/dist/profiles/domain-profile.js +130 -125
- package/dist/proxy.js +146 -172
- package/dist/rate-limiter.js +58 -65
- package/dist/redirect-tracker.js +60 -64
- package/dist/redis-frontier.js +116 -136
- package/dist/redis-queue.js +157 -144
- package/dist/response-cache.js +127 -140
- package/dist/responsive-detector.js +42 -46
- package/dist/retry-policy.js +46 -50
- package/dist/robots-txt.js +69 -83
- package/dist/screen-reader.js +45 -70
- package/dist/screenshot-masker.js +59 -77
- package/dist/seo-checker.js +88 -107
- package/dist/session-capture.js +192 -212
- package/dist/session-export.js +107 -133
- package/dist/sitemap-writer.js +38 -41
- package/dist/sitemap.js +141 -202
- package/dist/skyvern.js +123 -148
- package/dist/sqs-queue.js +42 -42
- package/dist/sso-token.js +74 -90
- package/dist/stealth/human-behavior.js +156 -173
- package/dist/telemetry.js +68 -82
- package/dist/test-executor.js +309 -335
- package/dist/tls-fingerprint.js +77 -88
- package/dist/url-pattern-completer.js +71 -90
- package/dist/visual-diff-verdict.js +23 -38
- package/dist/webhook-notifier.js +71 -69
- package/dist/websocket-detector.js +56 -68
- package/dist/worker.js +29 -38
- package/package.json +1 -1
- package/src/job-queue.ts +136 -0
package/dist/pdf-extractor.js
CHANGED
|
@@ -1,99 +1,81 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* PDF extraction — discovers and extracts text from PDF URLs found during crawl.
|
|
3
|
-
*
|
|
4
|
-
* When the HTTP engine or browser engine discovers a link to a .pdf file,
|
|
5
|
-
* this module fetches the PDF, extracts its text content, and returns it
|
|
6
|
-
* as a screen-compatible markdown string for test generation.
|
|
7
|
-
*
|
|
8
|
-
* Uses pdf-parse (already installed in API package, accessible via dynamic import).
|
|
9
|
-
*/
|
|
10
1
|
const PDF_EXT = /\.pdf(\?.*)?$/i;
|
|
11
|
-
const PDF_MIME =
|
|
12
|
-
|
|
13
|
-
|
|
2
|
+
const PDF_MIME = "application/pdf";
|
|
3
|
+
function isPdfUrl(url, contentType) {
|
|
4
|
+
return PDF_EXT.test(url) || (contentType?.includes(PDF_MIME) ?? false);
|
|
14
5
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
|
|
33
|
-
return null;
|
|
34
|
-
}
|
|
35
|
-
buffer = Buffer.from(arrayBuffer);
|
|
36
|
-
}
|
|
37
|
-
catch (e) {
|
|
38
|
-
console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
|
|
39
|
-
return null;
|
|
40
|
-
}
|
|
41
|
-
try {
|
|
42
|
-
// pdf-parse is installed in the API package — use dynamic import
|
|
43
|
-
// so this file compiles without adding pdf-parse to crawler's package.json
|
|
44
|
-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
45
|
-
// @ts-ignore – no types for pdf-parse/lib/pdf-parse.js
|
|
46
|
-
const pdfParse = await import('pdf-parse/lib/pdf-parse.js').catch(() => import('pdf-parse'));
|
|
47
|
-
const parse = pdfParse.default ?? pdfParse;
|
|
48
|
-
const data = await parse(buffer, { max: 0 }); // max: 0 = all pages
|
|
49
|
-
const title = data.info?.Title || extractTitleFromUrl(url);
|
|
50
|
-
const markdown = pdfTextToMarkdown(data.text ?? '', title);
|
|
51
|
-
return {
|
|
52
|
-
url,
|
|
53
|
-
title,
|
|
54
|
-
markdown,
|
|
55
|
-
pageCount: data.numpages ?? 0,
|
|
56
|
-
info: {
|
|
57
|
-
author: data.info?.Author ?? '',
|
|
58
|
-
subject: data.info?.Subject ?? '',
|
|
59
|
-
keywords: data.info?.Keywords ?? '',
|
|
60
|
-
creator: data.info?.Creator ?? '',
|
|
61
|
-
},
|
|
62
|
-
sizeBytes,
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
catch (e) {
|
|
66
|
-
console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
|
|
67
|
-
return null;
|
|
6
|
+
async function extractPdfFromUrl(url, opts = {}) {
|
|
7
|
+
const { userAgent = "ZeTa-Crawler/1.0", timeoutMs = 3e4, maxSizeBytes = 50 * 1024 * 1024 } = opts;
|
|
8
|
+
let buffer;
|
|
9
|
+
let sizeBytes;
|
|
10
|
+
try {
|
|
11
|
+
const response = await fetch(url, {
|
|
12
|
+
headers: { "User-Agent": userAgent, "Accept": "application/pdf,*/*" },
|
|
13
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
14
|
+
});
|
|
15
|
+
if (!response.ok) return null;
|
|
16
|
+
const contentType = response.headers.get("content-type") ?? "";
|
|
17
|
+
if (!contentType.includes("pdf") && !PDF_EXT.test(url)) return null;
|
|
18
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
19
|
+
sizeBytes = arrayBuffer.byteLength;
|
|
20
|
+
if (sizeBytes > maxSizeBytes) {
|
|
21
|
+
console.warn(`[pdf-extractor] PDF too large (${sizeBytes} bytes), skipping: ${url}`);
|
|
22
|
+
return null;
|
|
68
23
|
}
|
|
24
|
+
buffer = Buffer.from(arrayBuffer);
|
|
25
|
+
} catch (e) {
|
|
26
|
+
console.error(`[pdf-extractor] Failed to fetch PDF ${url}: ${e.message}`);
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
const pdfParse = await import("pdf-parse/lib/pdf-parse.js").catch(
|
|
31
|
+
() => import("pdf-parse")
|
|
32
|
+
);
|
|
33
|
+
const parse = pdfParse.default ?? pdfParse;
|
|
34
|
+
const data = await parse(buffer, { max: 0 });
|
|
35
|
+
const title = data.info?.Title || extractTitleFromUrl(url);
|
|
36
|
+
const markdown = pdfTextToMarkdown(data.text ?? "", title);
|
|
37
|
+
return {
|
|
38
|
+
url,
|
|
39
|
+
title,
|
|
40
|
+
markdown,
|
|
41
|
+
pageCount: data.numpages ?? 0,
|
|
42
|
+
info: {
|
|
43
|
+
author: data.info?.Author ?? "",
|
|
44
|
+
subject: data.info?.Subject ?? "",
|
|
45
|
+
keywords: data.info?.Keywords ?? "",
|
|
46
|
+
creator: data.info?.Creator ?? ""
|
|
47
|
+
},
|
|
48
|
+
sizeBytes
|
|
49
|
+
};
|
|
50
|
+
} catch (e) {
|
|
51
|
+
console.error(`[pdf-extractor] Failed to parse PDF ${url}: ${e.message}`);
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
69
54
|
}
|
|
70
55
|
function extractTitleFromUrl(url) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
}
|
|
56
|
+
try {
|
|
57
|
+
const { pathname } = new URL(url);
|
|
58
|
+
const filename = pathname.split("/").pop() ?? "document";
|
|
59
|
+
return filename.replace(/\.pdf$/i, "").replace(/[-_]/g, " ").trim();
|
|
60
|
+
} catch {
|
|
61
|
+
return "PDF Document";
|
|
62
|
+
}
|
|
79
63
|
}
|
|
80
64
|
function pdfTextToMarkdown(text, title) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
89
|
-
.replace(/[ \t]{2,}/g, ' ')
|
|
90
|
-
.trim();
|
|
91
|
-
return `# ${title}\n\n${cleaned}`;
|
|
65
|
+
if (!text.trim()) return `# ${title}
|
|
66
|
+
|
|
67
|
+
*No extractable text content.*`;
|
|
68
|
+
const cleaned = text.replace(/\f/g, "\n\n---\n\n").replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ").trim();
|
|
69
|
+
return `# ${title}
|
|
70
|
+
|
|
71
|
+
${cleaned}`;
|
|
92
72
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
* Used by http-engine and browser crawler to decide which links to pass to extractPdfFromUrl.
|
|
96
|
-
*/
|
|
97
|
-
export function filterPdfUrls(urls) {
|
|
98
|
-
return urls.filter(u => PDF_EXT.test(u));
|
|
73
|
+
function filterPdfUrls(urls) {
|
|
74
|
+
return urls.filter((u) => PDF_EXT.test(u));
|
|
99
75
|
}
|
|
76
|
+
export {
|
|
77
|
+
extractPdfFromUrl,
|
|
78
|
+
filterPdfUrls,
|
|
79
|
+
isPdfUrl
|
|
80
|
+
};
|
|
81
|
+
//# sourceMappingURL=pdf-extractor.js.map
|
|
@@ -1,105 +1,83 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
// CLS
|
|
47
|
-
try {
|
|
48
|
-
let cls = 0;
|
|
49
|
-
new PerformanceObserver((list) => {
|
|
50
|
-
for (const e of list.getEntries()) {
|
|
51
|
-
if (!e.hadRecentInput)
|
|
52
|
-
cls += e.value;
|
|
53
|
-
}
|
|
54
|
-
result.cls = Math.round(cls * 1000) / 1000;
|
|
55
|
-
clsDone = true;
|
|
56
|
-
tryResolve();
|
|
57
|
-
}).observe({ type: 'layout-shift', buffered: true });
|
|
58
|
-
}
|
|
59
|
-
catch {
|
|
60
|
-
clsDone = true;
|
|
61
|
-
}
|
|
62
|
-
// FCP
|
|
63
|
-
try {
|
|
64
|
-
const fcp = performance.getEntriesByName('first-contentful-paint')[0];
|
|
65
|
-
if (fcp)
|
|
66
|
-
result.fcp = Math.round(fcp.startTime);
|
|
67
|
-
}
|
|
68
|
-
catch { }
|
|
69
|
-
// Fallback resolve after 3s
|
|
70
|
-
setTimeout(() => resolve(result), 3_000);
|
|
71
|
-
});
|
|
72
|
-
}).catch(() => ({}));
|
|
73
|
-
}
|
|
74
|
-
catch { /* page closed or navigate already */ }
|
|
75
|
-
// Resource timing
|
|
76
|
-
let resources = {
|
|
77
|
-
scripts: { count: 0, totalBytes: 0 },
|
|
78
|
-
styles: { count: 0, totalBytes: 0 },
|
|
79
|
-
images: { count: 0, totalBytes: 0 },
|
|
80
|
-
fonts: { count: 0, totalBytes: 0 },
|
|
81
|
-
other: { count: 0, totalBytes: 0 },
|
|
82
|
-
};
|
|
83
|
-
let totalTransferBytes = 0;
|
|
84
|
-
let requestCount = 0;
|
|
85
|
-
try {
|
|
86
|
-
const entries = await page.evaluate(() => performance.getEntriesByType('resource').map((e) => ({
|
|
87
|
-
name: e.name,
|
|
88
|
-
initiatorType: e.initiatorType,
|
|
89
|
-
transferSize: e.transferSize ?? 0,
|
|
90
|
-
}))).catch(() => []);
|
|
91
|
-
for (const e of entries) {
|
|
92
|
-
requestCount++;
|
|
93
|
-
totalTransferBytes += e.transferSize;
|
|
94
|
-
const cat = e.initiatorType === 'script' ? 'scripts'
|
|
95
|
-
: e.initiatorType === 'css' || e.name?.endsWith('.css') ? 'styles'
|
|
96
|
-
: e.initiatorType === 'img' ? 'images'
|
|
97
|
-
: e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? 'fonts'
|
|
98
|
-
: 'other';
|
|
99
|
-
resources[cat].count++;
|
|
100
|
-
resources[cat].totalBytes += e.transferSize;
|
|
1
|
+
async function collectPerfMetrics(page, url) {
|
|
2
|
+
const measuredAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
3
|
+
let vitals = {};
|
|
4
|
+
try {
|
|
5
|
+
vitals = await page.evaluate(() => {
|
|
6
|
+
return new Promise((resolve) => {
|
|
7
|
+
const result = {};
|
|
8
|
+
const nav = performance.getEntriesByType("navigation")[0];
|
|
9
|
+
if (nav) {
|
|
10
|
+
result.ttfb = Math.round(nav.responseStart - nav.requestStart);
|
|
11
|
+
result.domContentLoaded = Math.round(nav.domContentLoadedEventEnd);
|
|
12
|
+
result.loadEventEnd = Math.round(nav.loadEventEnd);
|
|
13
|
+
}
|
|
14
|
+
let lcpDone = false;
|
|
15
|
+
let clsDone = false;
|
|
16
|
+
const tryResolve = () => {
|
|
17
|
+
if (lcpDone && clsDone) resolve(result);
|
|
18
|
+
};
|
|
19
|
+
try {
|
|
20
|
+
new PerformanceObserver((list) => {
|
|
21
|
+
const entries = list.getEntries();
|
|
22
|
+
if (entries.length > 0) result.lcp = Math.round(entries[entries.length - 1].startTime);
|
|
23
|
+
lcpDone = true;
|
|
24
|
+
tryResolve();
|
|
25
|
+
}).observe({ type: "largest-contentful-paint", buffered: true });
|
|
26
|
+
} catch {
|
|
27
|
+
lcpDone = true;
|
|
28
|
+
}
|
|
29
|
+
try {
|
|
30
|
+
let cls = 0;
|
|
31
|
+
new PerformanceObserver((list) => {
|
|
32
|
+
for (const e of list.getEntries()) {
|
|
33
|
+
if (!e.hadRecentInput) cls += e.value;
|
|
34
|
+
}
|
|
35
|
+
result.cls = Math.round(cls * 1e3) / 1e3;
|
|
36
|
+
clsDone = true;
|
|
37
|
+
tryResolve();
|
|
38
|
+
}).observe({ type: "layout-shift", buffered: true });
|
|
39
|
+
} catch {
|
|
40
|
+
clsDone = true;
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const fcp = performance.getEntriesByName("first-contentful-paint")[0];
|
|
44
|
+
if (fcp) result.fcp = Math.round(fcp.startTime);
|
|
45
|
+
} catch {
|
|
101
46
|
}
|
|
47
|
+
setTimeout(() => resolve(result), 3e3);
|
|
48
|
+
});
|
|
49
|
+
}).catch(() => ({}));
|
|
50
|
+
} catch {
|
|
51
|
+
}
|
|
52
|
+
let resources = {
|
|
53
|
+
scripts: { count: 0, totalBytes: 0 },
|
|
54
|
+
styles: { count: 0, totalBytes: 0 },
|
|
55
|
+
images: { count: 0, totalBytes: 0 },
|
|
56
|
+
fonts: { count: 0, totalBytes: 0 },
|
|
57
|
+
other: { count: 0, totalBytes: 0 }
|
|
58
|
+
};
|
|
59
|
+
let totalTransferBytes = 0;
|
|
60
|
+
let requestCount = 0;
|
|
61
|
+
try {
|
|
62
|
+
const entries = await page.evaluate(
|
|
63
|
+
() => performance.getEntriesByType("resource").map((e) => ({
|
|
64
|
+
name: e.name,
|
|
65
|
+
initiatorType: e.initiatorType,
|
|
66
|
+
transferSize: e.transferSize ?? 0
|
|
67
|
+
}))
|
|
68
|
+
).catch(() => []);
|
|
69
|
+
for (const e of entries) {
|
|
70
|
+
requestCount++;
|
|
71
|
+
totalTransferBytes += e.transferSize;
|
|
72
|
+
const cat = e.initiatorType === "script" ? "scripts" : e.initiatorType === "css" || e.name?.endsWith(".css") ? "styles" : e.initiatorType === "img" ? "images" : e.name?.match(/\.(woff|woff2|ttf|otf|eot)/) ? "fonts" : "other";
|
|
73
|
+
resources[cat].count++;
|
|
74
|
+
resources[cat].totalBytes += e.transferSize;
|
|
102
75
|
}
|
|
103
|
-
|
|
104
|
-
|
|
76
|
+
} catch {
|
|
77
|
+
}
|
|
78
|
+
return { url, vitals, resources, totalTransferBytes, requestCount, measuredAt };
|
|
105
79
|
}
|
|
80
|
+
export {
|
|
81
|
+
collectPerfMetrics
|
|
82
|
+
};
|
|
83
|
+
//# sourceMappingURL=performance-metrics.js.map
|
package/dist/phash.js
CHANGED
|
@@ -1,155 +1,136 @@
|
|
|
1
|
-
import sharp from
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
const buf = await sharp({
|
|
18
|
-
create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } },
|
|
19
|
-
}).png().toBuffer();
|
|
20
|
-
return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
|
|
21
|
-
}));
|
|
22
|
-
return sharp(imagePath).composite(overlays).png().toBuffer();
|
|
1
|
+
import sharp from "sharp";
|
|
2
|
+
async function applyMask(imagePath, ignoredRegions) {
|
|
3
|
+
const meta = await sharp(imagePath).metadata();
|
|
4
|
+
const W = meta.width ?? 1200;
|
|
5
|
+
const H = meta.height ?? 800;
|
|
6
|
+
const overlays = await Promise.all(
|
|
7
|
+
ignoredRegions.map(async (r) => {
|
|
8
|
+
const rW = Math.max(1, Math.round(r.width * W));
|
|
9
|
+
const rH = Math.max(1, Math.round(r.height * H));
|
|
10
|
+
const buf = await sharp({
|
|
11
|
+
create: { width: rW, height: rH, channels: 3, background: { r: 128, g: 128, b: 128 } }
|
|
12
|
+
}).png().toBuffer();
|
|
13
|
+
return { input: buf, top: Math.round(r.y * H), left: Math.round(r.x * W) };
|
|
14
|
+
})
|
|
15
|
+
);
|
|
16
|
+
return sharp(imagePath).composite(overlays).png().toBuffer();
|
|
23
17
|
}
|
|
24
|
-
/**
|
|
25
|
-
* Compute an 8x8 average hash (aHash) of an image file, with optional mask overlays.
|
|
26
|
-
* ignoredRegions are fractional (0-1 range), converted to pixels here.
|
|
27
|
-
* Masked regions are filled with neutral grey (128,128,128) before hashing.
|
|
28
|
-
*/
|
|
29
18
|
async function hashWithMasks(imagePath, ignoredRegions = []) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
36
|
-
return pixels.reduce((h, p, i) => (p >= mean ? h | (1n << BigInt(63 - i)) : h), 0n);
|
|
19
|
+
const pipeline = ignoredRegions.length > 0 ? sharp(await applyMask(imagePath, ignoredRegions)) : sharp(imagePath);
|
|
20
|
+
const buf = await pipeline.resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer();
|
|
21
|
+
const pixels = Array.from(buf);
|
|
22
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
23
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
37
24
|
}
|
|
38
25
|
function hammingDistance(a, b) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
26
|
+
let xor = a ^ b;
|
|
27
|
+
let count = 0;
|
|
28
|
+
while (xor !== 0n) {
|
|
29
|
+
count += Number(xor & 1n);
|
|
30
|
+
xor >>= 1n;
|
|
31
|
+
}
|
|
32
|
+
return count;
|
|
46
33
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
threshold: 0.1,
|
|
67
|
-
alpha: 0.3,
|
|
68
|
-
diffColor: [255, 0, 0],
|
|
69
|
-
aaColor: [255, 255, 0],
|
|
70
|
-
});
|
|
71
|
-
return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
|
|
34
|
+
async function generatePixelmatchDiff(pathA, pathB, ignoredRegions = []) {
|
|
35
|
+
const metaA = await sharp(pathA).metadata();
|
|
36
|
+
const W = metaA.width ?? 1200;
|
|
37
|
+
const H = metaA.height ?? 800;
|
|
38
|
+
const toRawRGBA = async (path) => {
|
|
39
|
+
const base = ignoredRegions.length > 0 ? sharp(await applyMask(path, ignoredRegions)) : sharp(path);
|
|
40
|
+
const { data } = await base.resize(W, H, { fit: "fill" }).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
41
|
+
return data;
|
|
42
|
+
};
|
|
43
|
+
const [rawA, rawB] = await Promise.all([toRawRGBA(pathA), toRawRGBA(pathB)]);
|
|
44
|
+
const { default: pixelmatch } = await import("pixelmatch");
|
|
45
|
+
const diffData = Buffer.alloc(W * H * 4);
|
|
46
|
+
pixelmatch(rawA, rawB, diffData, W, H, {
|
|
47
|
+
threshold: 0.1,
|
|
48
|
+
alpha: 0.3,
|
|
49
|
+
diffColor: [255, 0, 0],
|
|
50
|
+
aaColor: [255, 255, 0]
|
|
51
|
+
});
|
|
52
|
+
return sharp(diffData, { raw: { width: W, height: H, channels: 4 } }).png().toBuffer();
|
|
72
53
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
54
|
+
async function screenshotDiffPercent(pathA, pathB, ignoredRegions = []) {
|
|
55
|
+
const [hashA, hashB] = await Promise.all([
|
|
56
|
+
hashWithMasks(pathA, ignoredRegions),
|
|
57
|
+
hashWithMasks(pathB, ignoredRegions)
|
|
58
|
+
]);
|
|
59
|
+
let xor = hashA ^ hashB;
|
|
60
|
+
let dist = 0;
|
|
61
|
+
while (xor !== 0n) {
|
|
62
|
+
dist += Number(xor & 1n);
|
|
63
|
+
xor >>= 1n;
|
|
64
|
+
}
|
|
65
|
+
const pct = parseFloat((dist / 64 * 100).toFixed(1));
|
|
66
|
+
const changeLevel = pct === 0 ? "IDENTICAL" : pct < 10 ? "MINOR" : pct < 30 ? "MODERATE" : "SIGNIFICANT";
|
|
67
|
+
return { pHashDistance: pct, changeLevel };
|
|
87
68
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
69
|
+
async function gridLayoutDiff(pathA, pathB, ignoredRegions = []) {
|
|
70
|
+
const GRID_LABELS = [
|
|
71
|
+
"top-left",
|
|
72
|
+
"top-center",
|
|
73
|
+
"top-right",
|
|
74
|
+
"mid-left",
|
|
75
|
+
"mid-center",
|
|
76
|
+
"mid-right",
|
|
77
|
+
"bottom-left",
|
|
78
|
+
"bottom-center",
|
|
79
|
+
"bottom-right"
|
|
80
|
+
];
|
|
81
|
+
try {
|
|
82
|
+
const [sourceA, sourceB] = ignoredRegions.length > 0 ? await Promise.all([applyMask(pathA, ignoredRegions), applyMask(pathB, ignoredRegions)]) : [pathA, pathB];
|
|
83
|
+
const [metaA, metaB] = await Promise.all([
|
|
84
|
+
sharp(sourceA).metadata(),
|
|
85
|
+
sharp(sourceB).metadata()
|
|
86
|
+
]);
|
|
87
|
+
const W = Math.min(metaA.width ?? 1200, metaB.width ?? 1200);
|
|
88
|
+
const H = Math.min(metaA.height ?? 800, metaB.height ?? 800);
|
|
89
|
+
const cW = Math.floor(W / 3);
|
|
90
|
+
const cH = Math.floor(H / 3);
|
|
91
|
+
const changes = [];
|
|
92
|
+
for (let row = 0; row < 3; row++) {
|
|
93
|
+
for (let col = 0; col < 3; col++) {
|
|
94
|
+
const region = { left: col * cW, top: row * cH, width: cW, height: cH };
|
|
95
|
+
const label = GRID_LABELS[row * 3 + col];
|
|
96
|
+
const [hashA, hashB] = await Promise.all([
|
|
97
|
+
sharp(sourceA).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
|
|
98
|
+
const pixels = Array.from(buf);
|
|
99
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
100
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
101
|
+
}),
|
|
102
|
+
sharp(sourceB).extract(region).resize(8, 8, { fit: "fill" }).greyscale().raw().toBuffer().then((buf) => {
|
|
103
|
+
const pixels = Array.from(buf);
|
|
104
|
+
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
105
|
+
return pixels.reduce((h, p, i) => p >= mean ? h | 1n << BigInt(63 - i) : h, 0n);
|
|
106
|
+
})
|
|
109
107
|
]);
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
for (let row = 0; row < 3; row++) {
|
|
116
|
-
for (let col = 0; col < 3; col++) {
|
|
117
|
-
const region = { left: col * cW, top: row * cH, width: cW, height: cH };
|
|
118
|
-
const label = GRID_LABELS[row * 3 + col];
|
|
119
|
-
const [hashA, hashB] = await Promise.all([
|
|
120
|
-
sharp(sourceA).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
|
|
121
|
-
.then((buf) => {
|
|
122
|
-
const pixels = Array.from(buf);
|
|
123
|
-
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
124
|
-
return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
|
|
125
|
-
}),
|
|
126
|
-
sharp(sourceB).extract(region).resize(8, 8, { fit: 'fill' }).greyscale().raw().toBuffer()
|
|
127
|
-
.then((buf) => {
|
|
128
|
-
const pixels = Array.from(buf);
|
|
129
|
-
const mean = pixels.reduce((s, p) => s + p, 0) / pixels.length;
|
|
130
|
-
return pixels.reduce((h, p, i) => p >= mean ? h | (1n << BigInt(63 - i)) : h, 0n);
|
|
131
|
-
}),
|
|
132
|
-
]);
|
|
133
|
-
let xor = hashA ^ hashB;
|
|
134
|
-
let dist = 0;
|
|
135
|
-
while (xor !== 0n) {
|
|
136
|
-
dist += Number(xor & 1n);
|
|
137
|
-
xor >>= 1n;
|
|
138
|
-
}
|
|
139
|
-
const pct = (dist / 64) * 100;
|
|
140
|
-
if (pct >= 5) {
|
|
141
|
-
const severity = pct >= 20 ? 'HIGH' : pct >= 10 ? 'MEDIUM' : 'LOW';
|
|
142
|
-
changes.push({
|
|
143
|
-
location: label,
|
|
144
|
-
severity,
|
|
145
|
-
description: `${Math.round(pct)}% visual change in ${label.replace('-', ' ')} region`,
|
|
146
|
-
});
|
|
147
|
-
}
|
|
148
|
-
}
|
|
108
|
+
let xor = hashA ^ hashB;
|
|
109
|
+
let dist = 0;
|
|
110
|
+
while (xor !== 0n) {
|
|
111
|
+
dist += Number(xor & 1n);
|
|
112
|
+
xor >>= 1n;
|
|
149
113
|
}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
114
|
+
const pct = dist / 64 * 100;
|
|
115
|
+
if (pct >= 5) {
|
|
116
|
+
const severity = pct >= 20 ? "HIGH" : pct >= 10 ? "MEDIUM" : "LOW";
|
|
117
|
+
changes.push({
|
|
118
|
+
location: label,
|
|
119
|
+
severity,
|
|
120
|
+
description: `${Math.round(pct)}% visual change in ${label.replace("-", " ")} region`
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
}
|
|
154
124
|
}
|
|
125
|
+
return changes;
|
|
126
|
+
} catch {
|
|
127
|
+
return [];
|
|
128
|
+
}
|
|
155
129
|
}
|
|
130
|
+
export {
|
|
131
|
+
applyMask,
|
|
132
|
+
generatePixelmatchDiff,
|
|
133
|
+
gridLayoutDiff,
|
|
134
|
+
screenshotDiffPercent
|
|
135
|
+
};
|
|
136
|
+
//# sourceMappingURL=phash.js.map
|