dembrandt 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -6
- package/index.js +20 -2
- package/lib/extractors/colors.js +23 -0
- package/lib/extractors/index.js +405 -39
- package/lib/extractors/typography.js +32 -0
- package/lib/merger.js +1 -0
- package/mcp-server.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -51,13 +51,13 @@ npx skills add dembrandt/dembrandt-skills
|
|
|
51
51
|
|
|
52
52
|
## Dembrandt App (Beta)
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
Load extractions, track token drift, and compare snapshots. **[dembrandt.com/app](https://www.dembrandt.com/app)**
|
|
55
55
|
|
|
56
|
-
* **
|
|
57
|
-
* **
|
|
58
|
-
* **
|
|
59
|
-
* **
|
|
60
|
-
* **
|
|
56
|
+
* **Drift tracking.** Pin a snapshot as your baseline. Run another extraction later. Get a visual report of what changed.
|
|
57
|
+
* **Visual diff.** Color swatches, before/after values, delta scores per category.
|
|
58
|
+
* **Snapshot history.** GitHub-style calendar per domain.
|
|
59
|
+
* **Copy tokens.** Paste values straight into Copilot, Claude, or Cursor.
|
|
60
|
+
* **No login.** Your data stays in the browser, nothing is sent to any server.
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
## What to expect from extraction?
|
|
@@ -90,6 +90,11 @@ dembrandt example.com --crawl 10 --sitemap # Combine: up to 10 pages discovered
|
|
|
90
90
|
dembrandt example.com --no-sandbox # Disable Chromium sandbox (required for Docker/CI)
|
|
91
91
|
dembrandt example.com --browser=firefox # Use Firefox instead of Chromium (better for Cloudflare bypass)
|
|
92
92
|
dembrandt example.com --wcag # WCAG 2.1 contrast analysis — real DOM pairs, AA/AAA grades
|
|
93
|
+
dembrandt example.com --stealth # Opt-in anti-detection: navigator spoofing + human mouse simulation (use only when authorized)
|
|
94
|
+
dembrandt example.com --locale fi-FI --timezone Europe/Helsinki # Browser fingerprint: locale and timezone
|
|
95
|
+
dembrandt example.com --user-agent "Mozilla/5.0 ..." # Custom user agent string
|
|
96
|
+
dembrandt example.com --accept-language "fi,en;q=0.9" # Custom Accept-Language header
|
|
97
|
+
dembrandt example.com --screen-size 2560x1440 # Physical screen resolution to report
|
|
93
98
|
```
|
|
94
99
|
|
|
95
100
|
Default: formatted terminal display only. Use `--save-output` to persist results as JSON files. Browser automatically retries in visible mode if headless extraction fails.
|
package/index.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
* Dembrandt - Design Token Extraction CLI
|
|
5
5
|
*
|
|
6
6
|
* Extracts design tokens, brand colors, typography, spacing, and component styles
|
|
7
|
-
* from any website using Playwright
|
|
7
|
+
* from any website using Playwright.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { program } from "commander";
|
|
@@ -58,6 +58,12 @@ program
|
|
|
58
58
|
return n;
|
|
59
59
|
})
|
|
60
60
|
.option("--sitemap", "Discover pages from sitemap.xml instead of DOM links")
|
|
61
|
+
.option("--stealth", "Enable anti-detection scripts to bypass bot protection (use only when authorized)")
|
|
62
|
+
.option("--user-agent <string>", "Custom user agent string")
|
|
63
|
+
.option("--locale <string>", "Browser locale, e.g. en-GB, fi-FI (default: en-US)")
|
|
64
|
+
.option("--timezone <string>", "Browser timezone, e.g. Europe/Helsinki (default: America/New_York)")
|
|
65
|
+
.option("--accept-language <string>", "Custom Accept-Language header value")
|
|
66
|
+
.option("--screen-size <WxH>", "Physical screen resolution to report, e.g. 1920x1080 (default: 1920x1080)")
|
|
61
67
|
.action(async (input, paths, opts) => {
|
|
62
68
|
let url = input;
|
|
63
69
|
if (!url.startsWith("http://") && !url.startsWith("https://")) {
|
|
@@ -133,6 +139,13 @@ program
|
|
|
133
139
|
discoverLinks: isAutoCrawl ? crawlN - 1 : null,
|
|
134
140
|
wcag: opts.wcag,
|
|
135
141
|
includeRawColors: opts.rawColors,
|
|
142
|
+
stealth: opts.stealth,
|
|
143
|
+
userAgent: opts.userAgent,
|
|
144
|
+
locale: opts.locale,
|
|
145
|
+
timezoneId: opts.timezone,
|
|
146
|
+
acceptLanguage: opts.acceptLanguage,
|
|
147
|
+
screenSize: opts.screenSize,
|
|
148
|
+
_version: version,
|
|
136
149
|
});
|
|
137
150
|
|
|
138
151
|
// Build list of additional URLs to extract
|
|
@@ -181,6 +194,11 @@ program
|
|
|
181
194
|
darkMode: opts.darkMode,
|
|
182
195
|
mobile: opts.mobile,
|
|
183
196
|
slow: opts.slow,
|
|
197
|
+
stealth: opts.stealth,
|
|
198
|
+
userAgent: opts.userAgent,
|
|
199
|
+
locale: opts.locale,
|
|
200
|
+
timezoneId: opts.timezone,
|
|
201
|
+
acceptLanguage: opts.acceptLanguage,
|
|
184
202
|
});
|
|
185
203
|
delete pageResult._discoveredLinks;
|
|
186
204
|
allResults.push(pageResult);
|
|
@@ -209,7 +227,7 @@ program
|
|
|
209
227
|
err.message.includes("net::ERR_")
|
|
210
228
|
) {
|
|
211
229
|
spinner.warn(
|
|
212
|
-
"
|
|
230
|
+
"Navigation failed → retrying with visible browser"
|
|
213
231
|
);
|
|
214
232
|
console.error(chalk.dim(` ↳ Error: ${err.message}`));
|
|
215
233
|
console.error(chalk.dim(` ↳ URL: ${url}`));
|
package/lib/extractors/colors.js
CHANGED
|
@@ -316,6 +316,29 @@ export async function extractColors(page) {
|
|
|
316
316
|
const filteredCssVariables = {};
|
|
317
317
|
cssVarsByColor.forEach(({ value, vars }) => { filteredCssVariables[vars[0]] = value; });
|
|
318
318
|
|
|
319
|
+
// Fallback: pick most chromatic non-gray palette color as primary
|
|
320
|
+
if (!semanticColors.primary && perceptuallyDeduped.length > 0) {
|
|
321
|
+
function chroma(hex) {
|
|
322
|
+
if (!hex || !hex.startsWith('#')) return 0;
|
|
323
|
+
const r = parseInt(hex.slice(1, 3), 16) / 255;
|
|
324
|
+
const g = parseInt(hex.slice(3, 5), 16) / 255;
|
|
325
|
+
const b = parseInt(hex.slice(5, 7), 16) / 255;
|
|
326
|
+
const max = Math.max(r, g, b), min = Math.min(r, g, b);
|
|
327
|
+
const l = (max + min) / 2;
|
|
328
|
+
if (max === min) return 0;
|
|
329
|
+
const s = l > 0.5 ? (max - min) / (2 - max - min) : (max - min) / (max + min);
|
|
330
|
+
// Penalize near-black and near-white
|
|
331
|
+
if (l < 0.08 || l > 0.92) return 0;
|
|
332
|
+
return s;
|
|
333
|
+
}
|
|
334
|
+
const best = perceptuallyDeduped
|
|
335
|
+
.filter(c => c.confidence !== 'low')
|
|
336
|
+
.map(c => ({ c, chroma: chroma(c.normalized) }))
|
|
337
|
+
.filter(({ chroma }) => chroma > 0.15)
|
|
338
|
+
.sort((a, b) => b.chroma - a.chroma)[0];
|
|
339
|
+
if (best) semanticColors.primary = best.c.color;
|
|
340
|
+
}
|
|
341
|
+
|
|
319
342
|
return { semantic: semanticColors, palette: perceptuallyDeduped, cssVariables: filteredCssVariables, _raw: rawColors };
|
|
320
343
|
});
|
|
321
344
|
|
package/lib/extractors/index.js
CHANGED
|
@@ -11,22 +11,255 @@ import { extractWcagPairs } from './colors.js';
|
|
|
11
11
|
|
|
12
12
|
/** @typedef {import('../types.js').BrandingResult} BrandingResult */
|
|
13
13
|
|
|
14
|
+
// Gaussian noise via Box-Muller
|
|
15
|
+
function gaussian(mean = 0, std = 1) {
|
|
16
|
+
let u, v;
|
|
17
|
+
do { u = Math.random(); } while (u === 0);
|
|
18
|
+
do { v = Math.random(); } while (v === 0);
|
|
19
|
+
return mean + std * Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Cubic Bézier interpolation
|
|
23
|
+
function bezier(t, p0, p1, p2, p3) {
|
|
24
|
+
const mt = 1 - t;
|
|
25
|
+
return mt * mt * mt * p0 + 3 * mt * mt * t * p1 + 3 * mt * t * t * p2 + t * t * t * p3;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Physiological tremor: ~8-12Hz oscillation, amplitude varies with fatigue
|
|
29
|
+
function tremor(t, freq, amp) {
|
|
30
|
+
return amp * Math.sin(2 * Math.PI * freq * t) + gaussian(0, amp * 0.3);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// Velocity profile: ballistic phase + corrective phase (two-phase Fitts model)
|
|
34
|
+
// Humans move fast toward target then make fine corrections — not smooth decel
|
|
35
|
+
function velocityProfile(t, overshootProb = 0.3) {
|
|
36
|
+
const hasOvershoot = Math.random() < overshootProb;
|
|
37
|
+
if (t < 0.05) return t / 0.05 * 0.2; // startup latency
|
|
38
|
+
if (t < 0.55) return 0.2 + (t - 0.05) / 0.5; // ballistic acceleration
|
|
39
|
+
if (t < 0.72) return 1.0 - (t - 0.55) / 0.17 * 0.5; // deceleration
|
|
40
|
+
if (hasOvershoot && t < 0.88) return 0.5 + Math.sin((t - 0.72) / 0.16 * Math.PI) * 0.3; // overshoot
|
|
41
|
+
return 0.15 + Math.random() * 0.1; // corrective micro-movements
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Sleep helper
|
|
45
|
+
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
46
|
+
|
|
47
|
+
async function simulateHumanMouse(page) {
|
|
48
|
+
const vw = 1920, vh = 1080;
|
|
49
|
+
|
|
50
|
+
// Per-session behavioral fingerprint — each "user" has consistent quirks
|
|
51
|
+
const profile = {
|
|
52
|
+
handedness: Math.random() < 0.88 ? 'right' : 'left', // right-handed bias
|
|
53
|
+
tremFreq: 8 + Math.random() * 4, // physiological tremor 8-12Hz
|
|
54
|
+
tremAmp: 0.2 + Math.random() * 0.6, // tremor amplitude (fatigue)
|
|
55
|
+
driftBias: { x: gaussian(0, 0.15), y: gaussian(0, 0.08) }, // consistent directional drift
|
|
56
|
+
speedMult: 0.7 + Math.random() * 0.8, // this person moves fast or slow
|
|
57
|
+
overshootTendency: Math.random(), // how often they overshoot targets
|
|
58
|
+
attentionSpan: 0.4 + Math.random() * 0.6, // affects dwell times
|
|
59
|
+
fatigueRate: Math.random() * 0.3, // movements degrade over session
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
// Plausible entry: cursor was somewhere from before page load, not 0,0
|
|
63
|
+
// Right-handed users tend to park right-center; left-handed left-center
|
|
64
|
+
const entryX = profile.handedness === 'right'
|
|
65
|
+
? vw * 0.5 + Math.random() * vw * 0.4
|
|
66
|
+
: vw * 0.1 + Math.random() * vw * 0.4;
|
|
67
|
+
const entryY = vh * 0.15 + Math.random() * vh * 0.5;
|
|
68
|
+
|
|
69
|
+
let cx = entryX, cy = entryY;
|
|
70
|
+
await page.mouse.move(cx, cy);
|
|
71
|
+
|
|
72
|
+
// Weighted zones: humans spend time in predictable areas of a webpage
|
|
73
|
+
const targetZones = [
|
|
74
|
+
{ x: [60, 500], y: [15, 75], weight: 3 }, // navigation — high attention
|
|
75
|
+
{ x: [150, 1200], y: [80, 380], weight: 4 }, // hero/above-fold — high attention
|
|
76
|
+
{ x: [100, 900], y: [350, 720], weight: 3 }, // body content
|
|
77
|
+
{ x: [600, 1800], y: [15, 75], weight: 1 }, // right nav / utility links
|
|
78
|
+
{ x: [20, 200], y: [200, 900], weight: 1 }, // left sidebar / margin
|
|
79
|
+
{ x: [800, 1400], y: [300, 700], weight: 2 }, // mid-right content
|
|
80
|
+
];
|
|
81
|
+
const totalWeight = targetZones.reduce((s, z) => s + z.weight, 0);
|
|
82
|
+
|
|
83
|
+
function pickZone() {
|
|
84
|
+
let r = Math.random() * totalWeight;
|
|
85
|
+
for (const z of targetZones) { r -= z.weight; if (r <= 0) return z; }
|
|
86
|
+
return targetZones[0];
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const sequences = 3 + Math.floor(Math.random() * 5); // 3-7 movements
|
|
90
|
+
let sessionTime = 0;
|
|
91
|
+
|
|
92
|
+
for (let s = 0; s < sequences; s++) {
|
|
93
|
+
const fatigue = 1 + profile.fatigueRate * (s / sequences); // movements get sloppier
|
|
94
|
+
|
|
95
|
+
// Occasionally abandon a movement mid-way and redirect (changed mind)
|
|
96
|
+
const willAbort = Math.random() < 0.12;
|
|
97
|
+
|
|
98
|
+
const zone = pickZone();
|
|
99
|
+
let tx = zone.x[0] + Math.random() * (zone.x[1] - zone.x[0]);
|
|
100
|
+
let ty = zone.y[0] + Math.random() * (zone.y[1] - zone.y[0]);
|
|
101
|
+
|
|
102
|
+
// Aborted movement: pick intermediate abort point
|
|
103
|
+
const abortT = willAbort ? 0.25 + Math.random() * 0.45 : 1.0;
|
|
104
|
+
if (willAbort) {
|
|
105
|
+
const abortZone = pickZone();
|
|
106
|
+
const finalTx = abortZone.x[0] + Math.random() * (abortZone.x[1] - abortZone.x[0]);
|
|
107
|
+
const finalTy = abortZone.y[0] + Math.random() * (abortZone.y[1] - abortZone.y[0]);
|
|
108
|
+
// Abort destination is partway toward original target, then we'll redirect
|
|
109
|
+
tx = cx + (tx - cx) * abortT;
|
|
110
|
+
ty = cy + (ty - cy) * abortT;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const dist = Math.hypot(tx - cx, ty - cy);
|
|
114
|
+
if (dist < 5) continue; // skip negligible movements
|
|
115
|
+
|
|
116
|
+
// Two-segment path for longer distances (humans curve around obstacles mentally)
|
|
117
|
+
const useWaypoint = dist > 300 && Math.random() < 0.4;
|
|
118
|
+
const movements = useWaypoint ? [
|
|
119
|
+
// Waypoint slightly off the direct line
|
|
120
|
+
{
|
|
121
|
+
tx: cx + (tx - cx) * (0.3 + Math.random() * 0.25) + gaussian(0, 60),
|
|
122
|
+
ty: cy + (ty - cy) * (0.3 + Math.random() * 0.25) + gaussian(0, 80),
|
|
123
|
+
},
|
|
124
|
+
{ tx, ty },
|
|
125
|
+
] : [{ tx, ty }];
|
|
126
|
+
|
|
127
|
+
for (const { tx: etx, ty: ety } of movements) {
|
|
128
|
+
const segDist = Math.hypot(etx - cx, ety - cy);
|
|
129
|
+
|
|
130
|
+
// Bézier control points — asymmetric, biased by hand dominance
|
|
131
|
+
const lateralBias = profile.handedness === 'right' ? 1 : -1;
|
|
132
|
+
const cp1x = cx + (etx - cx) * (0.15 + Math.random() * 0.25) + gaussian(0, 35) * fatigue;
|
|
133
|
+
const cp1y = cy + (ety - cy) * (0.05 + Math.random() * 0.2) + gaussian(0, 50) * fatigue + lateralBias * gaussian(0, 15);
|
|
134
|
+
const cp2x = cx + (etx - cx) * (0.65 + Math.random() * 0.25) + gaussian(0, 25) * fatigue;
|
|
135
|
+
const cp2y = cy + (ety - cy) * (0.75 + Math.random() * 0.2) + gaussian(0, 35) * fatigue + lateralBias * gaussian(0, 10);
|
|
136
|
+
|
|
137
|
+
// Fitts's law: duration ~ a + b*log2(2D/W), simplified to distance-based
|
|
138
|
+
const targetWidth = 20 + Math.random() * 80; // perceived click target size
|
|
139
|
+
const fittsDuration = (300 + 200 * Math.log2(2 * segDist / targetWidth)) * profile.speedMult * fatigue;
|
|
140
|
+
const steps = Math.max(30, Math.floor(segDist * 0.18 * profile.speedMult));
|
|
141
|
+
|
|
142
|
+
let stepTime = 0;
|
|
143
|
+
for (let i = 1; i <= steps; i++) {
|
|
144
|
+
const t = i / steps;
|
|
145
|
+
const speed = velocityProfile(t, profile.overshootTendency);
|
|
146
|
+
const stepMs = (fittsDuration / steps) / Math.max(speed, 0.05);
|
|
147
|
+
|
|
148
|
+
const mx = bezier(t, cx, cp1x, cp2x, etx);
|
|
149
|
+
const my = bezier(t, cy, cp1y, cp2y, ety);
|
|
150
|
+
|
|
151
|
+
// Layered noise: micro-tremor + physiological oscillation + drift bias
|
|
152
|
+
const tSec = (sessionTime + stepTime) / 1000;
|
|
153
|
+
const tx_noise = tremor(tSec, profile.tremFreq, profile.tremAmp * fatigue)
|
|
154
|
+
+ profile.driftBias.x * (1 - speed); // drift more when slow
|
|
155
|
+
const ty_noise = tremor(tSec + 0.37, profile.tremFreq * 0.93, profile.tremAmp * 0.7 * fatigue)
|
|
156
|
+
+ profile.driftBias.y * (1 - speed);
|
|
157
|
+
|
|
158
|
+
await page.mouse.move(mx + tx_noise, my + ty_noise);
|
|
159
|
+
stepTime += stepMs;
|
|
160
|
+
|
|
161
|
+
// Attention catch: sudden freeze when "something interesting" is spotted
|
|
162
|
+
if (Math.random() < 0.03) {
|
|
163
|
+
const freezeMs = 80 + Math.random() * 300;
|
|
164
|
+
// During freeze: very slow drift, not absolute stillness
|
|
165
|
+
const freezeSteps = Math.ceil(freezeMs / 16);
|
|
166
|
+
for (let f = 0; f < freezeSteps; f++) {
|
|
167
|
+
await page.mouse.move(
|
|
168
|
+
mx + tx_noise + gaussian(0, 0.2),
|
|
169
|
+
my + ty_noise + gaussian(0, 0.15)
|
|
170
|
+
);
|
|
171
|
+
await sleep(16);
|
|
172
|
+
}
|
|
173
|
+
stepTime += freezeMs;
|
|
174
|
+
} else {
|
|
175
|
+
await sleep(Math.max(1, stepMs));
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
cx = etx + gaussian(0, 1.5 * fatigue); // landing imprecision
|
|
180
|
+
cy = ety + gaussian(0, 1.5 * fatigue);
|
|
181
|
+
sessionTime += fittsDuration;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Aborted: redirect to new target after brief confusion pause
|
|
185
|
+
if (willAbort) {
|
|
186
|
+
await sleep(80 + Math.random() * 250);
|
|
187
|
+
// Brief backward micro-movement (second-guessing)
|
|
188
|
+
if (Math.random() < 0.5) {
|
|
189
|
+
await page.mouse.move(cx - gaussian(0, 15), cy - gaussian(0, 10));
|
|
190
|
+
await sleep(40 + Math.random() * 80);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Dwell: two-phase — initial landing jitter, then resting drift
|
|
195
|
+
const dwellMs = (150 + Math.random() * 1800) * profile.attentionSpan * fatigue;
|
|
196
|
+
const phase1 = dwellMs * 0.3; // landing stabilization
|
|
197
|
+
const phase2 = dwellMs * 0.7; // at-rest
|
|
198
|
+
|
|
199
|
+
// Phase 1: damped oscillation as hand settles (like underdamped spring)
|
|
200
|
+
const landingSteps = Math.ceil(phase1 / 16);
|
|
201
|
+
for (let d = 0; d < landingSteps; d++) {
|
|
202
|
+
const decay = Math.exp(-d / landingSteps * 4);
|
|
203
|
+
await page.mouse.move(
|
|
204
|
+
cx + gaussian(0, 1.5 * decay),
|
|
205
|
+
cy + gaussian(0, 1.2 * decay)
|
|
206
|
+
);
|
|
207
|
+
await sleep(16);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
// Phase 2: resting — very slow Brownian drift
|
|
211
|
+
const restSteps = Math.ceil(phase2 / 50);
|
|
212
|
+
let rx = cx, ry = cy;
|
|
213
|
+
for (let d = 0; d < restSteps; d++) {
|
|
214
|
+
rx += gaussian(0, 0.3);
|
|
215
|
+
ry += gaussian(0, 0.2);
|
|
216
|
+
// Slow mean-reversion: hand drifts but not far
|
|
217
|
+
rx += (cx - rx) * 0.05;
|
|
218
|
+
ry += (cy - ry) * 0.05;
|
|
219
|
+
await page.mouse.move(rx, ry);
|
|
220
|
+
await sleep(50);
|
|
221
|
+
}
|
|
222
|
+
cx = rx; cy = ry;
|
|
223
|
+
|
|
224
|
+
// Inter-movement gap: bimodal — short gap (quick scan) or long gap (reading)
|
|
225
|
+
const isReading = Math.random() < 0.35;
|
|
226
|
+
const gapMs = isReading
|
|
227
|
+
? 800 + Math.random() * 2500 // reading pause
|
|
228
|
+
: 80 + Math.random() * 350; // quick scan gap
|
|
229
|
+
await sleep(gapMs);
|
|
230
|
+
sessionTime += dwellMs + gapMs;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
14
234
|
/**
|
|
15
235
|
* @param {string} url
|
|
16
236
|
* @param {import('ora').Ora} spinner
|
|
17
237
|
* @param {import('playwright-core').Browser} browser
|
|
18
|
-
* @param {{ slow?: boolean, darkMode?: boolean, mobile?: boolean, wcag?: boolean, screenshotPath?: string, discoverLinks?: number|null, navigationTimeout?: number }} [options]
|
|
238
|
+
* @param {{ slow?: boolean, darkMode?: boolean, mobile?: boolean, wcag?: boolean, screenshotPath?: string, discoverLinks?: number|null, navigationTimeout?: number, stealth?: boolean, userAgent?: string, locale?: string, timezoneId?: string, acceptLanguage?: string, screenSize?: string }} [options]
|
|
19
239
|
* @returns {Promise<BrandingResult>}
|
|
20
240
|
*/
|
|
21
241
|
export async function extractBranding(url, spinner, browser, options = {}) {
|
|
22
242
|
const timeoutMultiplier = options.slow ? 3 : 1;
|
|
23
243
|
const timeouts = [];
|
|
24
244
|
|
|
25
|
-
spinner.text = "Creating browser context
|
|
245
|
+
spinner.text = "Creating browser context...";
|
|
246
|
+
|
|
247
|
+
const locale = options.locale || "en-US";
|
|
248
|
+
const timezoneId = options.timezoneId || "America/New_York";
|
|
249
|
+
const acceptLanguage = options.acceptLanguage || `${locale},${locale.split('-')[0]};q=0.9,en;q=0.8`;
|
|
250
|
+
|
|
251
|
+
const [screenW, screenH] = options.screenSize
|
|
252
|
+
? options.screenSize.split('x').map(Number)
|
|
253
|
+
: [1920, 1080];
|
|
254
|
+
|
|
26
255
|
const contextOptions = {
|
|
27
|
-
viewport: { width:
|
|
28
|
-
|
|
29
|
-
|
|
256
|
+
viewport: { width: screenW, height: screenH },
|
|
257
|
+
screen: { width: screenW, height: screenH },
|
|
258
|
+
userAgent: options.userAgent || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
|
259
|
+
locale,
|
|
260
|
+
timezoneId,
|
|
261
|
+
extraHTTPHeaders: { "Accept-Language": acceptLanguage },
|
|
262
|
+
colorScheme: "light",
|
|
30
263
|
};
|
|
31
264
|
|
|
32
265
|
if (browser.browserType().name() === 'chromium') {
|
|
@@ -35,21 +268,88 @@ export async function extractBranding(url, spinner, browser, options = {}) {
|
|
|
35
268
|
|
|
36
269
|
const context = await browser.newContext(contextOptions);
|
|
37
270
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
271
|
+
if (options.stealth) {
|
|
272
|
+
const stealthLocale = locale;
|
|
273
|
+
await context.addInitScript(({ loc, sw, sh }) => {
|
|
274
|
+
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
|
|
275
|
+
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
|
276
|
+
Object.defineProperty(navigator, 'platform', { get: () => 'Win32' });
|
|
277
|
+
Object.defineProperty(navigator, 'maxTouchPoints', { get: () => 0 });
|
|
278
|
+
Object.defineProperty(navigator, 'language', { get: () => loc });
|
|
279
|
+
Object.defineProperty(navigator, 'languages', { get: () => [loc, loc.split('-')[0]] });
|
|
280
|
+
Object.defineProperty(screen, 'width', { get: () => sw });
|
|
281
|
+
Object.defineProperty(screen, 'height', { get: () => sh });
|
|
282
|
+
Object.defineProperty(screen, 'availWidth', { get: () => sw });
|
|
283
|
+
Object.defineProperty(screen, 'availHeight', { get: () => sh - 40 }); // taskbar
|
|
284
|
+
Object.defineProperty(screen, 'colorDepth', { get: () => 24 });
|
|
285
|
+
Object.defineProperty(screen, 'pixelDepth', { get: () => 24 });
|
|
286
|
+
Object.defineProperty(window, 'devicePixelRatio', { get: () => 1 });
|
|
287
|
+
Object.defineProperty(window, 'outerWidth', { get: () => sw });
|
|
288
|
+
Object.defineProperty(window, 'outerHeight', { get: () => sh });
|
|
289
|
+
|
|
290
|
+
// plugins/mimeTypes: headless has none, real Chrome has several
|
|
291
|
+
const pluginData = [
|
|
292
|
+
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
|
|
293
|
+
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
|
|
294
|
+
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
|
|
295
|
+
];
|
|
296
|
+
const pluginArray = pluginData.map(p => {
|
|
297
|
+
const plugin = Object.create(Plugin.prototype);
|
|
298
|
+
Object.defineProperty(plugin, 'name', { get: () => p.name });
|
|
299
|
+
Object.defineProperty(plugin, 'filename', { get: () => p.filename });
|
|
300
|
+
Object.defineProperty(plugin, 'description', { get: () => p.description });
|
|
301
|
+
Object.defineProperty(plugin, 'length', { get: () => 0 });
|
|
302
|
+
return plugin;
|
|
303
|
+
});
|
|
304
|
+
Object.defineProperty(navigator, 'plugins', {
|
|
305
|
+
get: () => Object.assign(Object.create(PluginArray.prototype), pluginArray, { length: pluginArray.length }),
|
|
306
|
+
});
|
|
307
|
+
Object.defineProperty(navigator, 'mimeTypes', {
|
|
308
|
+
get: () => Object.assign(Object.create(MimeTypeArray.prototype), { length: 0 }),
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
// hasFocus: headless often returns false, real browser returns true
|
|
312
|
+
document.hasFocus = () => true;
|
|
313
|
+
|
|
314
|
+
// connection: expose a plausible NetworkInformation object
|
|
315
|
+
if (!navigator.connection) {
|
|
316
|
+
Object.defineProperty(navigator, 'connection', {
|
|
317
|
+
get: () => ({ effectiveType: '4g', rtt: 50, downlink: 10, saveData: false }),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// history.length: fresh context always 1 — nudge to plausible value
|
|
322
|
+
try {
|
|
323
|
+
Object.defineProperty(history, 'length', { get: () => 2 + Math.floor(Math.random() * 4) });
|
|
324
|
+
} catch {}
|
|
325
|
+
|
|
326
|
+
window.chrome = { runtime: {}, loadTimes: () => {}, csi: () => {}, app: {} };
|
|
327
|
+
delete navigator.__proto__.webdriver;
|
|
328
|
+
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
|
|
329
|
+
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
|
|
330
|
+
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
|
|
331
|
+
}, { loc: stealthLocale, sw: screenW, sh: screenH });
|
|
332
|
+
}
|
|
50
333
|
|
|
51
334
|
const page = await context.newPage();
|
|
52
335
|
|
|
336
|
+
// Track font requests to identify self-hosted custom fonts
|
|
337
|
+
const fontRequests = new Set();
|
|
338
|
+
const thirdPartyFontHosts = ['fonts.googleapis.com', 'fonts.gstatic.com', 'typekit.net',
|
|
339
|
+
'adobe.com', 'fonts.com', 'cloud.typography.com', 'fast.fonts.net', 'use.fontawesome.com',
|
|
340
|
+
'kit.fontawesome.com', 'pro.fontawesome.com'];
|
|
341
|
+
page.on('response', (response) => {
|
|
342
|
+
const resUrl = response.url();
|
|
343
|
+
const ct = response.headers()['content-type'] || '';
|
|
344
|
+
if (ct.includes('font') || resUrl.match(/\.(woff2?|ttf|otf|eot)(\?|$)/i)) {
|
|
345
|
+
try {
|
|
346
|
+
const host = new URL(resUrl).hostname;
|
|
347
|
+
const isThirdParty = thirdPartyFontHosts.some(h => host.includes(h));
|
|
348
|
+
if (!isThirdParty) fontRequests.add(resUrl);
|
|
349
|
+
} catch {}
|
|
350
|
+
}
|
|
351
|
+
});
|
|
352
|
+
|
|
53
353
|
try {
|
|
54
354
|
let attempts = 0;
|
|
55
355
|
const maxAttempts = 2;
|
|
@@ -117,23 +417,66 @@ export async function extractBranding(url, spinner, browser, options = {}) {
|
|
|
117
417
|
timeouts.push('Main content selector');
|
|
118
418
|
}
|
|
119
419
|
|
|
120
|
-
|
|
121
|
-
|
|
420
|
+
if (options.stealth) {
|
|
421
|
+
await simulateHumanMouse(page);
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
spinner.start("Scrolling page to trigger lazy content...");
|
|
122
425
|
await page.evaluate(async () => {
|
|
123
|
-
const delay = (ms) => new Promise(r => setTimeout(r, ms));
|
|
124
426
|
const scrollStep = 600;
|
|
125
427
|
const maxHeight = Math.min(document.body.scrollHeight, 30000);
|
|
126
428
|
let y = 0;
|
|
127
429
|
while (y < maxHeight) {
|
|
128
430
|
y = Math.min(y + scrollStep, maxHeight);
|
|
129
431
|
window.scrollTo(0, y);
|
|
130
|
-
await
|
|
432
|
+
await new Promise(r => setTimeout(r, 100));
|
|
131
433
|
}
|
|
132
434
|
window.scrollTo(0, 0);
|
|
133
435
|
});
|
|
134
436
|
spinner.stop();
|
|
135
437
|
console.log(color.success(` ✓ Full page scrolled (lazy content triggered)`));
|
|
136
438
|
|
|
439
|
+
spinner.start("Dismissing cookie consent banners...");
|
|
440
|
+
const dismissed = await page.evaluate(async () => {
|
|
441
|
+
const selectors = [
|
|
442
|
+
// Generic accept patterns
|
|
443
|
+
'button[id*="accept"]', 'button[class*="accept"]',
|
|
444
|
+
'button[id*="agree"]', 'button[class*="agree"]',
|
|
445
|
+
'button[id*="consent"]', 'button[class*="consent"]',
|
|
446
|
+
'[data-testid*="accept"]', '[data-testid*="agree"]',
|
|
447
|
+
// Common consent libraries
|
|
448
|
+
'#onetrust-accept-btn-handler',
|
|
449
|
+
'.cc-btn.cc-allow', '.cc-accept',
|
|
450
|
+
'[aria-label*="Accept"]', '[aria-label*="agree"]',
|
|
451
|
+
// EU/GDPR common patterns
|
|
452
|
+
'button[data-cookiebanner]',
|
|
453
|
+
'.cookiebanner button', '#cookiebanner button',
|
|
454
|
+
'[class*="cookie"] button[class*="primary"]',
|
|
455
|
+
'[id*="cookie"] button[class*="primary"]',
|
|
456
|
+
'[class*="gdpr"] button', '[id*="gdpr"] button',
|
|
457
|
+
// CMP patterns
|
|
458
|
+
'.sp-message-open .message-button',
|
|
459
|
+
'#sp-cc-accept', '.optanon-allow-all',
|
|
460
|
+
];
|
|
461
|
+
for (const sel of selectors) {
|
|
462
|
+
try {
|
|
463
|
+
const el = document.querySelector(sel);
|
|
464
|
+
if (el && el.offsetParent !== null) {
|
|
465
|
+
el.click();
|
|
466
|
+
return sel;
|
|
467
|
+
}
|
|
468
|
+
} catch {}
|
|
469
|
+
}
|
|
470
|
+
return null;
|
|
471
|
+
});
|
|
472
|
+
spinner.stop();
|
|
473
|
+
if (dismissed) {
|
|
474
|
+
console.log(color.success(` ✓ Cookie banner dismissed (${dismissed})`));
|
|
475
|
+
await page.waitForTimeout(600);
|
|
476
|
+
} else {
|
|
477
|
+
console.log(color.info(` i No cookie banner detected`));
|
|
478
|
+
}
|
|
479
|
+
|
|
137
480
|
spinner.start("Final content stabilization...");
|
|
138
481
|
const stabilizationTime = 4000 * timeoutMultiplier;
|
|
139
482
|
await page.waitForTimeout(stabilizationTime);
|
|
@@ -188,23 +531,23 @@ export async function extractBranding(url, spinner, browser, options = {}) {
|
|
|
188
531
|
gradients,
|
|
189
532
|
motion,
|
|
190
533
|
] = await Promise.all([
|
|
191
|
-
extractLogo(page, url),
|
|
192
|
-
extractColors(page),
|
|
193
|
-
extractTypography(page),
|
|
194
|
-
extractSpacing(page),
|
|
195
|
-
extractBorderRadius(page),
|
|
196
|
-
extractBorders(page),
|
|
197
|
-
extractShadows(page),
|
|
198
|
-
extractButtonStyles(page),
|
|
199
|
-
extractInputStyles(page),
|
|
200
|
-
extractLinkStyles(page),
|
|
201
|
-
extractBadgeStyles(page),
|
|
202
|
-
extractBreakpoints(page),
|
|
203
|
-
detectIconSystem(page),
|
|
204
|
-
detectFrameworks(page),
|
|
205
|
-
extractSiteName(page),
|
|
206
|
-
extractGradients(page),
|
|
207
|
-
extractMotion(page),
|
|
534
|
+
extractLogo(page, url).catch(() => ({ logo: null, instances: [], favicons: [], manifest: null })),
|
|
535
|
+
extractColors(page).catch(() => ({ semantic: {}, palette: [], cssVariables: [], _raw: [] })),
|
|
536
|
+
extractTypography(page).catch(() => ({ styles: [], sources: {} })),
|
|
537
|
+
extractSpacing(page).catch(() => ({ scaleType: 'unknown', commonValues: [] })),
|
|
538
|
+
extractBorderRadius(page).catch(() => ({ values: [] })),
|
|
539
|
+
extractBorders(page).catch(() => ({ combinations: [] })),
|
|
540
|
+
extractShadows(page).catch(() => []),
|
|
541
|
+
extractButtonStyles(page).catch(() => []),
|
|
542
|
+
extractInputStyles(page).catch(() => []),
|
|
543
|
+
extractLinkStyles(page).catch(() => []),
|
|
544
|
+
extractBadgeStyles(page).catch(() => ({ all: [], byVariant: {} })),
|
|
545
|
+
extractBreakpoints(page).catch(() => []),
|
|
546
|
+
detectIconSystem(page).catch(() => []),
|
|
547
|
+
detectFrameworks(page).catch(() => []),
|
|
548
|
+
extractSiteName(page).catch(() => null),
|
|
549
|
+
extractGradients(page).catch(() => []),
|
|
550
|
+
extractMotion(page).catch(() => ({ durations: [], easings: [], byContext: {} })),
|
|
208
551
|
]);
|
|
209
552
|
|
|
210
553
|
const { logo, instances: logoInstances, favicons, manifest } = logoResult;
|
|
@@ -606,13 +949,36 @@ export async function extractBranding(url, spinner, browser, options = {}) {
|
|
|
606
949
|
const result = {
|
|
607
950
|
url: page.url(),
|
|
608
951
|
extractedAt: new Date().toISOString(),
|
|
952
|
+
meta: {
|
|
953
|
+
dembrandtVersion: options._version || null,
|
|
954
|
+
flags: {
|
|
955
|
+
...(options.stealth && { stealth: true }),
|
|
956
|
+
...(options.darkMode && { darkMode: true }),
|
|
957
|
+
...(options.mobile && { mobile: true }),
|
|
958
|
+
...(options.slow && { slow: true }),
|
|
959
|
+
...(options.userAgent && { userAgent: options.userAgent }),
|
|
960
|
+
...(options.locale && { locale: options.locale }),
|
|
961
|
+
...(options.timezoneId && { timezone: options.timezoneId }),
|
|
962
|
+
...(options.acceptLanguage && { acceptLanguage: options.acceptLanguage }),
|
|
963
|
+
...(options.screenSize && { screenSize: options.screenSize }),
|
|
964
|
+
},
|
|
965
|
+
},
|
|
609
966
|
siteName,
|
|
610
967
|
logo,
|
|
611
968
|
logoInstances,
|
|
612
969
|
favicons,
|
|
613
970
|
...(manifest ? { manifest } : {}),
|
|
614
971
|
colors,
|
|
615
|
-
typography
|
|
972
|
+
typography: {
|
|
973
|
+
...typography,
|
|
974
|
+
sources: {
|
|
975
|
+
...typography.sources,
|
|
976
|
+
selfHostedFonts: [...fontRequests].map(u => u.split('/').pop().split('?')[0]),
|
|
977
|
+
customFonts: typography.sources?.customFonts?.length
|
|
978
|
+
? typography.sources.customFonts
|
|
979
|
+
: fontRequests.size > 0 ? [...fontRequests].map(u => u.split('/').pop().split('?')[0]) : [],
|
|
980
|
+
}
|
|
981
|
+
},
|
|
616
982
|
spacing,
|
|
617
983
|
borderRadius,
|
|
618
984
|
borders,
|
|
@@ -32,6 +32,38 @@ export async function extractTypography(page) {
|
|
|
32
32
|
sources.adobeFonts = true;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
// Detect truly custom fonts: @font-face rules loading from own domain
|
|
36
|
+
const thirdPartyHosts = ['googleapis.com', 'gstatic.com', 'typekit.net', 'adobe.com',
|
|
37
|
+
'fonts.com', 'cloud.typography.com', 'fast.fonts.net', 'use.fontawesome.com'];
|
|
38
|
+
const pageHost = window.location.hostname;
|
|
39
|
+
try {
|
|
40
|
+
for (const sheet of document.styleSheets) {
|
|
41
|
+
try {
|
|
42
|
+
for (const rule of sheet.cssRules || []) {
|
|
43
|
+
if (rule instanceof CSSFontFaceRule) {
|
|
44
|
+
const src = rule.style.getPropertyValue('src') || '';
|
|
45
|
+
const family = (rule.style.getPropertyValue('font-family') || '').replace(/['"]/g, '').trim();
|
|
46
|
+
if (!family) continue;
|
|
47
|
+
const isThirdParty = thirdPartyHosts.some(h => src.includes(h));
|
|
48
|
+
const isSameOrigin = src.includes(pageHost) || src.startsWith('/') || src.startsWith('./') || (!src.includes('http') && src.includes('url('));
|
|
49
|
+
if (!isThirdParty && (isSameOrigin || src.includes('url(')) && !sources.customFonts.includes(family)) {
|
|
50
|
+
sources.customFonts.push(family);
|
|
51
|
+
}
|
|
52
|
+
const familyLower = family.toLowerCase();
|
|
53
|
+
if (
|
|
54
|
+
familyLower.includes('variable') ||
|
|
55
|
+
familyLower.includes(' vf') ||
|
|
56
|
+
familyLower.endsWith('-var') ||
|
|
57
|
+
(src.includes('woff2') && rule.style.getPropertyValue('font-variation-settings'))
|
|
58
|
+
) {
|
|
59
|
+
sources.variableFonts.add(family);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
} catch {}
|
|
64
|
+
}
|
|
65
|
+
} catch {}
|
|
66
|
+
|
|
35
67
|
let fontDisplay = null;
|
|
36
68
|
try {
|
|
37
69
|
for (const sheet of document.styleSheets) {
|
package/lib/merger.js
CHANGED
package/mcp-server.js
CHANGED
|
@@ -15,7 +15,7 @@ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"
|
|
|
15
15
|
import { z } from "zod";
|
|
16
16
|
import { chromium } from "playwright-core";
|
|
17
17
|
import { readFileSync } from "node:fs";
|
|
18
|
-
import { extractBranding } from "./lib/extractors.js";
|
|
18
|
+
import { extractBranding } from "./lib/extractors/index.js";
|
|
19
19
|
|
|
20
20
|
const { version } = JSON.parse(readFileSync(new URL("./package.json", import.meta.url), "utf8"));
|
|
21
21
|
|