crawlforge-mcp-server 6.24.0 → 6.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/server.js +2 -2
- package/src/core/ActionExecutor.js +14 -0
- package/src/schemas/toolOutputSchemas.js +1 -1
- package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-brand-style-guide/SKILL.md +107 -0
- package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md +2 -1
- package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md +1 -0
- package/src/tools/scrape/_brandingColor.js +185 -0
- package/src/tools/scrape/_brandingComputed.js +262 -0
- package/src/tools/scrape/_brandingExtractor.js +62 -90
- package/src/tools/scrape/_brandingTokens.js +350 -0
- package/src/tools/scrape/unifiedScrape.js +57 -16
- package/src/tools/search/adapters/dataforseoSearch.js +70 -8
- package/src/tools/search/queryExpander.js +16 -1
- package/src/tools/search/serpRank.js +22 -9
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* _brandingComputed.js — colour roles, buttons and type from the rendered page.
|
|
3
|
+
*
|
|
4
|
+
* brandingProbe runs in the browser and only COLLECTS computed
|
|
5
|
+
* styles: the page background, body text, h1/h2, a
|
|
6
|
+
* few links and the button-sized controls near the
|
|
7
|
+
* top of the page. page.evaluate serializes it, so it
|
|
8
|
+
* must not close over anything in this module.
|
|
9
|
+
* buildComputedBranding decides, in Node where it can be tested, which
|
|
10
|
+
* control is the primary button and what each role
|
|
11
|
+
* is. Every value it returns is source:"computed".
|
|
12
|
+
* inferRoles the fallback when no render ran: role guesses from
|
|
13
|
+
* CSS variable names, marked source:"inferred".
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { parseColor, toHex, chroma, luminance, distance } from './_brandingColor.js';
|
|
17
|
+
|
|
18
|
+
export function brandingProbe() {
|
|
19
|
+
const canvas = document.createElement('canvas');
|
|
20
|
+
canvas.width = canvas.height = 1;
|
|
21
|
+
const ctx = canvas.getContext('2d', { willReadFrequently: true });
|
|
22
|
+
// Any computed colour (oklch, color(), lab) to rgba() through one canvas pixel.
|
|
23
|
+
const rgba = (c) => {
|
|
24
|
+
ctx.clearRect(0, 0, 1, 1);
|
|
25
|
+
ctx.fillStyle = 'rgba(0,0,0,0)';
|
|
26
|
+
ctx.fillStyle = c || 'rgba(0,0,0,0)';
|
|
27
|
+
ctx.fillRect(0, 0, 1, 1);
|
|
28
|
+
const [r, g, b, a] = ctx.getImageData(0, 0, 1, 1).data;
|
|
29
|
+
return `rgba(${r},${g},${b},${Math.round((a / 255) * 1000) / 1000})`;
|
|
30
|
+
};
|
|
31
|
+
// The colour a reader sees is the fill of the element holding the text. On
|
|
32
|
+
// a layered headline that is a child span, not the h1: stripe.com's h1
|
|
33
|
+
// carries a decorative green while its first line is painted #061b31.
|
|
34
|
+
const textColor = (el) => {
|
|
35
|
+
const walker = document.createTreeWalker(el, NodeFilter.SHOW_TEXT, {
|
|
36
|
+
acceptNode: (n) => (n.textContent.trim() ? NodeFilter.FILTER_ACCEPT : NodeFilter.FILTER_SKIP),
|
|
37
|
+
});
|
|
38
|
+
const holder = walker.nextNode()?.parentElement || el;
|
|
39
|
+
const s = getComputedStyle(holder);
|
|
40
|
+
return rgba(s.webkitTextFillColor || s.color);
|
|
41
|
+
};
|
|
42
|
+
const style = (el) => {
|
|
43
|
+
const s = getComputedStyle(el);
|
|
44
|
+
return {
|
|
45
|
+
color: textColor(el),
|
|
46
|
+
background: rgba(s.backgroundColor),
|
|
47
|
+
borderColor: rgba(s.borderTopColor),
|
|
48
|
+
borderWidth: s.borderTopWidth,
|
|
49
|
+
borderStyle: s.borderTopStyle,
|
|
50
|
+
borderRadius: s.borderTopLeftRadius,
|
|
51
|
+
padding: `${s.paddingTop} ${s.paddingRight} ${s.paddingBottom} ${s.paddingLeft}`,
|
|
52
|
+
fontFamily: s.fontFamily,
|
|
53
|
+
fontSize: s.fontSize,
|
|
54
|
+
fontWeight: s.fontWeight,
|
|
55
|
+
lineHeight: s.lineHeight,
|
|
56
|
+
letterSpacing: s.letterSpacing,
|
|
57
|
+
textTransform: s.textTransform,
|
|
58
|
+
boxShadow: s.boxShadow,
|
|
59
|
+
};
|
|
60
|
+
};
|
|
61
|
+
const OVERLAY = '[id*="cookie" i],[class*="cookie" i],[id*="consent" i],[class*="consent" i],[aria-modal="true"],dialog';
|
|
62
|
+
// Painted, on the page and not a 1px screen-reader-only box (a "Skip to
|
|
63
|
+
// content" link sits off-screen until focused).
|
|
64
|
+
const shown = (el) => {
|
|
65
|
+
const r = el.getBoundingClientRect();
|
|
66
|
+
const s = getComputedStyle(el);
|
|
67
|
+
return r.width > 1 && r.height > 1 && r.right > 0 && r.bottom > 0 && s.visibility !== 'hidden' &&
|
|
68
|
+
s.display !== 'none' && Number(s.opacity) > 0 && !el.closest(OVERLAY);
|
|
69
|
+
};
|
|
70
|
+
const text = (el) => (el.innerText || el.value || '').replace(/\s+/g, ' ').trim().slice(0, 60);
|
|
71
|
+
const find = (selector, ok = () => true) => {
|
|
72
|
+
for (const el of document.querySelectorAll(selector)) if (shown(el) && ok(el)) return el;
|
|
73
|
+
return null;
|
|
74
|
+
};
|
|
75
|
+
const describe = (el) => (el ? { tag: el.tagName.toLowerCase(), text: text(el), style: style(el) } : null);
|
|
76
|
+
|
|
77
|
+
const collect = () => {
|
|
78
|
+
const links = [];
|
|
79
|
+
for (const el of document.querySelectorAll('p a, main a, a')) {
|
|
80
|
+
if (links.length >= 6) break;
|
|
81
|
+
if (shown(el) && text(el).length > 1) links.push(describe(el));
|
|
82
|
+
}
|
|
83
|
+
const buttons = [];
|
|
84
|
+
const limit = window.innerHeight * 1.5;
|
|
85
|
+
for (const el of document.querySelectorAll('button, a, [role="button"], input[type="submit"], input[type="button"]')) {
|
|
86
|
+
if (buttons.length >= 40) break;
|
|
87
|
+
const r = el.getBoundingClientRect();
|
|
88
|
+
const t = text(el);
|
|
89
|
+
if (r.top > limit || r.height < 24 || r.height > 96 || r.width < 40 || r.width > 480 || !t || t.length > 40) continue;
|
|
90
|
+
if (!shown(el)) continue;
|
|
91
|
+
buttons.push({
|
|
92
|
+
...describe(el),
|
|
93
|
+
rect: { x: Math.round(r.x), y: Math.round(r.y), w: Math.round(r.width), h: Math.round(r.height) },
|
|
94
|
+
inHeader: Boolean(el.closest('header, nav')),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return {
|
|
98
|
+
url: location.href,
|
|
99
|
+
viewport: { w: window.innerWidth, h: window.innerHeight },
|
|
100
|
+
background: { body: rgba(getComputedStyle(document.body).backgroundColor), html: rgba(getComputedStyle(document.documentElement).backgroundColor) },
|
|
101
|
+
body: describe(document.body),
|
|
102
|
+
heading: describe(find('h1')),
|
|
103
|
+
h2: describe(find('h2')),
|
|
104
|
+
// Body copy: a sentence-length paragraph at a reading size, so a large
|
|
105
|
+
// lede set in a <p> is not taken for the body text.
|
|
106
|
+
paragraph: describe(find('main p, article p, p', (el) => {
|
|
107
|
+
const size = parseFloat(getComputedStyle(el).fontSize);
|
|
108
|
+
return text(el).length >= 40 && size >= 14 && size <= 20;
|
|
109
|
+
}) || find('main p, article p, p', (el) => text(el).length >= 40)),
|
|
110
|
+
links,
|
|
111
|
+
buttons,
|
|
112
|
+
};
|
|
113
|
+
};
|
|
114
|
+
// Web fonts change the computed family only once loaded; wait at most 2s.
|
|
115
|
+
return Promise.race([document.fonts.ready, new Promise((resolve) => setTimeout(resolve, 2000))]).then(collect, collect);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const WHITE = { r: 255, g: 255, b: 255, a: 1 };
|
|
119
|
+
// Two colours closer than this read as the same to a reader.
|
|
120
|
+
const SAME = 24;
|
|
121
|
+
|
|
122
|
+
const opaque = (v) => {
|
|
123
|
+
const c = parseColor(v);
|
|
124
|
+
return c && c.a > 0.05 ? c : null;
|
|
125
|
+
};
|
|
126
|
+
const firstFamily = (stack) => (stack || '').split(',')[0].trim().replace(/^['"]|['"]$/g, '') || null;
|
|
127
|
+
|
|
128
|
+
function compact(obj) {
|
|
129
|
+
return Object.fromEntries(Object.entries(obj).filter(([, v]) => v != null && v !== '' && v !== 'none'));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Colour roles, button components and type from a brandingProbe() result.
|
|
134
|
+
* @param {object} probe
|
|
135
|
+
* @returns {{ colorScheme: string, roles: object, components: object, typography: object }}
|
|
136
|
+
*/
|
|
137
|
+
export function buildComputedBranding(probe) {
|
|
138
|
+
const pageBg = opaque(probe.background?.body) || opaque(probe.background?.html) || WHITE;
|
|
139
|
+
const filled = (s) => {
|
|
140
|
+
const bg = opaque(s.background);
|
|
141
|
+
return Boolean(bg && distance(bg, pageBg) > SAME);
|
|
142
|
+
};
|
|
143
|
+
const outlined = (s) => {
|
|
144
|
+
const border = opaque(s.borderColor);
|
|
145
|
+
return parseFloat(s.borderWidth) >= 1 && s.borderStyle !== 'none' && Boolean(border && distance(border, pageBg) > SAME);
|
|
146
|
+
};
|
|
147
|
+
|
|
148
|
+
const roles = {};
|
|
149
|
+
const role = (name, c, evidence) => {
|
|
150
|
+
if (c) roles[name] = { value: toHex(c), source: 'computed', evidence };
|
|
151
|
+
};
|
|
152
|
+
role('background', pageBg, 'page background');
|
|
153
|
+
const textEl = probe.paragraph || probe.body;
|
|
154
|
+
role('text', textEl && opaque(textEl.style.color), probe.paragraph ? 'body paragraph' : 'body');
|
|
155
|
+
role('heading', probe.heading && opaque(probe.heading.style.color), 'h1');
|
|
156
|
+
const link = (probe.links || []).find((l) => !filled(l.style) && !outlined(l.style));
|
|
157
|
+
role('link', link && opaque(link.style.color), link && `link "${link.text}"`);
|
|
158
|
+
|
|
159
|
+
// The primary button is the filled control with the most visual weight:
|
|
160
|
+
// area, boosted by saturation so a brand-coloured CTA beats a grey one.
|
|
161
|
+
const buttons = (probe.buttons || []).map((b, i) => ({ ...b, i, bg: opaque(b.style.background) }));
|
|
162
|
+
const filledButtons = buttons.filter((b) => filled(b.style));
|
|
163
|
+
const weight = (b) => b.rect.w * b.rect.h * (1 + 2 * chroma(b.bg));
|
|
164
|
+
const primary = [...filledButtons].sort((x, y) => weight(y) - weight(x) || x.i - y.i)[0] || null;
|
|
165
|
+
// The secondary is the outline button nearest the primary and in the same
|
|
166
|
+
// band (160px vertically), which keeps it in the CTA pair: linear.app has
|
|
167
|
+
// no partner for its header CTA, and its nearest outline is an "iOS" pill.
|
|
168
|
+
const gap = (b) => Math.hypot(b.rect.x - primary.rect.x, b.rect.y - primary.rect.y);
|
|
169
|
+
const near = (b) => b !== primary && Math.abs(b.rect.y - primary.rect.y) <= 160;
|
|
170
|
+
const secondary = primary
|
|
171
|
+
? buttons.filter((b) => near(b) && !filled(b.style) && outlined(b.style)).sort((x, y) => gap(x) - gap(y))[0] ||
|
|
172
|
+
filledButtons.find((b) => near(b) && distance(b.bg, primary.bg) > SAME) ||
|
|
173
|
+
null
|
|
174
|
+
: null;
|
|
175
|
+
if (primary) {
|
|
176
|
+
role('primary', primary.bg, `button "${primary.text}"`);
|
|
177
|
+
role('onPrimary', opaque(primary.style.color), `label of button "${primary.text}"`);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const component = (b) => compact({
|
|
181
|
+
text: b.text,
|
|
182
|
+
background: filled(b.style) ? toHex(b.bg) : null,
|
|
183
|
+
color: opaque(b.style.color) ? toHex(opaque(b.style.color)) : null,
|
|
184
|
+
borderColor: outlined(b.style) ? toHex(opaque(b.style.borderColor)) : null,
|
|
185
|
+
borderWidth: outlined(b.style) ? b.style.borderWidth : null,
|
|
186
|
+
// A pill computes to whatever its huge radius was (calc(infinity * 1px) is 3.35544e+07px).
|
|
187
|
+
borderRadius: b.style.borderRadius === '0px' ? null : parseFloat(b.style.borderRadius) >= 9999 ? '9999px' : b.style.borderRadius,
|
|
188
|
+
padding: b.style.padding,
|
|
189
|
+
fontFamily: firstFamily(b.style.fontFamily),
|
|
190
|
+
fontSize: b.style.fontSize,
|
|
191
|
+
fontWeight: b.style.fontWeight,
|
|
192
|
+
textTransform: b.style.textTransform,
|
|
193
|
+
boxShadow: b.style.boxShadow,
|
|
194
|
+
source: 'computed',
|
|
195
|
+
});
|
|
196
|
+
const components = {};
|
|
197
|
+
if (primary) components.buttonPrimary = component(primary);
|
|
198
|
+
if (secondary) components.buttonSecondary = component(secondary);
|
|
199
|
+
|
|
200
|
+
const type = (el) => el && compact({
|
|
201
|
+
fontFamily: firstFamily(el.style.fontFamily),
|
|
202
|
+
fontStack: el.style.fontFamily,
|
|
203
|
+
fontSize: el.style.fontSize,
|
|
204
|
+
fontWeight: el.style.fontWeight,
|
|
205
|
+
lineHeight: el.style.lineHeight,
|
|
206
|
+
letterSpacing: el.style.letterSpacing === 'normal' ? null : el.style.letterSpacing,
|
|
207
|
+
color: opaque(el.style.color) ? toHex(opaque(el.style.color)) : null,
|
|
208
|
+
source: 'computed',
|
|
209
|
+
});
|
|
210
|
+
const typography = compact({ body: type(textEl), h1: type(probe.heading), h2: type(probe.h2) });
|
|
211
|
+
|
|
212
|
+
return {
|
|
213
|
+
colorScheme: schemeOf(toHex(pageBg)),
|
|
214
|
+
roles,
|
|
215
|
+
components,
|
|
216
|
+
typography,
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Role guesses from CSS variable names, for when the page was not rendered.
|
|
222
|
+
* A name must end in the role word (optionally -default/-primary/-base), so
|
|
223
|
+
* --color-background counts and --input-bg-hover does not.
|
|
224
|
+
* @param {Record<string, string|null>} resolved - '--name' -> resolved value
|
|
225
|
+
* @param {Array<{value: string, source: string}>} colors - the frequency list
|
|
226
|
+
*/
|
|
227
|
+
export function inferRoles(resolved, colors = []) {
|
|
228
|
+
const roles = {};
|
|
229
|
+
const pick = (re, ok = () => true) => {
|
|
230
|
+
for (const [name, value] of Object.entries(resolved)) {
|
|
231
|
+
if (value == null || !re.test(name)) continue;
|
|
232
|
+
const c = parseColor(value);
|
|
233
|
+
if (c && c.a > 0.5 && ok(c)) return { value: toHex(c), source: 'inferred', evidence: `CSS variable ${name}` };
|
|
234
|
+
}
|
|
235
|
+
return null;
|
|
236
|
+
};
|
|
237
|
+
const end = (words) => new RegExp(`(?:^--|-)(?:${words})(?:-(?:default|primary|base|500|600))?$`, 'i');
|
|
238
|
+
const saturated = (c) => chroma(c) >= 0.25;
|
|
239
|
+
|
|
240
|
+
const fromList = (source, evidence) => {
|
|
241
|
+
const hit = colors.find((c) => c.source === source && saturated(parseColor(c.value)));
|
|
242
|
+
return hit ? { value: hit.value, source: 'inferred', evidence } : null;
|
|
243
|
+
};
|
|
244
|
+
const primary = pick(end('primary|brand|accent'), saturated) ||
|
|
245
|
+
fromList('theme-color', 'meta theme-color') ||
|
|
246
|
+
fromList('css-var', 'most frequent saturated colour in brand-named CSS variables');
|
|
247
|
+
const background = pick(end('background|bg|surface'));
|
|
248
|
+
const text = pick(end('foreground|fg|text'));
|
|
249
|
+
const link = pick(end('link'));
|
|
250
|
+
|
|
251
|
+
if (background) roles.background = background;
|
|
252
|
+
if (text) roles.text = text;
|
|
253
|
+
if (link) roles.link = link;
|
|
254
|
+
if (primary) roles.primary = primary;
|
|
255
|
+
return roles;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** 'light' or 'dark' from a background role, or null without one. */
|
|
259
|
+
export function schemeOf(backgroundHex) {
|
|
260
|
+
const c = parseColor(backgroundHex);
|
|
261
|
+
return c ? (luminance(c) < 0.4 ? 'dark' : 'light') : null;
|
|
262
|
+
}
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* _brandingExtractor.js —
|
|
2
|
+
* _brandingExtractor.js — design-token / branding extraction.
|
|
3
3
|
*
|
|
4
|
-
* Extracts a site's visual identity from HTML + CSS
|
|
4
|
+
* Extracts a site's visual identity from HTML + CSS:
|
|
5
5
|
* - colors (CSS custom properties, theme-color, declarations) -> hex + frequency
|
|
6
6
|
* - fonts (font-family, @font-face, Google/Adobe webfont links)
|
|
7
7
|
* - logo (favicons, og:image, header img/svg candidates)
|
|
8
|
-
* - tokens (
|
|
8
|
+
* - tokens (CSS custom properties, var() chains resolved, grouped; _brandingTokens.js)
|
|
9
|
+
* - roles (background, text, heading, link, primary) plus button
|
|
10
|
+
* components and type, from a rendered page's computed styles
|
|
11
|
+
* when the caller passes a probe result (_brandingComputed.js),
|
|
12
|
+
* otherwise guessed from variable names and marked "inferred"
|
|
9
13
|
*
|
|
10
|
-
*
|
|
11
|
-
* styles and JS-injected CSS are not visible without rendering. Linked CSS is
|
|
14
|
+
* Without a probe result, JS-injected CSS is not visible. Linked CSS is
|
|
12
15
|
* fetched through the SSRF guard, capped in count/size/time.
|
|
13
16
|
*
|
|
14
17
|
* Every helper is defensive: failures push to `warnings` and never throw.
|
|
@@ -16,6 +19,9 @@
|
|
|
16
19
|
|
|
17
20
|
import { safeFetch } from '../../utils/ssrfGuard.js';
|
|
18
21
|
import { mediaAppliesToScreen } from '../../utils/hiddenContent.js';
|
|
22
|
+
import { parseColor, toHex } from './_brandingColor.js';
|
|
23
|
+
import { resolveValue, groupTokens, toDtcg } from './_brandingTokens.js';
|
|
24
|
+
import { buildComputedBranding, inferRoles, schemeOf } from './_brandingComputed.js';
|
|
19
25
|
|
|
20
26
|
const GENERIC_FAMILIES = new Set([
|
|
21
27
|
'serif', 'sans-serif', 'monospace', 'cursive', 'fantasy', 'system-ui',
|
|
@@ -23,14 +29,6 @@ const GENERIC_FAMILIES = new Set([
|
|
|
23
29
|
'initial', 'unset', 'revert', 'emoji', 'math', 'fangsong',
|
|
24
30
|
]);
|
|
25
31
|
|
|
26
|
-
const NAMED_COLORS = {
|
|
27
|
-
black: '#000000', white: '#ffffff', red: '#ff0000', green: '#008000',
|
|
28
|
-
blue: '#0000ff', yellow: '#ffff00', orange: '#ffa500', purple: '#800080',
|
|
29
|
-
gray: '#808080', grey: '#808080', silver: '#c0c0c0', navy: '#000080',
|
|
30
|
-
teal: '#008080', maroon: '#800000', olive: '#808000', lime: '#00ff00',
|
|
31
|
-
aqua: '#00ffff', fuchsia: '#ff00ff',
|
|
32
|
-
};
|
|
33
|
-
|
|
34
32
|
// CSS variable names that strongly indicate brand colors.
|
|
35
33
|
const BRAND_VAR_RE = /(primary|secondary|brand|accent|theme|background|surface|foreground|text|link)/i;
|
|
36
34
|
|
|
@@ -38,71 +36,13 @@ function clamp(n, lo, hi) {
|
|
|
38
36
|
return Math.max(lo, Math.min(hi, n));
|
|
39
37
|
}
|
|
40
38
|
|
|
41
|
-
function toHex2(n) {
|
|
42
|
-
return clamp(Math.round(n), 0, 255).toString(16).padStart(2, '0');
|
|
43
|
-
}
|
|
44
|
-
|
|
45
39
|
/**
|
|
46
40
|
* Normalize a CSS color token to lowercase #rrggbb (drops alpha). Returns null
|
|
47
41
|
* for values we can't confidently parse (gradients, currentColor, var(), etc.).
|
|
48
42
|
*/
|
|
49
43
|
export function normalizeColor(raw) {
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
if (NAMED_COLORS[v]) return NAMED_COLORS[v];
|
|
54
|
-
|
|
55
|
-
// hex
|
|
56
|
-
let m = v.match(/^#([0-9a-f]{3,8})$/);
|
|
57
|
-
if (m) {
|
|
58
|
-
const h = m[1];
|
|
59
|
-
if (h.length === 3) return '#' + h.split('').map((c) => c + c).join('');
|
|
60
|
-
if (h.length === 4) return '#' + h.slice(0, 3).split('').map((c) => c + c).join('');
|
|
61
|
-
if (h.length === 6) return '#' + h;
|
|
62
|
-
if (h.length === 8) return '#' + h.slice(0, 6);
|
|
63
|
-
return null;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
// rgb()/rgba()
|
|
67
|
-
m = v.match(/^rgba?\(([^)]+)\)$/);
|
|
68
|
-
if (m) {
|
|
69
|
-
const parts = m[1].split(/[,/\s]+/).filter(Boolean);
|
|
70
|
-
if (parts.length >= 3) {
|
|
71
|
-
const [r, g, b] = parts;
|
|
72
|
-
const pn = (x) => (x.endsWith('%') ? (parseFloat(x) / 100) * 255 : parseFloat(x));
|
|
73
|
-
const rr = pn(r), gg = pn(g), bb = pn(b);
|
|
74
|
-
if ([rr, gg, bb].every((x) => !Number.isNaN(x))) return '#' + toHex2(rr) + toHex2(gg) + toHex2(bb);
|
|
75
|
-
}
|
|
76
|
-
return null;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
// hsl()/hsla()
|
|
80
|
-
m = v.match(/^hsla?\(([^)]+)\)$/);
|
|
81
|
-
if (m) {
|
|
82
|
-
const parts = m[1].split(/[,/\s]+/).filter(Boolean);
|
|
83
|
-
if (parts.length >= 3) {
|
|
84
|
-
const h = parseFloat(parts[0]);
|
|
85
|
-
const s = parseFloat(parts[1]) / 100;
|
|
86
|
-
const l = parseFloat(parts[2]) / 100;
|
|
87
|
-
if (![h, s, l].some((x) => Number.isNaN(x))) {
|
|
88
|
-
const c = (1 - Math.abs(2 * l - 1)) * s;
|
|
89
|
-
const x = c * (1 - Math.abs(((h / 60) % 2) - 1));
|
|
90
|
-
const mm = l - c / 2;
|
|
91
|
-
let r = 0, g = 0, b = 0;
|
|
92
|
-
const hh = ((h % 360) + 360) % 360;
|
|
93
|
-
if (hh < 60) [r, g, b] = [c, x, 0];
|
|
94
|
-
else if (hh < 120) [r, g, b] = [x, c, 0];
|
|
95
|
-
else if (hh < 180) [r, g, b] = [0, c, x];
|
|
96
|
-
else if (hh < 240) [r, g, b] = [0, x, c];
|
|
97
|
-
else if (hh < 300) [r, g, b] = [x, 0, c];
|
|
98
|
-
else [r, g, b] = [c, 0, x];
|
|
99
|
-
return '#' + toHex2((r + mm) * 255) + toHex2((g + mm) * 255) + toHex2((b + mm) * 255);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
return null;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
return null;
|
|
44
|
+
const c = parseColor(raw);
|
|
45
|
+
return c ? toHex(c) : null;
|
|
106
46
|
}
|
|
107
47
|
|
|
108
48
|
function resolveUrl(href, base) {
|
|
@@ -245,7 +185,7 @@ function extractColors(cssText, $, cssVariables) {
|
|
|
245
185
|
// brand-ish CSS variables (high confidence)
|
|
246
186
|
for (const [name, value] of Object.entries(cssVariables)) {
|
|
247
187
|
if (BRAND_VAR_RE.test(name)) {
|
|
248
|
-
const hex = normalizeColor(value);
|
|
188
|
+
const hex = normalizeColor(resolveValue(value, cssVariables, new Set([name])));
|
|
249
189
|
if (hex) add(hex, 'css-var');
|
|
250
190
|
}
|
|
251
191
|
}
|
|
@@ -396,33 +336,40 @@ function extractLogo($, pageUrl) {
|
|
|
396
336
|
}
|
|
397
337
|
|
|
398
338
|
function extractTokens(cssText, cssVariables) {
|
|
339
|
+
// Declarations that reference a variable count under the value it resolves to.
|
|
399
340
|
const collect = (re) => {
|
|
400
341
|
const counts = new Map();
|
|
401
342
|
let m;
|
|
402
343
|
while ((m = re.exec(cssText)) !== null) {
|
|
403
|
-
const v = m[1].trim();
|
|
344
|
+
const v = resolveValue(m[1].replace(/!important/i, '').trim(), cssVariables);
|
|
404
345
|
if (v && !v.includes('var(')) counts.set(v, (counts.get(v) || 0) + 1);
|
|
405
346
|
}
|
|
406
347
|
return [...counts.entries()].sort((a, b) => b[1] - a[1]).slice(0, 8).map(([value, count]) => ({ value, count }));
|
|
407
348
|
};
|
|
408
349
|
|
|
409
|
-
const spacing = {};
|
|
410
|
-
for (const [name, value] of Object.entries(cssVariables)) {
|
|
411
|
-
if (/(space|spacing|gap|radius|size)/i.test(name)) spacing[name] = value;
|
|
412
|
-
}
|
|
413
|
-
|
|
414
350
|
return {
|
|
415
351
|
radii: collect(/border-radius\s*:\s*([^;}{]+)/gi),
|
|
416
352
|
shadows: collect(/box-shadow\s*:\s*([^;}{]+)/gi),
|
|
417
|
-
spacingVariables: spacing,
|
|
418
353
|
};
|
|
419
354
|
}
|
|
420
355
|
|
|
356
|
+
// What each `source` value in the result means, returned with it so a reader
|
|
357
|
+
// of the output can audit any value without this file.
|
|
358
|
+
const PROVENANCE = {
|
|
359
|
+
computed: 'read from the rendered page with getComputedStyle',
|
|
360
|
+
'css-variable': 'a CSS custom property, var() references resolved',
|
|
361
|
+
'css-var': 'counted in brand-named CSS custom properties',
|
|
362
|
+
'theme-color': 'the page\'s theme-color meta tag',
|
|
363
|
+
declaration: 'counted in CSS declarations',
|
|
364
|
+
inferred: 'a guess from CSS variable names; no rendered element confirmed it',
|
|
365
|
+
};
|
|
366
|
+
|
|
421
367
|
/**
|
|
422
368
|
* Extract the full branding object from a loaded cheerio $.
|
|
423
369
|
* @param {import('cheerio').CheerioAPI} $
|
|
424
370
|
* @param {string} pageUrl
|
|
425
|
-
* @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number, media?: 'screen'|'unconditional-screen', inlineStyles?: boolean }} [opts]
|
|
371
|
+
* @param {{ fetchLinkedCss?: boolean, maxStylesheets?: number, perFileTimeoutMs?: number, timeoutMs?: number, overallTimeoutMs?: number, stylesheetConcurrency?: number, media?: 'screen'|'unconditional-screen', inlineStyles?: boolean, computed?: object|null, rawVariables?: boolean, dtcg?: boolean }} [opts]
|
|
372
|
+
* `computed` is a brandingProbe() result from a render of the same page.
|
|
426
373
|
* @returns {Promise<object>}
|
|
427
374
|
*/
|
|
428
375
|
export async function extractBranding($, pageUrl, opts = {}) {
|
|
@@ -445,26 +392,51 @@ export async function extractBranding($, pageUrl, opts = {}) {
|
|
|
445
392
|
const colors = safe(() => extractColors(sources.cssText, $, cssVariables), [], 'colors');
|
|
446
393
|
const fontInfo = safe(() => extractFonts(sources.cssText, $, cssVariables), { fonts: [], genericFallbacks: [], webfontProviders: [], fontFaces: [] }, 'fonts');
|
|
447
394
|
const logo = safe(() => extractLogo($, pageUrl), { favicons: [], ogImage: null, candidates: [], inlineHeaderSvg: null }, 'logo');
|
|
448
|
-
const
|
|
395
|
+
const declTokens = safe(() => extractTokens(sources.cssText, cssVariables), { radii: [], shadows: [] }, 'tokens');
|
|
396
|
+
const tokens = safe(() => groupTokens(cssVariables), null, 'token grouping');
|
|
397
|
+
|
|
398
|
+
const computed = options.computed ? safe(() => buildComputedBranding(options.computed), null, 'computed styles') : null;
|
|
399
|
+
const resolved = safe(() => Object.fromEntries(Object.entries(cssVariables).map(([k, v]) => [k, resolveValue(v, cssVariables, new Set([k]))])), {}, 'variable resolution');
|
|
400
|
+
// Computed roles win; an inferred guess fills only a role no element showed.
|
|
401
|
+
const roles = { ...(computed?.roles || {}) };
|
|
402
|
+
for (const [name, guess] of Object.entries(safe(() => inferRoles(resolved, colors), {}, 'role inference'))) {
|
|
403
|
+
if (!roles[name]) roles[name] = guess;
|
|
404
|
+
}
|
|
449
405
|
|
|
450
|
-
|
|
451
|
-
|
|
406
|
+
const result = {
|
|
407
|
+
colorScheme: computed?.colorScheme || schemeOf(roles.background?.value) || undefined,
|
|
408
|
+
roles,
|
|
409
|
+
components: computed?.components || {},
|
|
410
|
+
typography: computed?.typography || {},
|
|
452
411
|
fonts: fontInfo.fonts,
|
|
453
412
|
genericFallbacks: fontInfo.genericFallbacks,
|
|
454
413
|
webfontProviders: fontInfo.webfontProviders,
|
|
455
414
|
fontFaces: fontInfo.fontFaces,
|
|
456
415
|
logo,
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
416
|
+
colors,
|
|
417
|
+
tokens: tokens ? { source: 'css-variable', ...tokens } : undefined,
|
|
418
|
+
radii: declTokens.radii,
|
|
419
|
+
shadows: declTokens.shadows,
|
|
420
|
+
};
|
|
421
|
+
if (options.rawVariables) result.cssVariables = cssVariables;
|
|
422
|
+
if (options.dtcg && tokens) {
|
|
423
|
+
result.dtcg = safe(() => toDtcg({ tokens, roles, fonts: fontInfo.fonts, url: pageUrl }), undefined, 'dtcg');
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
return {
|
|
427
|
+
...result,
|
|
461
428
|
sources: {
|
|
462
429
|
styleBlocks: sources.styleBlocks,
|
|
463
430
|
inlineStyleEls: sources.inlineStyleEls,
|
|
464
431
|
linkedStylesheetsFetched: sources.fetchedUrls,
|
|
432
|
+
rendered: Boolean(computed),
|
|
465
433
|
},
|
|
434
|
+
provenance: PROVENANCE,
|
|
466
435
|
notes: [
|
|
467
|
-
|
|
436
|
+
computed
|
|
437
|
+
? 'Roles, components and typography were read from the rendered page (source "computed"); tokens come from CSS custom properties with var() references resolved.'
|
|
438
|
+
: 'Not rendered: roles marked "inferred" are guesses from CSS variable names, and components and typography are empty. JS-injected styles are not reflected.',
|
|
439
|
+
...(options.rawVariables ? [] : ['cssVariables (every custom property as found) is left out; pass brandingOptions.rawVariables:true to include it.']),
|
|
468
440
|
],
|
|
469
441
|
warnings: warnings.length ? warnings : undefined,
|
|
470
442
|
};
|