crawlforge-mcp-server 6.24.0 → 6.25.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/server.js +2 -2
- package/src/core/ActionExecutor.js +14 -0
- package/src/schemas/toolOutputSchemas.js +1 -1
- package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-brand-style-guide/SKILL.md +107 -0
- package/src/skills/agent-skills/crawlforge-browser-sessions/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md +2 -1
- package/src/skills/agent-skills/crawlforge-stealth-browsing/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-web-scraping/SKILL.md +1 -1
- package/src/skills/agent-skills/crawlforge-web-scraping/references/tool-reference.md +1 -0
- package/src/tools/scrape/_brandingColor.js +185 -0
- package/src/tools/scrape/_brandingComputed.js +262 -0
- package/src/tools/scrape/_brandingExtractor.js +62 -90
- package/src/tools/scrape/_brandingTokens.js +350 -0
- package/src/tools/scrape/unifiedScrape.js +57 -16
- package/src/tools/search/adapters/dataforseoSearch.js +70 -8
- package/src/tools/search/queryExpander.js +16 -1
- package/src/tools/search/serpRank.js +22 -9
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crawlforge-mcp-server",
|
|
3
|
-
"version": "6.
|
|
3
|
+
"version": "6.25.1",
|
|
4
4
|
"mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
|
|
5
5
|
"description": "CrawlForge MCP Server - Professional Model Context Protocol server with 31 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
|
|
6
6
|
"main": "server.js",
|
|
@@ -135,9 +135,9 @@
|
|
|
135
135
|
"sentiment": "5.0.2",
|
|
136
136
|
"turndown": "^7.2.4",
|
|
137
137
|
"turndown-plugin-gfm": "^1.0.2",
|
|
138
|
-
"undici": "^7.
|
|
138
|
+
"undici": "^7.30.1",
|
|
139
139
|
"winston": "^3.11.0",
|
|
140
|
-
"zod": "^4.
|
|
140
|
+
"zod": "^4.6.7"
|
|
141
141
|
},
|
|
142
142
|
"optionalDependencies": {
|
|
143
143
|
"camoufox": "^0.1.19",
|
|
@@ -151,7 +151,7 @@
|
|
|
151
151
|
"jest": "^30.3.0"
|
|
152
152
|
},
|
|
153
153
|
"overrides": {
|
|
154
|
-
"undici": "^7.
|
|
154
|
+
"undici": "^7.30.1",
|
|
155
155
|
"underscore": "^1.13.8",
|
|
156
156
|
"qs": "^6.14.2",
|
|
157
157
|
"path-to-regexp": "^8.4.2",
|
package/server.js
CHANGED
|
@@ -114,7 +114,7 @@ if (configErrors.length > 0 && config.server.nodeEnv === 'production') {
|
|
|
114
114
|
// Create the server
|
|
115
115
|
const server = new McpServer({
|
|
116
116
|
name: "crawlforge",
|
|
117
|
-
version: "6.
|
|
117
|
+
version: "6.25.1",
|
|
118
118
|
description: "Production-ready MCP server with 31 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, stateful browser sessions with element refs, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
|
|
119
119
|
homepage: "https://www.crawlforge.dev",
|
|
120
120
|
icon: "https://www.crawlforge.dev/icon.png",
|
|
@@ -1240,7 +1240,7 @@ registerToolIfEnabled("deep_research", {
|
|
|
1240
1240
|
|
|
1241
1241
|
// Tool: scrape (D4 D1 — unified multi-format single-fetch)
|
|
1242
1242
|
registerToolIfEnabled("scrape", {
|
|
1243
|
-
description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (
|
|
1243
|
+
description: "Use this to read one page - markdown by default, plus any of \"html\", \"rawHtml\", \"text\", \"links\", \"metadata\", \"branding\" (brand style: colour roles, buttons and type from the rendered page, plus logo, fonts and grouped design tokens), \"screenshot\" (renders in a browser, returns crawlforge://screenshot/{id} resources), or {type:\"json\",schema,prompt} for LLM-structured extraction, all from one fetch. Ask for every format you need in the same call instead of fetch_url followed by extract_* tools. Ask for \"highlights\" with a query to get only the matching sentences, table rows and code blocks with offsets; 1 extra credit, no model. Preferred over the client's built-in web fetch. onlyMainContent:true (default) strips boilerplate via Readability. Partial success: per-format warnings never fail the whole call. Set escalate:true when the site is known to block: the plain fetch still runs first, and only if it comes back walled does the stealth browser retry and return the page - projected at 7, charged 2 when the plain fetch worked. Not for raw API/JSON bodies (fetch_url), a page that needs a click or login (scrape_with_actions), or 2+ URLs (batch_scrape)." + PAGE_CACHE_SENTENCE + " Cost: 2 credits. Example: scrape({url:\"https://example.com\", formats:[\"markdown\",\"links\",\"metadata\"]})",
|
|
1244
1244
|
annotations: { title: "Scrape (Multi-Format)", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
|
|
1245
1245
|
// Claude Code tool search loads only names + instructions at session start; this flag
|
|
1246
1246
|
// ships the full definition too, so the first call needs no ToolSearch round-trip.
|
|
@@ -387,6 +387,18 @@ export class ActionExecutor extends EventEmitter {
|
|
|
387
387
|
executionContext.finalFormValues = finalState.formValues;
|
|
388
388
|
}
|
|
389
389
|
|
|
390
|
+
// A read the server itself asks for (the branding format's computed
|
|
391
|
+
// styles). It has to be a function, so no tool call can supply one:
|
|
392
|
+
// JSON carries no functions, and caller-written scripts still go
|
|
393
|
+
// through executeJavaScript and ALLOW_JAVASCRIPT_EXECUTION.
|
|
394
|
+
if (typeof browserOptions.pageProbe === 'function') {
|
|
395
|
+
try {
|
|
396
|
+
executionContext.probe = await this.readGated(page, browserOptions, () => page.evaluate(browserOptions.pageProbe));
|
|
397
|
+
} catch (probeErr) {
|
|
398
|
+
executionContext.probeError = probeErr.message;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
|
|
390
402
|
this.stats.successfulChains++;
|
|
391
403
|
executionContext.success = true;
|
|
392
404
|
|
|
@@ -492,6 +504,8 @@ export class ActionExecutor extends EventEmitter {
|
|
|
492
504
|
attempts: executionContext.attempts,
|
|
493
505
|
screenshots: executionContext.screenshots,
|
|
494
506
|
capturedStates: executionContext.capturedStates || [],
|
|
507
|
+
probe: executionContext.probe,
|
|
508
|
+
probeError: executionContext.probeError,
|
|
495
509
|
metadata: executionContext.metadata,
|
|
496
510
|
stats: {
|
|
497
511
|
totalActions: executionContext.results.length,
|
|
@@ -116,7 +116,7 @@ const scrapeFormatShapes = {
|
|
|
116
116
|
text: z.string().optional(),
|
|
117
117
|
links: scrapeLinksShape.optional(),
|
|
118
118
|
metadata: scrapeMetadataShape.optional(),
|
|
119
|
-
branding: z.record(z.unknown()).optional().describe('
|
|
119
|
+
branding: z.record(z.unknown()).optional().describe('Brand style: colour roles, button components and type (computed or inferred), logo, fonts and grouped design tokens'),
|
|
120
120
|
screenshots: z.array(z.object({}).passthrough()).optional().describe('Present for the "screenshot" format; each item carries a resourceUri once published'),
|
|
121
121
|
json: z.unknown().optional().describe('Result of the {type:"json"} format (LLM-structured extraction)'),
|
|
122
122
|
highlights: z.array(scrapeHighlightUnitShape).optional().describe('Result of the {type:"highlights"} format: the units matching the query, best first, verbatim with offsets'),
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-batch-automation
|
|
3
3
|
description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: crawlforge-brand-style-guide
|
|
3
|
+
description: "Builds a brand style guide or a design-token file from a live website with the branding format of CrawlForge's scrape tool. Use when the user wants a company's brand colours, fonts, logo, button styles or design tokens, asks to extract a website's style, design system or brand kit, wants a style guide made from a URL, wants to match a site's look, or wants tokens for Figma, Tokens Studio or Style Dictionary (W3C DTCG JSON). One scrape call returns colour roles, the primary and secondary buttons and type read from the rendered page, the logo and favicons, font families and grouped CSS tokens, each value labelled with where it came from."
|
|
4
|
+
metadata:
|
|
5
|
+
version: 6.25.1
|
|
6
|
+
source: crawlforge-mcp-server
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# CrawlForge Brand Style Guide
|
|
10
|
+
|
|
11
|
+
Turn a website into a style guide or a design-token file. CrawlForge collects
|
|
12
|
+
the brand data in one `scrape` call; you write the guide from it.
|
|
13
|
+
|
|
14
|
+
## When to use
|
|
15
|
+
|
|
16
|
+
- "Make a style guide from stripe.com" / "build a brand kit for this site"
|
|
17
|
+
- "What colours and fonts does this company use?"
|
|
18
|
+
- "Grab this site's logo and button styles"
|
|
19
|
+
- "Match the look of this website"
|
|
20
|
+
- "Export this site's design tokens for Figma / Style Dictionary"
|
|
21
|
+
|
|
22
|
+
## The one call
|
|
23
|
+
|
|
24
|
+
```json
|
|
25
|
+
{
|
|
26
|
+
"tool": "scrape",
|
|
27
|
+
"params": {
|
|
28
|
+
"url": "https://stripe.com",
|
|
29
|
+
"formats": ["branding", "screenshot"],
|
|
30
|
+
"brandingOptions": { "dtcg": true }
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Cost: 2 credits. `branding` and `screenshot` share one browser render. Add
|
|
36
|
+
`"metadata"` if you need the site name and description too. Leave out `dtcg`
|
|
37
|
+
unless the user wants a token file: it can double the size of the result.
|
|
38
|
+
|
|
39
|
+
If the result is `truncated: true`, do not scrape again. Read the parts with
|
|
40
|
+
`read_result` and a `json_path`, such as `content.branding.roles` or
|
|
41
|
+
`content.branding.dtcg`.
|
|
42
|
+
|
|
43
|
+
## What comes back in `content.branding`
|
|
44
|
+
|
|
45
|
+
| Key | What it holds |
|
|
46
|
+
|-----|---------------|
|
|
47
|
+
| `colorScheme` | `light` or `dark`, from the page background |
|
|
48
|
+
| `roles` | `background`, `text`, `heading`, `link`, `primary`, `onPrimary`: each `{ value, source, evidence }` |
|
|
49
|
+
| `components` | `buttonPrimary` and `buttonSecondary`: fill, label colour, border, radius, padding, font, size and weight |
|
|
50
|
+
| `typography` | `body`, `h1`, `h2`: family, stack, size, weight, line height, letter spacing, colour |
|
|
51
|
+
| `logo` | `inlineHeaderSvg` (the header logo as SVG), `favicons`, `ogImage`, image `candidates` |
|
|
52
|
+
| `fonts`, `fontFaces` | font families ranked by use, and their `@font-face` sources |
|
|
53
|
+
| `tokens` | the site's CSS custom properties, grouped: `colors.ramps`, `colors.semantic`, `typography`, `spacing`, `radius`, `shadow`, plus `stats` on what was dropped |
|
|
54
|
+
| `colors` | every colour found, ranked by frequency |
|
|
55
|
+
| `dtcg` | with `dtcg: true`, the tokens and roles as W3C Design Tokens (2025.10) JSON |
|
|
56
|
+
| `provenance` | what each `source` value means |
|
|
57
|
+
|
|
58
|
+
### Reading `source`
|
|
59
|
+
|
|
60
|
+
- `computed`: read from an element on the rendered page. The `evidence` says
|
|
61
|
+
which, for example `button "Get started"`. Trust these first.
|
|
62
|
+
- `css-variable`, `css-var`, `declaration`, `theme-color`: found in the
|
|
63
|
+
site's CSS or HTML.
|
|
64
|
+
- `inferred`: a guess from CSS variable names, used when no element showed
|
|
65
|
+
the role. Say in the guide that the value is a guess, or confirm it on the
|
|
66
|
+
screenshot first.
|
|
67
|
+
|
|
68
|
+
`sources.rendered` tells you whether the page was rendered. When it is
|
|
69
|
+
`false`, `warnings` says why, and `components` and `typography` are empty.
|
|
70
|
+
|
|
71
|
+
## Writing the guide
|
|
72
|
+
|
|
73
|
+
Use the screenshot to check the roles before you write. Then cover:
|
|
74
|
+
|
|
75
|
+
1. **Logo**: the header SVG on light and dark grounds, plus the favicon and
|
|
76
|
+
share-image links.
|
|
77
|
+
2. **Colour roles**: one swatch per role, with its hex, its token name when a
|
|
78
|
+
ramp step matches, and what it is used for.
|
|
79
|
+
3. **Palette**: the ramps from `tokens.colors.ramps`, in step order.
|
|
80
|
+
4. **Typography**: the families, the type scale from `tokens.typography`, and
|
|
81
|
+
the body and heading styles from `typography`.
|
|
82
|
+
5. **Spacing and radius**, from `tokens.spacing` and `tokens.radius`.
|
|
83
|
+
6. **Buttons**: the primary and secondary buttons rebuilt from `components`.
|
|
84
|
+
7. **Sources**: label each value as read from the page, found in CSS, or
|
|
85
|
+
inferred, so a reader can check it.
|
|
86
|
+
|
|
87
|
+
For a page or artifact, show the swatches and buttons visually. For a token
|
|
88
|
+
file, save `dtcg` as `tokens.json`.
|
|
89
|
+
|
|
90
|
+
## Limits to tell the user about
|
|
91
|
+
|
|
92
|
+
- The guide is unofficial when the site is not the user's own. Say so on the
|
|
93
|
+
page, and point to the company's own brand resources for anything that will
|
|
94
|
+
ship.
|
|
95
|
+
- Many brand fonts are licensed. The `@font-face` URL shows which font a site
|
|
96
|
+
uses, not permission to use it, so render specimens in a fallback face.
|
|
97
|
+
- CSS holds no usage rules: no logo clear space, minimum sizes or do's and
|
|
98
|
+
don'ts. Those live in brand documents.
|
|
99
|
+
- `dtcg` follows the DTCG 2025.10 format, where colours and dimensions are
|
|
100
|
+
objects rather than strings. Support for 2025.10 varies by tool and version
|
|
101
|
+
(Style Dictionary lists it as in progress for v5), so tell the user to check
|
|
102
|
+
their tool before relying on the file.
|
|
103
|
+
- `buttonSecondary` is the outline button next to the primary. A site with no
|
|
104
|
+
partner button returns none.
|
|
105
|
+
- A blocked site comes back with a warning: retry with `escalate: true`. A
|
|
106
|
+
page that needed the stealth browser is not re-rendered for computed
|
|
107
|
+
styles, so its roles are `inferred`.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-browser-sessions
|
|
3
3
|
description: "Keeps one browser page alive across several tool calls with CrawlForge's browser_session tool. Use when the user needs to log in and then read pages behind that login, work through a multi-step flow (search, filter, paginate, fill a wizard), or look at a page before deciding what to click. The loop is: open a session on a URL, snapshot it to list the interactive elements with stable refs (@e1, @e2), act on those refs, read the content, close. Unlike scrape_with_actions, which is one-shot and closes its browser when the call returns, a session survives between calls, so one login is paid for once and a wrong selector costs one call instead of a whole chain."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-change-tracking
|
|
3
3
|
description: "Monitors web pages for changes over time with CrawlForge's track_changes tool. Use when the user wants to track changes to a page, watch a URL, monitor competitor pricing, detect when content updates, get notified of regulation or product-availability changes, or diff a page against a saved baseline. Workflow: create a baseline with operation create_baseline, then periodically compare with operation compare to get a change percentage and a diff; supports CSS-selector scoping, webhooks, and scheduled monitoring."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-deep-research
|
|
3
3
|
description: "Runs multi-source web research and autonomous question-answering with CrawlForge's deep_research, agent, search_web, and reddit_search tools. Use when the user wants to research a topic, do a deep dive, compare competitors, gather facts with citations, answer a question from the web, search the web, or get a synthesized report from many sources, or search Reddit posts, comments, and threads. deep_research expands queries, fetches and dedupes sources, then synthesizes; agent autonomously plans searches and navigation from a plain-English prompt with no URLs needed; search_web returns ranked results; reddit_search searches Reddit via community archives (reddit.com blocks scrapers). Caps costs via max_urls and confirms before expensive runs."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-getting-started
|
|
3
3
|
description: "Orientation and tool-selection guide for the CrawlForge MCP server's 31 web tools. Use when the user is getting started with CrawlForge, asks which CrawlForge tool to use, how to set up the API key, how skills or the CLI work, what a tool costs in credits, or when one tool fails and a fallback is needed. Routes requests to the right specialized skill (web scraping, deep research, stealth, structured extraction, change tracking, batch automation, browser sessions), and explains MCP-tools-vs-CLI, the Ollama-first LLM fallback chain, and per-tool credit costs."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -52,6 +52,7 @@ stored at `~/.crawlforge/config.json`.
|
|
|
52
52
|
| Watch a page for changes / monitor pricing | **crawlforge-change-tracking** |
|
|
53
53
|
| Scrape many URLs, run browser actions, generate llms.txt | **crawlforge-batch-automation** |
|
|
54
54
|
| Keep a browser open across calls: log in then read, click through a multi-step flow | **crawlforge-browser-sessions** |
|
|
55
|
+
| Make a style guide or design-token file from a website's brand | **crawlforge-brand-style-guide** |
|
|
55
56
|
|
|
56
57
|
## The 31 tools at a glance
|
|
57
58
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-stealth-browsing
|
|
3
3
|
description: "Bypasses bot detection with CrawlForge's stealth_mode tool and looks up a country's locale settings with localization. Use when a site returns 403 or 429, CAPTCHAs, 'please enable JavaScript', or empty content, or is protected by Cloudflare, DataDome, or PerimeterX, or when the user needs a page in a specific locale, language, or timezone. stealth_mode runs a stealth browser (engine auto by default: camoufox/Firefox for advanced fingerprinting, playwright/Chromium when pinned for speed) and can screenshot; localization returns a country's Accept-Language, timezone and currency for you to pass to fetch_url or stealth_mode - it applies nothing itself and uses no proxy. Explains when to escalate from a normal scrape to stealth, and why a hard block needs a residential proxy."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-structured-extraction
|
|
3
3
|
description: "Extracts structured JSON and analyzes content with CrawlForge's extract_structured, extract_with_llm, extract_embedded_state, scrape_structured, scrape_template, process_document, analyze_content, summarize_content, and list_ollama_models tools. Use when the user wants to extract specific fields, pull data into a JSON schema, extract by natural-language prompt, scrape with CSS selectors, read a page's embedded JavaScript state (__NEXT_DATA__, React Server Components, Nuxt, Apollo, Redux), get product, profile, or repo data from known sites (Amazon, LinkedIn, GitHub, YouTube, Reddit, and more), parse a PDF or DOCX, summarize a page, or analyze sentiment, entities, or keywords. Defaults to local Ollama for LLM extraction; OpenAI and Anthropic optional."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
name: crawlforge-web-scraping
|
|
3
3
|
description: "Scrapes web pages and returns clean Markdown, HTML, plain text, links, or metadata using CrawlForge's scrape, fetch_url, extract_content, extract_text, extract_links, extract_metadata, map_site, and crawl_deep tools. Use when the user wants to scrape a URL, fetch a page, get the markdown or text of a website, extract links or metadata, read an article, map a site's URLs, generate a sitemap, or crawl a whole website or documentation site. Prefer the unified scrape tool to get multiple formats in one call; use map_site to discover URLs and crawl_deep to walk an entire site."
|
|
4
4
|
metadata:
|
|
5
|
-
version: 6.
|
|
5
|
+
version: 6.25.1
|
|
6
6
|
source: crawlforge-mcp-server
|
|
7
7
|
---
|
|
8
8
|
|
|
@@ -10,6 +10,7 @@ CrawlForge credits.
|
|
|
10
10
|
| `url` | string (URL) | — | Required. |
|
|
11
11
|
| `formats` | array | `["markdown"]` | Any of `markdown`, `html`, `rawHtml`, `text`, `links`, `metadata`, `branding`, `screenshot`, `{type:"json", schema?, prompt?}`, `{type:"highlights", query, max_highlights?, mode?}` (the matching sentences, table rows and code blocks, verbatim, with offsets into the markdown; +1 credit once per call) or `{type:"question", question, mode?}` (an answer assembled from the evidence units; same +1). `mode:"model"` adds 3 credits and needs an LLM route. |
|
|
12
12
|
| `onlyMainContent` | boolean | `true` | Strip boilerplate via Readability. |
|
|
13
|
+
| `brandingOptions` | object | — | For `branding`: `computeStyles` (default `true`: render the page and read colour roles, the primary and secondary buttons and type from computed styles; `false` stays static and marks roles `inferred`), `rawVariables` (default `false`: add every CSS custom property as `cssVariables`), `dtcg` (default `false`: add the tokens as W3C Design Tokens 2025.10 JSON as `dtcg`, which can double the result; read it back with `read_result` and `json_path: "content.branding.dtcg"`), `fetchLinkedCss`, `maxStylesheets`. Every role and component carries `source` (`computed`, `css-variable`, `inferred`…); `provenance` in the result defines each. |
|
|
13
14
|
| `timeoutMs` | number | `15000` | 1000–60000. |
|
|
14
15
|
| `max_inline_chars` | number | `40000` | 1,000–10,000,000. A larger result returns `preview` + `result_handle` for `read_result` (below). |
|
|
15
16
|
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* _brandingColor.js — CSS colour parsing for the branding format.
|
|
3
|
+
*
|
|
4
|
+
* parseColor turns hex, rgb(), hsl(), oklch(), oklab(), lab(), lch() and the
|
|
5
|
+
* basic named colours into { r, g, b, a } (channels 0-255, alpha 0-1). The
|
|
6
|
+
* wide-gamut forms matter because Tailwind v4 writes its palette in oklch and
|
|
7
|
+
* its build (Lightning CSS) ships it as lab(). The helpers below it
|
|
8
|
+
* compare colours: how saturated one is, how light, how far apart two are.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const NAMED_COLORS = {
|
|
12
|
+
black: '#000000', white: '#ffffff', red: '#ff0000', green: '#008000',
|
|
13
|
+
blue: '#0000ff', yellow: '#ffff00', orange: '#ffa500', purple: '#800080',
|
|
14
|
+
gray: '#808080', grey: '#808080', silver: '#c0c0c0', navy: '#000080',
|
|
15
|
+
teal: '#008080', maroon: '#800000', olive: '#808000', lime: '#00ff00',
|
|
16
|
+
aqua: '#00ffff', fuchsia: '#ff00ff',
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function clamp(n, lo, hi) {
|
|
20
|
+
return Math.max(lo, Math.min(hi, n));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function toHex2(n) {
|
|
24
|
+
return clamp(Math.round(n), 0, 255).toString(16).padStart(2, '0');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// A channel or alpha argument: `none` is 0, a percentage scales to `full`.
|
|
28
|
+
function num(part, full = 1) {
|
|
29
|
+
if (part === 'none') return 0;
|
|
30
|
+
if (part.endsWith('%')) return (parseFloat(part) / 100) * full;
|
|
31
|
+
return parseFloat(part);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function alphaOf(part) {
|
|
35
|
+
if (part == null) return 1;
|
|
36
|
+
const a = num(part);
|
|
37
|
+
return Number.isNaN(a) ? 1 : clamp(a, 0, 1);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function hslToRgb(h, s, l) {
|
|
41
|
+
const c = (1 - Math.abs(2 * l - 1)) * s;
|
|
42
|
+
const x = c * (1 - Math.abs(((h / 60) % 2) - 1));
|
|
43
|
+
const mm = l - c / 2;
|
|
44
|
+
let r = 0, g = 0, b = 0;
|
|
45
|
+
const hh = ((h % 360) + 360) % 360;
|
|
46
|
+
if (hh < 60) [r, g, b] = [c, x, 0];
|
|
47
|
+
else if (hh < 120) [r, g, b] = [x, c, 0];
|
|
48
|
+
else if (hh < 180) [r, g, b] = [0, c, x];
|
|
49
|
+
else if (hh < 240) [r, g, b] = [0, x, c];
|
|
50
|
+
else if (hh < 300) [r, g, b] = [x, 0, c];
|
|
51
|
+
else [r, g, b] = [c, 0, x];
|
|
52
|
+
return [(r + mm) * 255, (g + mm) * 255, (b + mm) * 255];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Linear-light channels -> gamma-encoded sRGB 0-255, clipped to the gamut.
|
|
56
|
+
function encode(lin) {
|
|
57
|
+
return lin.map((x) => {
|
|
58
|
+
const v = clamp(x, 0, 1);
|
|
59
|
+
return (v <= 0.0031308 ? 12.92 * v : 1.055 * v ** (1 / 2.4) - 0.055) * 255;
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// CIE Lab (D50) -> XYZ D50 -> Bradford to D65 -> linear sRGB (CSS Color 4 matrices).
|
|
64
|
+
function labToRgb(L, a, b) {
|
|
65
|
+
const e = 216 / 24389;
|
|
66
|
+
const k = 24389 / 27;
|
|
67
|
+
const fy = (L + 16) / 116;
|
|
68
|
+
const fx = fy + a / 500;
|
|
69
|
+
const fz = fy - b / 200;
|
|
70
|
+
const x = (fx ** 3 > e ? fx ** 3 : (116 * fx - 16) / k) * 0.3457 / 0.3585;
|
|
71
|
+
const y = L > k * e ? fy ** 3 : L / k;
|
|
72
|
+
const z = (fz ** 3 > e ? fz ** 3 : (116 * fz - 16) / k) * (1 - 0.3457 - 0.3585) / 0.3585;
|
|
73
|
+
const X = 0.9554734527042182 * x - 0.023098536874261423 * y + 0.0632593086610217 * z;
|
|
74
|
+
const Y = -0.028369706963208136 * x + 1.0099954580058226 * y + 0.021041398966943008 * z;
|
|
75
|
+
const Z = 0.012314001688319899 * x - 0.020507696433477912 * y + 1.3303659366080753 * z;
|
|
76
|
+
return encode([
|
|
77
|
+
3.2409699419045226 * X - 1.537383177570094 * Y - 0.4986107602930034 * Z,
|
|
78
|
+
-0.9692436362808796 * X + 1.8759675015077202 * Y + 0.04155505740717559 * Z,
|
|
79
|
+
0.05563007969699366 * X - 0.20397695888897652 * Y + 1.0569715142428786 * Z,
|
|
80
|
+
]);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// OKLab -> sRGB (Björn Ottosson's reference matrices), clipped to the gamut.
|
|
84
|
+
function oklabToRgb(L, a, b) {
|
|
85
|
+
const l = (L + 0.3963377774 * a + 0.2158037573 * b) ** 3;
|
|
86
|
+
const m = (L - 0.1055613458 * a - 0.0638541728 * b) ** 3;
|
|
87
|
+
const s = (L - 0.0894841775 * a - 1.2914855480 * b) ** 3;
|
|
88
|
+
return encode([
|
|
89
|
+
4.0767416621 * l - 3.3077115913 * m + 0.2309699292 * s,
|
|
90
|
+
-1.2684380046 * l + 2.6097574011 * m - 0.3413193965 * s,
|
|
91
|
+
-0.0041960863 * l - 0.7034186147 * m + 1.7076147010 * s,
|
|
92
|
+
]);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Parse a CSS colour into { r, g, b, a }. Returns null for anything that is
|
|
97
|
+
* not a literal colour (gradients, currentColor, var(), transparent).
|
|
98
|
+
*/
|
|
99
|
+
export function parseColor(raw) {
|
|
100
|
+
if (!raw) return null;
|
|
101
|
+
const v = String(raw).trim().toLowerCase();
|
|
102
|
+
|
|
103
|
+
const named = NAMED_COLORS[v];
|
|
104
|
+
if (named) return parseColor(named);
|
|
105
|
+
|
|
106
|
+
let m = v.match(/^#([0-9a-f]{3,8})$/);
|
|
107
|
+
if (m) {
|
|
108
|
+
let h = m[1];
|
|
109
|
+
if (h.length === 3 || h.length === 4) h = h.split('').map((c) => c + c).join('');
|
|
110
|
+
if (h.length !== 6 && h.length !== 8) return null;
|
|
111
|
+
return {
|
|
112
|
+
r: parseInt(h.slice(0, 2), 16),
|
|
113
|
+
g: parseInt(h.slice(2, 4), 16),
|
|
114
|
+
b: parseInt(h.slice(4, 6), 16),
|
|
115
|
+
a: h.length === 8 ? parseInt(h.slice(6, 8), 16) / 255 : 1,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
m = v.match(/^(rgba?|hsla?|oklch|oklab|lch|lab)\(([^()]+)\)$/);
|
|
120
|
+
if (!m) return null;
|
|
121
|
+
const parts = m[2].split(/[,/\s]+/).filter(Boolean);
|
|
122
|
+
if (parts.length < 3) return null;
|
|
123
|
+
const a = alphaOf(parts[3]);
|
|
124
|
+
let rgb;
|
|
125
|
+
switch (m[1]) {
|
|
126
|
+
case 'rgb':
|
|
127
|
+
case 'rgba':
|
|
128
|
+
rgb = parts.slice(0, 3).map((p) => num(p, 255));
|
|
129
|
+
break;
|
|
130
|
+
case 'hsl':
|
|
131
|
+
case 'hsla':
|
|
132
|
+
rgb = hslToRgb(parseFloat(parts[0]), num(parts[1].endsWith('%') ? parts[1] : parts[1] + '%'), num(parts[2].endsWith('%') ? parts[2] : parts[2] + '%'));
|
|
133
|
+
break;
|
|
134
|
+
case 'oklch': {
|
|
135
|
+
const L = num(parts[0]);
|
|
136
|
+
const C = num(parts[1], 0.4);
|
|
137
|
+
const H = (num(parts[2]) * Math.PI) / 180;
|
|
138
|
+
rgb = oklabToRgb(L, C * Math.cos(H), C * Math.sin(H));
|
|
139
|
+
break;
|
|
140
|
+
}
|
|
141
|
+
case 'oklab':
|
|
142
|
+
rgb = oklabToRgb(num(parts[0]), num(parts[1], 0.4), num(parts[2], 0.4));
|
|
143
|
+
break;
|
|
144
|
+
case 'lch': {
|
|
145
|
+
const C = num(parts[1], 150);
|
|
146
|
+
const H = (num(parts[2]) * Math.PI) / 180;
|
|
147
|
+
rgb = labToRgb(num(parts[0], 100), C * Math.cos(H), C * Math.sin(H));
|
|
148
|
+
break;
|
|
149
|
+
}
|
|
150
|
+
case 'lab':
|
|
151
|
+
rgb = labToRgb(num(parts[0], 100), num(parts[1], 125), num(parts[2], 125));
|
|
152
|
+
break;
|
|
153
|
+
}
|
|
154
|
+
if (rgb.some((x) => Number.isNaN(x))) return null;
|
|
155
|
+
return { r: rgb[0], g: rgb[1], b: rgb[2], a };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** '#rrggbb', alpha dropped. */
|
|
159
|
+
export function toHex({ r, g, b }) {
|
|
160
|
+
return '#' + toHex2(r) + toHex2(g) + toHex2(b);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** '#rrggbb', or '#rrggbbaa' when the colour is translucent. */
|
|
164
|
+
export function toHexAlpha(c) {
|
|
165
|
+
return c.a < 1 ? toHex(c) + toHex2(c.a * 255) : toHex(c);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** 0 for a grey, 1 for a fully saturated colour. */
|
|
169
|
+
export function chroma({ r, g, b }) {
|
|
170
|
+
return (Math.max(r, g, b) - Math.min(r, g, b)) / 255;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** WCAG relative luminance, 0 (black) to 1 (white). */
|
|
174
|
+
export function luminance({ r, g, b }) {
|
|
175
|
+
const lin = (x) => {
|
|
176
|
+
const v = x / 255;
|
|
177
|
+
return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4;
|
|
178
|
+
};
|
|
179
|
+
return 0.2126 * lin(r) + 0.7152 * lin(g) + 0.0722 * lin(b);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Euclidean distance in RGB, 0 to ~441. */
|
|
183
|
+
export function distance(x, y) {
|
|
184
|
+
return Math.hypot(x.r - y.r, x.g - y.g, x.b - y.b);
|
|
185
|
+
}
|