@webcrawl/mcp 0.0.0-stage → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -2
- package/dist/bin.d.ts +2 -0
- package/dist/bin.js +23 -0
- package/dist/format.d.ts +13 -0
- package/dist/format.js +126 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/server.d.ts +18 -0
- package/dist/server.js +195 -0
- package/package.json +59 -4
package/README.md
CHANGED
|
@@ -1,3 +1,51 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @webcrawl/mcp
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Model Context Protocol (MCP) server for **[Webcrawl Anything](https://github.com/brainfish-ai/webcrawl-anything)** and **[webcrawl.app](https://webcrawl.app)**.
|
|
4
|
+
|
|
5
|
+
Gives Claude Code, Cursor, Codex, Windsurf, and Gemini CLI live access to web search, structured verticals (shopping, flights, hotels, real-estate, local business, CVEs), anti-bot page scraping, PDF/Office OCR, and YouTube transcripts.
|
|
6
|
+
|
|
7
|
+
## One-Command Agent Setup
|
|
8
|
+
|
|
9
|
+
Register `@webcrawl/mcp` and install the `webcrawl-anything` Agent Skill (`SKILL.md`) automatically:
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npx webcrawl setup
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Manual MCP Configuration
|
|
16
|
+
|
|
17
|
+
Add to `.cursor/mcp.json`, `~/.claude.json`, or your MCP client config:
|
|
18
|
+
|
|
19
|
+
```json
|
|
20
|
+
{
|
|
21
|
+
"mcpServers": {
|
|
22
|
+
"webcrawl": {
|
|
23
|
+
"command": "npx",
|
|
24
|
+
"args": ["-y", "@webcrawl/mcp@latest"],
|
|
25
|
+
"env": {
|
|
26
|
+
"WEBCRAWL_MCP_SCHEMA_PROFILE": "compact",
|
|
27
|
+
"WEBCRAWL_API_KEY": "wc_live_..."
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Compact Schema Profile
|
|
35
|
+
|
|
36
|
+
Set `WEBCRAWL_MCP_SCHEMA_PROFILE=compact` (default when installed via `npx webcrawl setup`) to minimize tool schema token usage in context-constrained agents while keeping full server-side validation. Set `WEBCRAWL_MCP_SCHEMA_PROFILE=full` for verbose parameter descriptions.
|
|
37
|
+
|
|
38
|
+
## Tools Provided
|
|
39
|
+
|
|
40
|
+
- `web_search` — Search the web or structured verticals (`shopping`, `flights`, `hotels`, `real-estate`, `business`, `security`, `academic`, `tech`, `news`).
|
|
41
|
+
- `web_scrape` — Fetch a URL and return clean Markdown with target evidence.
|
|
42
|
+
- `web_parse` / `web_parse_status` — Parse PDFs (native + OCR), Word/Excel/PowerPoint files, YouTube videos, and audio/video into Markdown.
|
|
43
|
+
- `web_map` — Discover all URLs on a domain (`sitemap.xml`, `robots.txt`, and link graph).
|
|
44
|
+
- `web_crawl` — Crawl up to 25 pages starting from a URL.
|
|
45
|
+
- `web_extract` — Extract structured JSON from a URL using a prompt or JSON Schema.
|
|
46
|
+
- `web_rerank` — Rerank candidate documents by relevance to a query.
|
|
47
|
+
- `search_categories` — List all supported search categories, subcategories, and filters.
|
|
48
|
+
|
|
49
|
+
## License
|
|
50
|
+
|
|
51
|
+
MIT
|
package/dist/bin.d.ts
ADDED
package/dist/bin.js
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
|
+
import { WebcrawlClient, resolveConfig } from "@webcrawl/client";
|
|
4
|
+
import { createServer, resolveSchemaProfile } from "./server.js";
|
|
5
|
+
const cfg = resolveConfig();
|
|
6
|
+
let client;
|
|
7
|
+
if (cfg.effectiveMode === "local") {
|
|
8
|
+
try {
|
|
9
|
+
// Load the shared local engine when running from the monorepo / local install without an API key
|
|
10
|
+
const localEngineSpec = ["..", "..", "..", "src", "local-engine.js"].join("/");
|
|
11
|
+
const localMod = (await import(localEngineSpec));
|
|
12
|
+
client = localMod.createLocalEngineClient();
|
|
13
|
+
process.stderr.write("webcrawl-mcp: running in zero-key LOCAL engine mode. Set WEBCRAWL_API_KEY (or run `webcrawl login`) to upgrade to webcrawl.app cloud.\n");
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
process.stderr.write("webcrawl-mcp: no API key configured; falling back to https://webcrawl.app (set WEBCRAWL_API_KEY or run `webcrawl login`).\n");
|
|
17
|
+
client = new WebcrawlClient({ apiKey: cfg.apiKey, baseUrl: cfg.baseUrl, userAgent: "webcrawl-mcp/0.1" });
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
client = new WebcrawlClient({ apiKey: cfg.apiKey, baseUrl: cfg.baseUrl, userAgent: "webcrawl-mcp/0.1" });
|
|
22
|
+
}
|
|
23
|
+
await createServer(client, { schemaProfile: resolveSchemaProfile() }).connect(new StdioServerTransport());
|
package/dist/format.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Hard cap on text returned by any tool so one call cannot flood the model context. */
|
|
2
|
+
export declare const MAX_OUTPUT_CHARS = 24000;
|
|
3
|
+
export declare function truncate(text: string, max?: number): string;
|
|
4
|
+
type Rec = Record<string, unknown>;
|
|
5
|
+
/** Compact, model-friendly digest of a search response. */
|
|
6
|
+
export declare function formatSearch(res: Rec): string;
|
|
7
|
+
export declare function formatCredits(res: Rec): string;
|
|
8
|
+
export declare function formatScrape(res: Rec): string;
|
|
9
|
+
/** Markdown plus a one-line note on how the file was read (and what was not). */
|
|
10
|
+
export declare function formatParse(res: Rec): string;
|
|
11
|
+
export declare function formatMap(res: Rec): string;
|
|
12
|
+
export declare function formatCrawl(res: Rec): string;
|
|
13
|
+
export {};
|
package/dist/format.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/** Hard cap on text returned by any tool so one call cannot flood the model context. */
|
|
2
|
+
export const MAX_OUTPUT_CHARS = 24_000;
|
|
3
|
+
export function truncate(text, max = MAX_OUTPUT_CHARS) {
|
|
4
|
+
if (text.length <= max) {
|
|
5
|
+
return text;
|
|
6
|
+
}
|
|
7
|
+
return `${text.slice(0, max)}\n\n[truncated ${text.length - max} characters]`;
|
|
8
|
+
}
|
|
9
|
+
const isRec = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
10
|
+
const str = (v) => (typeof v === "string" ? v : typeof v === "number" ? String(v) : "");
|
|
11
|
+
const first = (o, keys) => {
|
|
12
|
+
for (const k of keys) {
|
|
13
|
+
const v = str(o[k]);
|
|
14
|
+
if (v) {
|
|
15
|
+
return v;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return "";
|
|
19
|
+
};
|
|
20
|
+
const VERTICALS = ["products", "offers", "listings", "flights", "hotels", "businesses", "cves"];
|
|
21
|
+
function cveLine(o, i) {
|
|
22
|
+
const score = str(o.cvssScore);
|
|
23
|
+
const sev = str(o.severity);
|
|
24
|
+
const affected = Array.isArray(o.affected) ? o.affected.filter(isRec) : [];
|
|
25
|
+
const aff = affected
|
|
26
|
+
.slice(0, 3)
|
|
27
|
+
.map((a) => [str(a.product), str(a.versions), a.fixed ? `fixed ${str(a.fixed)}` : ""].filter(Boolean).join(" "))
|
|
28
|
+
.join("; ");
|
|
29
|
+
const head = [score && `CVSS ${score}`, sev, o.kev === true ? "KEV" : "", str(o.published).slice(0, 10)]
|
|
30
|
+
.filter(Boolean)
|
|
31
|
+
.join(" · ");
|
|
32
|
+
const summary = (str(o.title) || str(o.summary)).slice(0, 220);
|
|
33
|
+
return `${i + 1}. ${str(o.id)}${head ? ` (${head})` : ""}${summary ? `\n ${summary}` : ""}${aff ? `\n affected: ${aff}` : ""}${o.url ? `\n ${str(o.url)}` : ""}`;
|
|
34
|
+
}
|
|
35
|
+
function line(o, i) {
|
|
36
|
+
const title = first(o, ["title", "name", "address", "airline", "productName"]) || "(untitled)";
|
|
37
|
+
const price = first(o, ["priceText", "price", "pricePerNight", "minPrice"]);
|
|
38
|
+
const src = first(o, ["provider", "portal", "vendor", "domain", "source"]);
|
|
39
|
+
const url = first(o, ["url", "link"]);
|
|
40
|
+
const meta = [price, src].filter(Boolean).join(" · ");
|
|
41
|
+
return `${i + 1}. ${title}${meta ? ` (${meta})` : ""}${url ? `\n ${url}` : ""}`;
|
|
42
|
+
}
|
|
43
|
+
/** Compact, model-friendly digest of a search response. */
|
|
44
|
+
export function formatSearch(res) {
|
|
45
|
+
const parts = [];
|
|
46
|
+
for (const key of VERTICALS) {
|
|
47
|
+
const arr = res[key];
|
|
48
|
+
if (Array.isArray(arr) && arr.length) {
|
|
49
|
+
parts.push(`${key} (${arr.length}):\n${arr.filter(isRec).map(key === "cves" ? cveLine : line).join("\n")}`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const results = Array.isArray(res.results) ? res.results : Array.isArray(res.data) ? res.data : [];
|
|
53
|
+
if (results.length) {
|
|
54
|
+
parts.push(`results (${results.length}):\n${results
|
|
55
|
+
.filter(isRec)
|
|
56
|
+
.map((r, i) => {
|
|
57
|
+
const desc = first(r, ["description", "snippet"]).slice(0, 240);
|
|
58
|
+
return `${line(r, i)}${desc ? `\n ${desc}` : ""}`;
|
|
59
|
+
})
|
|
60
|
+
.join("\n")}`);
|
|
61
|
+
}
|
|
62
|
+
return parts.length ? parts.join("\n\n") : "No results.";
|
|
63
|
+
}
|
|
64
|
+
export function formatCredits(res) {
|
|
65
|
+
const c = res.credits;
|
|
66
|
+
if (!isRec(c)) {
|
|
67
|
+
return "";
|
|
68
|
+
}
|
|
69
|
+
const bits = [
|
|
70
|
+
typeof c.charged === "number" ? `charged ${c.charged}` : "",
|
|
71
|
+
typeof c.remaining === "number" ? `${c.remaining} remaining` : "",
|
|
72
|
+
].filter(Boolean);
|
|
73
|
+
return bits.length ? `\n\n[credits: ${bits.join(", ")}]` : "";
|
|
74
|
+
}
|
|
75
|
+
export function formatScrape(res) {
|
|
76
|
+
const d = isRec(res.data) ? res.data : res;
|
|
77
|
+
const title = str(d.title) || (isRec(d.metadata) ? str(d.metadata.title) : "");
|
|
78
|
+
const md = str(d.markdown) || str(d.content);
|
|
79
|
+
return `${title ? `# ${title}\n\n` : ""}${md || "(no content returned)"}`;
|
|
80
|
+
}
|
|
81
|
+
/** Markdown plus a one-line note on how the file was read (and what was not). */
|
|
82
|
+
export function formatParse(res) {
|
|
83
|
+
if (typeof res.id === "string" && res.status === "running") {
|
|
84
|
+
return `Parsing in the background (run ${res.id}). Check it with web_parse_status.`;
|
|
85
|
+
}
|
|
86
|
+
const meta = isRec(res.metadata) && isRec(res.metadata.parse) ? res.metadata.parse : {};
|
|
87
|
+
const notes = [];
|
|
88
|
+
if (typeof meta.kind === "string") {
|
|
89
|
+
notes.push(meta.kind);
|
|
90
|
+
}
|
|
91
|
+
if (typeof meta.pages === "number") {
|
|
92
|
+
notes.push(`${meta.pages} pages`);
|
|
93
|
+
}
|
|
94
|
+
if (Array.isArray(meta.pagesOcr) && meta.pagesOcr.length) {
|
|
95
|
+
notes.push(`OCR on pages ${meta.pagesOcr.join(", ")}`);
|
|
96
|
+
}
|
|
97
|
+
if (Array.isArray(meta.pagesSkipped) && meta.pagesSkipped.length) {
|
|
98
|
+
notes.push(`not read: ${meta.pagesSkipped.filter(isRec).map((s) => `p${s.page} (${str(s.reason)})`).join(", ")}`);
|
|
99
|
+
}
|
|
100
|
+
if (typeof meta.durationSeconds === "number") {
|
|
101
|
+
notes.push(`${Math.round(meta.durationSeconds)}s of audio`);
|
|
102
|
+
}
|
|
103
|
+
const warnings = Array.isArray(meta.warnings) ? meta.warnings.filter((w) => typeof w === "string") : [];
|
|
104
|
+
const head = notes.length ? `[${notes.join("; ")}]\n\n` : "";
|
|
105
|
+
const tail = warnings.length ? `\n\n[warnings: ${warnings.join(" | ")}]` : "";
|
|
106
|
+
return head + formatScrape(res) + tail;
|
|
107
|
+
}
|
|
108
|
+
export function formatMap(res) {
|
|
109
|
+
const links = Array.isArray(res.links) ? res.links : Array.isArray(res.urls) ? res.urls : [];
|
|
110
|
+
const urls = links.map((l) => (isRec(l) ? str(l.url) : str(l))).filter(Boolean);
|
|
111
|
+
return urls.length ? `${urls.length} URLs:\n${urls.join("\n")}` : "No URLs found.";
|
|
112
|
+
}
|
|
113
|
+
export function formatCrawl(res) {
|
|
114
|
+
const pages = Array.isArray(res.data) ? res.data : Array.isArray(res.pages) ? res.pages : [];
|
|
115
|
+
if (!pages.length) {
|
|
116
|
+
return "No pages crawled.";
|
|
117
|
+
}
|
|
118
|
+
const perPage = Math.max(1_000, Math.floor(MAX_OUTPUT_CHARS / pages.length));
|
|
119
|
+
return pages
|
|
120
|
+
.filter(isRec)
|
|
121
|
+
.map((p) => {
|
|
122
|
+
const url = str(p.url) || (isRec(p.metadata) ? str(p.metadata.sourceURL) || str(p.metadata.url) : "");
|
|
123
|
+
return `## ${str(p.title) || url || "page"}\n${url}\n\n${truncate(str(p.markdown), perPage)}`;
|
|
124
|
+
})
|
|
125
|
+
.join("\n\n---\n\n");
|
|
126
|
+
}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
package/dist/server.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { WebcrawlClient } from "@webcrawl/client";
|
|
3
|
+
export type SchemaProfile = "compact" | "full";
|
|
4
|
+
export interface McpServerOptions {
|
|
5
|
+
version?: string;
|
|
6
|
+
schemaProfile?: SchemaProfile;
|
|
7
|
+
}
|
|
8
|
+
export type WebcrawlLikeClient = Pick<WebcrawlClient, "baseUrl" | "search" | "scrape" | "parse" | "run" | "map" | "crawl" | "extract" | "rerank" | "categories"> & {
|
|
9
|
+
targets?: () => Promise<Record<string, unknown>>;
|
|
10
|
+
};
|
|
11
|
+
export declare function resolveSchemaProfile(env?: NodeJS.ProcessEnv): SchemaProfile;
|
|
12
|
+
/**
|
|
13
|
+
* Builds the webcrawl MCP server around an API or local engine client. Shared by
|
|
14
|
+
* the stdio binary (`npx webcrawl-mcp`) and the hosted Streamable HTTP endpoint (`webcrawl.app/mcp`).
|
|
15
|
+
*/
|
|
16
|
+
export declare function createWebcrawlMcpServer(client: WebcrawlLikeClient, versionOrOpts?: string | McpServerOptions): McpServer;
|
|
17
|
+
/** Back-compat alias used by the stdio binary. */
|
|
18
|
+
export declare const createServer: typeof createWebcrawlMcpServer;
|
package/dist/server.js
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { describeError } from "@webcrawl/client";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { formatCredits, formatCrawl, formatMap, formatParse, formatScrape, formatSearch, truncate } from "./format.js";
|
|
5
|
+
const INSTRUCTIONS = `Webcrawl Anything gives you live web & target inspection access (zero-key local engine or webcrawl.app cloud).
|
|
6
|
+
- web_search: find pages or structured results. Set category for flights, hotels, real-estate, shopping, news, local business, security (CVEs), academic or tech. Results are localised to the user.
|
|
7
|
+
- web_scrape: read one URL as clean markdown with target evidence (websites, SPAs, GitHub, ArXiv, llms.txt).
|
|
8
|
+
- web_parse: read a PDF, Office file, YouTube video, image, or audio/video at a URL as markdown (OCR and transcripts included). For long files set background and poll web_parse_status.
|
|
9
|
+
- web_map: list the URLs on a site before deciding what to scrape.
|
|
10
|
+
- web_crawl: read several pages of one site (max 25).
|
|
11
|
+
- web_extract: pull structured JSON from one URL with a prompt or schema.
|
|
12
|
+
- web_rerank: order your own documents by relevance to a query.
|
|
13
|
+
- search_categories: list valid categories, subcategories, filters, and supported crawl targets.`;
|
|
14
|
+
const ok = (text) => ({ content: [{ type: "text", text: truncate(text) }] });
|
|
15
|
+
export function resolveSchemaProfile(env = process.env) {
|
|
16
|
+
const raw = (env.WEBCRAWL_MCP_SCHEMA_PROFILE ||
|
|
17
|
+
env.WEBCRAWL_MCP_INPUT_SCHEMA_PROFILE ||
|
|
18
|
+
"full")
|
|
19
|
+
.trim()
|
|
20
|
+
.toLowerCase();
|
|
21
|
+
return raw === "compact" ? "compact" : "full";
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Builds the webcrawl MCP server around an API or local engine client. Shared by
|
|
25
|
+
* the stdio binary (`npx webcrawl-mcp`) and the hosted Streamable HTTP endpoint (`webcrawl.app/mcp`).
|
|
26
|
+
*/
|
|
27
|
+
export function createWebcrawlMcpServer(client, versionOrOpts = "0.1.0") {
|
|
28
|
+
const opts = typeof versionOrOpts === "string" ? { version: versionOrOpts } : versionOrOpts;
|
|
29
|
+
const version = opts.version ?? "0.1.0";
|
|
30
|
+
const compact = (opts.schemaProfile ?? resolveSchemaProfile()) === "compact";
|
|
31
|
+
const desc = (schema, text) => compact ? schema : schema.describe(text);
|
|
32
|
+
const server = new McpServer({ name: "webcrawl", version }, { instructions: INSTRUCTIONS });
|
|
33
|
+
const run = async (fn) => {
|
|
34
|
+
try {
|
|
35
|
+
return ok(await fn());
|
|
36
|
+
}
|
|
37
|
+
catch (e) {
|
|
38
|
+
return { isError: true, content: [{ type: "text", text: describeError(e, client.baseUrl) }] };
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
server.registerTool("web_search", {
|
|
42
|
+
title: "Web search",
|
|
43
|
+
description: compact
|
|
44
|
+
? "Search the web or structured verticals (shopping, flights, hotels, real-estate, business, security CVEs, academic, tech)."
|
|
45
|
+
: "Search the web. Use `category` for structured results (flights, hotels, real-estate, shopping with prices, news, local business, security: CVE records with CVSS, affected and fixed versions). Results are localised to the user unless a place is given.",
|
|
46
|
+
inputSchema: {
|
|
47
|
+
query: desc(z.string().min(1), "What to search for"),
|
|
48
|
+
category: desc(z.string().optional(), "e.g. general, news, shopping, flights, hotels, real-estate, business, security, academic, tech. See search_categories."),
|
|
49
|
+
subcategory: desc(z.string().optional(), "e.g. buy or rent for real-estate, cve for security"),
|
|
50
|
+
limit: desc(z.number().int().min(1).max(20).optional(), "Max results, default 5"),
|
|
51
|
+
country: desc(z.string().optional(), "ISO country code, e.g. AU"),
|
|
52
|
+
city: z.string().optional(),
|
|
53
|
+
sites: desc(z.array(z.string()).optional(), "Only these domains, e.g. [\"booking.com\"]"),
|
|
54
|
+
include_content: desc(z.boolean().optional(), "Also fetch page markdown for each result (slower, more credits)"),
|
|
55
|
+
},
|
|
56
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
57
|
+
}, ({ query, category, subcategory, limit, country, city, sites, include_content }) => run(async () => {
|
|
58
|
+
const res = await client.search({
|
|
59
|
+
query,
|
|
60
|
+
category,
|
|
61
|
+
subcategory,
|
|
62
|
+
limit,
|
|
63
|
+
country,
|
|
64
|
+
city,
|
|
65
|
+
sites,
|
|
66
|
+
scrapeContent: include_content,
|
|
67
|
+
});
|
|
68
|
+
return formatSearch(res) + formatCredits(res);
|
|
69
|
+
}));
|
|
70
|
+
server.registerTool("web_scrape", {
|
|
71
|
+
title: "Scrape a page",
|
|
72
|
+
description: compact
|
|
73
|
+
? "Fetch one URL as clean markdown."
|
|
74
|
+
: "Fetch one URL and return its main content as markdown. Handles JavaScript-heavy and bot-protected pages.",
|
|
75
|
+
inputSchema: {
|
|
76
|
+
url: z.string().url(),
|
|
77
|
+
no_cache: desc(z.boolean().optional(), "Skip cached copy"),
|
|
78
|
+
},
|
|
79
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
80
|
+
}, ({ url, no_cache }) => run(async () => {
|
|
81
|
+
const res = await client.scrape({ url, formats: ["markdown"], noCache: no_cache });
|
|
82
|
+
return formatScrape(res) + formatCredits(res);
|
|
83
|
+
}));
|
|
84
|
+
server.registerTool("web_parse", {
|
|
85
|
+
title: "Read a file",
|
|
86
|
+
description: compact
|
|
87
|
+
? "Read a PDF, Office file, YouTube link, image, or audio/video URL as Markdown."
|
|
88
|
+
: "Read a PDF, Word/Excel/PowerPoint file, image, or audio/video file at a URL and return Markdown. Scanned PDF pages are OCR'd; audio/video is transcribed with timestamps. Pages it could not read are listed, never guessed.",
|
|
89
|
+
inputSchema: {
|
|
90
|
+
url: z.string().url(),
|
|
91
|
+
ocr: desc(z.enum(["auto", "force", "off"]).optional(), "auto (default) OCRs only scanned pages"),
|
|
92
|
+
pages: desc(z.array(z.number().int().min(1)).max(2000).optional(), "1-indexed PDF pages to read"),
|
|
93
|
+
language: desc(z.string().max(8).optional(), "Spoken language hint, e.g. en"),
|
|
94
|
+
background: desc(z.boolean().optional(), "Run as a background job for long files; returns a run id"),
|
|
95
|
+
},
|
|
96
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
97
|
+
}, ({ url, ocr, pages, language, background }) => run(async () => {
|
|
98
|
+
const res = await client.parse({ url, ocr, pages, language, async: background });
|
|
99
|
+
return formatParse(res) + formatCredits(res);
|
|
100
|
+
}));
|
|
101
|
+
server.registerTool("web_parse_status", {
|
|
102
|
+
title: "Check a file read",
|
|
103
|
+
description: "Check a background web_parse run and return its Markdown when it has finished.",
|
|
104
|
+
inputSchema: { id: z.string().min(1) },
|
|
105
|
+
annotations: { readOnlyHint: true, openWorldHint: false },
|
|
106
|
+
}, ({ id }) => run(async () => {
|
|
107
|
+
const res = await client.run(id);
|
|
108
|
+
const result = res.result && typeof res.result === "object"
|
|
109
|
+
? res.result
|
|
110
|
+
: null;
|
|
111
|
+
if (!result) {
|
|
112
|
+
return `Run ${id} is ${String(res.status ?? "running")}.`;
|
|
113
|
+
}
|
|
114
|
+
return formatParse(result) + formatCredits(result);
|
|
115
|
+
}));
|
|
116
|
+
server.registerTool("web_map", {
|
|
117
|
+
title: "Map a site",
|
|
118
|
+
description: compact
|
|
119
|
+
? "List URLs on a website."
|
|
120
|
+
: "List the URLs on a website (sitemap, robots and links). Use before scraping to pick the right pages.",
|
|
121
|
+
inputSchema: {
|
|
122
|
+
url: z.string().url(),
|
|
123
|
+
search: desc(z.string().optional(), "Only URLs matching this text"),
|
|
124
|
+
limit: z.number().int().min(1).max(5000).optional(),
|
|
125
|
+
},
|
|
126
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
127
|
+
}, ({ url, search, limit }) => run(async () => {
|
|
128
|
+
const res = await client.map({ url, search, limit });
|
|
129
|
+
return formatMap(res) + formatCredits(res);
|
|
130
|
+
}));
|
|
131
|
+
server.registerTool("web_crawl", {
|
|
132
|
+
title: "Crawl a site",
|
|
133
|
+
description: compact
|
|
134
|
+
? "Crawl up to 25 pages starting from a URL."
|
|
135
|
+
: "Read several pages starting from a URL and return each as markdown. Capped at 25 pages.",
|
|
136
|
+
inputSchema: {
|
|
137
|
+
url: z.string().url(),
|
|
138
|
+
max_pages: desc(z.number().int().min(1).max(25).optional(), "Default 5"),
|
|
139
|
+
},
|
|
140
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
141
|
+
}, ({ url, max_pages }) => run(async () => {
|
|
142
|
+
const res = await client.crawl({ url, maxPages: max_pages });
|
|
143
|
+
return formatCrawl(res) + formatCredits(res);
|
|
144
|
+
}));
|
|
145
|
+
server.registerTool("web_extract", {
|
|
146
|
+
title: "Extract structured data",
|
|
147
|
+
description: compact
|
|
148
|
+
? "Extract structured JSON from a URL."
|
|
149
|
+
: "Pull structured JSON from one URL. Describe what you want with `prompt`, or pass a JSON `schema`.",
|
|
150
|
+
inputSchema: {
|
|
151
|
+
url: z.string().url(),
|
|
152
|
+
prompt: z.string().optional(),
|
|
153
|
+
schema: desc(z.record(z.string(), z.any()).optional(), "JSON Schema for the output"),
|
|
154
|
+
},
|
|
155
|
+
annotations: { readOnlyHint: true, openWorldHint: true },
|
|
156
|
+
}, ({ url, prompt, schema }) => run(async () => {
|
|
157
|
+
const res = await client.extract({ url, prompt, schema });
|
|
158
|
+
return JSON.stringify(res.data ?? res, null, 2) + formatCredits(res);
|
|
159
|
+
}));
|
|
160
|
+
server.registerTool("web_rerank", {
|
|
161
|
+
title: "Rerank documents",
|
|
162
|
+
description: compact
|
|
163
|
+
? "Order documents by relevance to a query."
|
|
164
|
+
: "Order your own documents by relevance to a query. Returns the best matches first.",
|
|
165
|
+
inputSchema: {
|
|
166
|
+
query: z.string().min(1),
|
|
167
|
+
documents: z.array(z.string()).min(1).max(200),
|
|
168
|
+
top_k: z.number().int().min(1).optional(),
|
|
169
|
+
},
|
|
170
|
+
annotations: { readOnlyHint: true, openWorldHint: false },
|
|
171
|
+
}, ({ query, documents, top_k }) => run(async () => {
|
|
172
|
+
const res = await client.rerank({ query, documents, topK: top_k });
|
|
173
|
+
const items = Array.isArray(res.results)
|
|
174
|
+
? res.results
|
|
175
|
+
: Array.isArray(res.data)
|
|
176
|
+
? res.data
|
|
177
|
+
: [];
|
|
178
|
+
const lines = items.map((r, i) => {
|
|
179
|
+
const o = (r ?? {});
|
|
180
|
+
const text = String(o.description ?? o.markdown ?? o.title ?? "").slice(0, 200);
|
|
181
|
+
const score = typeof o.score === "number" ? ` [${o.score.toFixed(3)}]` : "";
|
|
182
|
+
return `${i + 1}.${score} ${text}`;
|
|
183
|
+
});
|
|
184
|
+
return (lines.length ? lines.join("\n") : JSON.stringify(res, null, 2)) + formatCredits(res);
|
|
185
|
+
}));
|
|
186
|
+
server.registerTool("search_categories", {
|
|
187
|
+
title: "Search categories",
|
|
188
|
+
description: "List the search categories, subcategories and filters web_search supports.",
|
|
189
|
+
inputSchema: {},
|
|
190
|
+
annotations: { readOnlyHint: true, openWorldHint: false },
|
|
191
|
+
}, () => run(async () => JSON.stringify(await client.categories(), null, 2)));
|
|
192
|
+
return server;
|
|
193
|
+
}
|
|
194
|
+
/** Back-compat alias used by the stdio binary. */
|
|
195
|
+
export const createServer = createWebcrawlMcpServer;
|
package/package.json
CHANGED
|
@@ -1,6 +1,61 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@webcrawl/mcp",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.2.2",
|
|
4
|
+
"description": "MCP server for Webcrawl Anything — zero-key local engine or webcrawl.app cloud (search, scrape, parse PDF/Office/YouTube, map, crawl, extract, rerank)",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Brainfish AI <support@webcrawl.app> (https://webcrawl.app)",
|
|
8
|
+
"homepage": "https://webcrawl.app",
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "git+https://github.com/brainfish-ai/webcrawl-anything.git",
|
|
12
|
+
"directory": "packages/mcp"
|
|
13
|
+
},
|
|
14
|
+
"bugs": {
|
|
15
|
+
"url": "https://github.com/brainfish-ai/webcrawl-anything/issues"
|
|
16
|
+
},
|
|
17
|
+
"keywords": [
|
|
18
|
+
"mcp",
|
|
19
|
+
"model-context-protocol",
|
|
20
|
+
"webcrawl",
|
|
21
|
+
"web-search",
|
|
22
|
+
"web-scraper",
|
|
23
|
+
"pdf-parser",
|
|
24
|
+
"claude-code",
|
|
25
|
+
"cursor",
|
|
26
|
+
"codex",
|
|
27
|
+
"ai-agents"
|
|
28
|
+
],
|
|
29
|
+
"publishConfig": {
|
|
30
|
+
"access": "public"
|
|
31
|
+
},
|
|
32
|
+
"bin": {
|
|
33
|
+
"webcrawl-mcp": "./dist/bin.js"
|
|
34
|
+
},
|
|
35
|
+
"files": [
|
|
36
|
+
"dist",
|
|
37
|
+
"README.md"
|
|
38
|
+
],
|
|
39
|
+
"main": "./dist/index.js",
|
|
40
|
+
"types": "./dist/index.d.ts",
|
|
41
|
+
"exports": {
|
|
42
|
+
".": {
|
|
43
|
+
"development": "./src/index.ts",
|
|
44
|
+
"types": "./dist/index.d.ts",
|
|
45
|
+
"default": "./dist/index.js"
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
"scripts": {
|
|
49
|
+
"build": "tsc -p tsconfig.build.json",
|
|
50
|
+
"prepack": "npm run build",
|
|
51
|
+
"test": "node --conditions=development --import tsx --test test/*.test.ts"
|
|
52
|
+
},
|
|
53
|
+
"dependencies": {
|
|
54
|
+
"@modelcontextprotocol/sdk": "^1.32.2",
|
|
55
|
+
"@webcrawl/client": "^1.2.2",
|
|
56
|
+
"zod": "^4.0.0"
|
|
57
|
+
},
|
|
58
|
+
"engines": {
|
|
59
|
+
"node": ">=20"
|
|
60
|
+
}
|
|
61
|
+
}
|