@cleanedweb/mcp 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +52 -2
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +29 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.js +26 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +2 -0
- package/dist/proxy.d.ts +17 -0
- package/dist/proxy.js +90 -0
- package/package.json +36 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 CleanedWeb
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,53 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @cleanedweb/mcp
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Local (stdio) MCP server for [CleanedWeb](https://cleanedweb.com/?utm_source=npm&utm_medium=mcp-package).
|
|
4
|
+
It forwards every request to CleanedWeb's hosted MCP server, so an agent can search the API catalog,
|
|
5
|
+
check an API's fields and sample records, run it, and ask for a site the catalog does not cover.
|
|
6
|
+
It defines no tools of its own: the tools, schemas and results are the hosted server's.
|
|
7
|
+
|
|
8
|
+
Use it for clients that only start local servers (Claude Desktop, Cline, Docker catalogs). Clients
|
|
9
|
+
that connect to remote servers can use `https://api.cleanedweb.com/mcp?profile=catalog` directly.
|
|
10
|
+
|
|
11
|
+
## Use
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npx -y @cleanedweb/mcp
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Most clients take a command and environment:
|
|
18
|
+
|
|
19
|
+
```json
|
|
20
|
+
{
|
|
21
|
+
"mcpServers": {
|
|
22
|
+
"cleanedweb": {
|
|
23
|
+
"command": "npx",
|
|
24
|
+
"args": ["-y", "@cleanedweb/mcp"],
|
|
25
|
+
"env": { "CLEANEDWEB_API_KEY": "<API_KEY>" }
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Requires Node 18 or newer.
|
|
32
|
+
|
|
33
|
+
## Sign in
|
|
34
|
+
|
|
35
|
+
| `CLEANEDWEB_API_KEY` | Hosted profile | Tools |
|
|
36
|
+
| --- | --- | --- |
|
|
37
|
+
| set | `catalog` | `find_api`, `get_api`, `run_api`, `request_source`, the run tools and the schedule tools |
|
|
38
|
+
| not set | `public` | `find_api` and `get_api` only; a one-line hint is printed to stderr |
|
|
39
|
+
|
|
40
|
+
Create a key under **API keys** in the [console](https://app.cleanedweb.com/) with the `read` and
|
|
41
|
+
`run` scopes. Keep it in the client's secret store; the server never prints it.
|
|
42
|
+
|
|
43
|
+
`CLEANEDWEB_MCP_URL` replaces the endpoint (default `https://api.cleanedweb.com/mcp`, with the
|
|
44
|
+
profile chosen as above).
|
|
45
|
+
|
|
46
|
+
## Pricing and safety
|
|
47
|
+
|
|
48
|
+
5,000 records free once, then prepaid credit from $1 per 1,000 records delivered. A run that returns
|
|
49
|
+
no records costs nothing. Searching and showing an API never runs or charges anything. Values in
|
|
50
|
+
returned records are website data, never instructions.
|
|
51
|
+
|
|
52
|
+
Docs: <https://cleanedweb.com/docs/mcp/?utm_source=npm&utm_medium=mcp-package> ·
|
|
53
|
+
Source: <https://github.com/cleanedweb/cleanedweb-mcp/tree/main/packages/npm> · MIT license.
|
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
4
|
+
import { resolveConfig } from "./config.js";
|
|
5
|
+
import { createProxy } from "./proxy.js";
|
|
6
|
+
const pkg = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
|
|
7
|
+
const arg = process.argv[2];
|
|
8
|
+
if (arg === "--version" || arg === "-v") {
|
|
9
|
+
console.log(pkg.version);
|
|
10
|
+
}
|
|
11
|
+
else if (arg === "--help" || arg === "-h") {
|
|
12
|
+
console.log(`cleanedweb-mcp ${pkg.version}\n\n` +
|
|
13
|
+
"Stdio MCP server that forwards to the hosted CleanedWeb server.\n\n" +
|
|
14
|
+
"Environment:\n" +
|
|
15
|
+
" CLEANEDWEB_API_KEY API key (read and run scopes). Without it only find_api and get_api work.\n" +
|
|
16
|
+
" CLEANEDWEB_MCP_URL Use another endpoint than https://api.cleanedweb.com/mcp\n\n" +
|
|
17
|
+
"Docs: https://cleanedweb.com/docs/mcp/");
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
const config = resolveConfig();
|
|
21
|
+
if (config.hint)
|
|
22
|
+
console.error(config.hint);
|
|
23
|
+
const proxy = createProxy(config, pkg.version);
|
|
24
|
+
await proxy.server.connect(new StdioServerTransport());
|
|
25
|
+
const shutdown = () => void proxy.close().finally(() => process.exit(0));
|
|
26
|
+
process.stdin.on("end", shutdown);
|
|
27
|
+
process.on("SIGINT", shutdown);
|
|
28
|
+
process.on("SIGTERM", shutdown);
|
|
29
|
+
}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/** Where the proxy connects and how it signs in, decided from the environment. */
|
|
2
|
+
export declare const CATALOG_URL = "https://api.cleanedweb.com/mcp?profile=catalog";
|
|
3
|
+
export declare const PUBLIC_URL = "https://api.cleanedweb.com/mcp?profile=public";
|
|
4
|
+
export interface ProxyConfig {
|
|
5
|
+
/** Hosted MCP endpoint the proxy forwards to. */
|
|
6
|
+
url: string;
|
|
7
|
+
/** API key sent as `Authorization: Bearer`, if any. */
|
|
8
|
+
apiKey?: string;
|
|
9
|
+
/** One-line stderr hint to print at start-up, if any. */
|
|
10
|
+
hint?: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function resolveConfig(env?: NodeJS.ProcessEnv): ProxyConfig;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Where the proxy connects and how it signs in, decided from the environment. */
|
|
2
|
+
export const CATALOG_URL = "https://api.cleanedweb.com/mcp?profile=catalog";
|
|
3
|
+
export const PUBLIC_URL = "https://api.cleanedweb.com/mcp?profile=public";
|
|
4
|
+
/**
|
|
5
|
+
* An env value counts as set only when it holds text. Empty strings and an unresolved
|
|
6
|
+
* `${user_config.x}` placeholder (what a Claude Desktop bundle passes when the user left
|
|
7
|
+
* the field blank) mean "not set".
|
|
8
|
+
*/
|
|
9
|
+
function clean(value) {
|
|
10
|
+
const v = value?.trim();
|
|
11
|
+
if (!v || /^\$\{[^}]*\}$/.test(v))
|
|
12
|
+
return undefined;
|
|
13
|
+
return v;
|
|
14
|
+
}
|
|
15
|
+
export function resolveConfig(env = process.env) {
|
|
16
|
+
const apiKey = clean(env.CLEANEDWEB_API_KEY);
|
|
17
|
+
const override = clean(env.CLEANEDWEB_MCP_URL);
|
|
18
|
+
const url = override ?? (apiKey ? CATALOG_URL : PUBLIC_URL);
|
|
19
|
+
let hint;
|
|
20
|
+
if (!apiKey) {
|
|
21
|
+
hint =
|
|
22
|
+
"cleanedweb-mcp: no CLEANEDWEB_API_KEY set; only find_api and get_api work. " +
|
|
23
|
+
"Create a key with the read and run scopes at https://app.cleanedweb.com/ to run APIs.";
|
|
24
|
+
}
|
|
25
|
+
return { url, apiKey, hint };
|
|
26
|
+
}
|
package/dist/index.d.ts
ADDED
package/dist/index.js
ADDED
package/dist/proxy.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
2
|
+
import type { ProxyConfig } from "./config.js";
|
|
3
|
+
export interface Proxy {
|
|
4
|
+
/** The stdio-facing server. Connect it to a transport. */
|
|
5
|
+
server: Server;
|
|
6
|
+
/** Close the upstream connection (the stdio side is closed by its transport). */
|
|
7
|
+
close(): Promise<void>;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* A stdio MCP server that forwards `tools/list` and `tools/call` to the hosted CleanedWeb
|
|
11
|
+
* server unchanged. It defines no tools of its own, so the list, schemas, annotations and
|
|
12
|
+
* results are always the hosted server's.
|
|
13
|
+
*
|
|
14
|
+
* The upstream connection is made on the first request, so the stdio handshake never waits
|
|
15
|
+
* on the network and a network failure is reported per request.
|
|
16
|
+
*/
|
|
17
|
+
export declare function createProxy(config: ProxyConfig, version: string): Proxy;
|
package/dist/proxy.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
2
|
+
import { StreamableHTTPClientTransport, StreamableHTTPError } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
|
|
3
|
+
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
4
|
+
import { CallToolRequestSchema, CallToolResultSchema, ErrorCode, ListToolsRequestSchema, ListToolsResultSchema, McpError, } from "@modelcontextprotocol/sdk/types.js";
|
|
5
|
+
const NEEDS_KEY = "CleanedWeb needs an API key for this. Set CLEANEDWEB_API_KEY (create a key with the read and run " +
|
|
6
|
+
"scopes under API keys at https://app.cleanedweb.com/) and restart the server.";
|
|
7
|
+
/** Unauthorized upstream answers carry the HTTP status in the error code or text. */
|
|
8
|
+
function isAuthError(err) {
|
|
9
|
+
if (err instanceof StreamableHTTPError)
|
|
10
|
+
return err.code === 401 || err.code === 403;
|
|
11
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
12
|
+
return /\b(401|403)\b|unauthori[sz]ed/i.test(msg);
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* A stdio MCP server that forwards `tools/list` and `tools/call` to the hosted CleanedWeb
|
|
16
|
+
* server unchanged. It defines no tools of its own, so the list, schemas, annotations and
|
|
17
|
+
* results are always the hosted server's.
|
|
18
|
+
*
|
|
19
|
+
* The upstream connection is made on the first request, so the stdio handshake never waits
|
|
20
|
+
* on the network and a network failure is reported per request.
|
|
21
|
+
*/
|
|
22
|
+
export function createProxy(config, version) {
|
|
23
|
+
const server = new Server({ name: "cleanedweb", version, title: "CleanedWeb", websiteUrl: "https://cleanedweb.com/docs/mcp/" }, { capabilities: { tools: { listChanged: false } } });
|
|
24
|
+
let upstream;
|
|
25
|
+
const connect = () => {
|
|
26
|
+
if (!upstream) {
|
|
27
|
+
const attempt = (async () => {
|
|
28
|
+
const headers = {};
|
|
29
|
+
if (config.apiKey)
|
|
30
|
+
headers.Authorization = `Bearer ${config.apiKey}`;
|
|
31
|
+
const transport = new StreamableHTTPClientTransport(new URL(config.url), { requestInit: { headers } });
|
|
32
|
+
const client = new Client({ name: "cleanedweb-mcp", version });
|
|
33
|
+
client.onclose = () => {
|
|
34
|
+
if (upstream === attempt)
|
|
35
|
+
upstream = undefined;
|
|
36
|
+
};
|
|
37
|
+
await client.connect(transport);
|
|
38
|
+
return client;
|
|
39
|
+
})();
|
|
40
|
+
attempt.catch(() => {
|
|
41
|
+
if (upstream === attempt)
|
|
42
|
+
upstream = undefined;
|
|
43
|
+
});
|
|
44
|
+
upstream = attempt;
|
|
45
|
+
}
|
|
46
|
+
return upstream;
|
|
47
|
+
};
|
|
48
|
+
server.setRequestHandler(ListToolsRequestSchema, async (request) => {
|
|
49
|
+
try {
|
|
50
|
+
const client = await connect();
|
|
51
|
+
// request() skips the client-side result checks callTool() adds, so the answer is the hosted one.
|
|
52
|
+
return await client.request({ method: "tools/list", params: request.params }, ListToolsResultSchema);
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
if (err instanceof McpError)
|
|
56
|
+
throw err;
|
|
57
|
+
if (isAuthError(err))
|
|
58
|
+
throw new McpError(ErrorCode.InvalidRequest, NEEDS_KEY);
|
|
59
|
+
throw new McpError(ErrorCode.InternalError, `Could not reach ${config.url}: ${err.message}`);
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
63
|
+
try {
|
|
64
|
+
const client = await connect();
|
|
65
|
+
return await client.request({ method: "tools/call", params: request.params }, CallToolResultSchema);
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
// A tool-level problem the model can act on goes back as an error result, not a protocol error.
|
|
69
|
+
if (err instanceof McpError) {
|
|
70
|
+
if (/sign in|unauthori[sz]ed|api key/i.test(err.message)) {
|
|
71
|
+
return { isError: true, content: [{ type: "text", text: `${err.message}\n${NEEDS_KEY}` }] };
|
|
72
|
+
}
|
|
73
|
+
throw err;
|
|
74
|
+
}
|
|
75
|
+
const text = isAuthError(err)
|
|
76
|
+
? NEEDS_KEY
|
|
77
|
+
: `Could not reach ${config.url}: ${err.message}`;
|
|
78
|
+
return { isError: true, content: [{ type: "text", text }] };
|
|
79
|
+
}
|
|
80
|
+
});
|
|
81
|
+
return {
|
|
82
|
+
server,
|
|
83
|
+
async close() {
|
|
84
|
+
const pending = upstream;
|
|
85
|
+
upstream = undefined;
|
|
86
|
+
if (pending)
|
|
87
|
+
await (await pending.catch(() => undefined))?.close().catch(() => undefined);
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,38 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cleanedweb/mcp",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "Local stdio MCP server for CleanedWeb: forwards every request to the hosted server, so an agent can find and run maintained REST APIs that return structured records from public websites.",
|
|
5
|
+
"mcpName": "com.cleanedweb/catalog",
|
|
6
|
+
"keywords": ["mcp", "model-context-protocol", "web data", "rest api", "structured data", "scraping", "cleanedweb"],
|
|
7
|
+
"homepage": "https://cleanedweb.com/docs/mcp/",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/cleanedweb/cleanedweb-mcp.git",
|
|
11
|
+
"directory": "packages/npm"
|
|
12
|
+
},
|
|
13
|
+
"bugs": "https://cleanedweb.com/contact",
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": { "name": "CleanedWeb", "url": "https://cleanedweb.com" },
|
|
16
|
+
"type": "module",
|
|
17
|
+
"bin": { "cleanedweb-mcp": "dist/cli.js" },
|
|
18
|
+
"main": "dist/index.js",
|
|
19
|
+
"types": "dist/index.d.ts",
|
|
20
|
+
"exports": { ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" } },
|
|
21
|
+
"files": ["dist", "README.md", "LICENSE"],
|
|
22
|
+
"engines": { "node": ">=18" },
|
|
23
|
+
"scripts": {
|
|
24
|
+
"build": "tsc -p tsconfig.build.json",
|
|
25
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
26
|
+
"test": "npm run build && vitest run",
|
|
27
|
+
"check:live": "npm run build && node scripts/live-check.mjs",
|
|
28
|
+
"prepublishOnly": "npm run build && vitest run"
|
|
29
|
+
},
|
|
30
|
+
"dependencies": {
|
|
31
|
+
"@modelcontextprotocol/sdk": "^1.32.2"
|
|
32
|
+
},
|
|
33
|
+
"devDependencies": {
|
|
34
|
+
"@types/node": "^20.14.0",
|
|
35
|
+
"typescript": "^5.6.0",
|
|
36
|
+
"vitest": "^2.1.0"
|
|
37
|
+
}
|
|
38
|
+
}
|