crawlforge-mcp-server 5.10.0 → 6.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/package.json +7 -5
- package/server.js +210 -251
- package/src/cli/commands/login.js +176 -0
- package/src/cli/index.js +2 -0
- package/src/core/ActionExecutor.js +1 -1
- package/src/core/AuthManager.js +19 -6
- package/src/core/ChangeTracker.js +1 -1
- package/src/core/ElicitationHelper.js +192 -62
- package/src/core/SamplingClient.js +8 -2
- package/src/core/analysis/ContentAnalyzer.js +1 -1
- package/src/core/processing/BrowserProcessor.js +1 -1
- package/src/core/processing/ContentProcessor.js +1 -1
- package/src/core/processing/PDFProcessor.js +2 -2
- package/src/server/registerTool.js +1 -1
- package/src/server/requestContext.js +50 -0
- package/src/server/specHygiene.js +17 -22
- package/src/server/transports/stdio.js +2 -3
- package/src/server/transports/streamableHttp.js +142 -67
- package/src/server/withAuth.js +47 -9
- package/src/tools/advanced/batchScrape/index.js +29 -19
- package/src/tools/agent/agent.js +9 -4
- package/src/tools/crawl/crawlDeep.js +14 -8
- package/src/tools/extract/analyzeContent.js +1 -1
- package/src/tools/extract/extractContent.js +1 -1
- package/src/tools/extract/extractStructured.js +62 -43
- package/src/tools/extract/processDocument.js +1 -1
- package/src/tools/extract/summarizeContent.js +1 -1
- package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
- package/src/tools/research/deepResearch.js +11 -5
- package/src/tools/tracking/trackChanges/schema.js +4 -4
- package/src/utils/HumanBehaviorSimulator.js +7 -7
- package/src/server/taskSupport.js +0 -233
- package/src/server/transports/http.js +0 -22
|
@@ -69,3 +69,53 @@ export function setActualCost(n) {
|
|
|
69
69
|
export function reportedActualCost() {
|
|
70
70
|
return requestContext.getStore()?.actualCost ?? null;
|
|
71
71
|
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Record the McpServer instance that is actually serving this request, and the
|
|
75
|
+
* wire era it speaks.
|
|
76
|
+
*
|
|
77
|
+
* Neither HTTP leg serves from the top-level McpServer that server.js
|
|
78
|
+
* registers everything on: the 2025-era path connects one clone per session,
|
|
79
|
+
* and the modern leg builds a fresh clone per request (see
|
|
80
|
+
* transports/streamableHttp.js). Only a clone is ever `.connect()`ed, so only a
|
|
81
|
+
* clone has a negotiated protocol version, the client's declared capabilities,
|
|
82
|
+
* and a channel to send a server-to-client request on. The template has none of
|
|
83
|
+
* those, which is why anything reading them off it (ElicitationHelper) got
|
|
84
|
+
* `undefined` on every HTTP request.
|
|
85
|
+
*
|
|
86
|
+
* Stdio stamps nothing: there the top-level instance IS the connected one, and
|
|
87
|
+
* the accessors below return null so callers fall back to it.
|
|
88
|
+
*
|
|
89
|
+
* @param {object|null} server the serving McpServer
|
|
90
|
+
* @param {'legacy'|'modern'|null} [era] the wire era it serves
|
|
91
|
+
*/
|
|
92
|
+
export function setServingServer(server, era = null) {
|
|
93
|
+
const store = requestContext.getStore();
|
|
94
|
+
if (!store) return;
|
|
95
|
+
store.servingServer = server ?? null;
|
|
96
|
+
store.servingEra = era;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The McpServer serving this request, or null on stdio / outside a context. */
|
|
100
|
+
export function servingServer() {
|
|
101
|
+
return requestContext.getStore()?.servingServer ?? null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** The wire era serving this request: 'legacy' | 'modern' | null (stdio). */
|
|
105
|
+
export function servingEra() {
|
|
106
|
+
return requestContext.getStore()?.servingEra ?? null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The JSON-RPC id of the request being served, or null when unknown.
|
|
111
|
+
*
|
|
112
|
+
* A server-to-client request sent from inside a tool has to say which inbound
|
|
113
|
+
* request it belongs to: the 2025-era streamable HTTP transport routes an
|
|
114
|
+
* unrelated request to the standalone GET SSE stream and silently DROPS it when
|
|
115
|
+
* the client never opened one, which turns an elicitation prompt into a
|
|
116
|
+
* 60-second stall before it fails open. withAuth stamps this from the SDK's
|
|
117
|
+
* per-request `ctx`; stdio has no streams to pick between and ignores it.
|
|
118
|
+
*/
|
|
119
|
+
export function servingRequestId() {
|
|
120
|
+
return requestContext.getStore()?.servingRequestId ?? null;
|
|
121
|
+
}
|
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* specHygiene — Phase 6 protocol-hygiene wrapper.
|
|
3
3
|
*
|
|
4
|
-
* Applies
|
|
4
|
+
* Applies four MCP wire-level upgrades to an already-registered McpServer
|
|
5
5
|
* without changing how tools/prompts are declared anywhere else:
|
|
6
6
|
*
|
|
7
7
|
* 1. JSON Schema 2020-12 dialect stamping on every tool's inputSchema /
|
|
8
|
-
* outputSchema
|
|
9
|
-
*
|
|
8
|
+
* outputSchema. Under the v2 SDK + zod 4 the conversion already emits
|
|
9
|
+
* 2020-12 with `$defs`, so the `definitions` -> `$defs` rewrite below is
|
|
10
|
+
* now defensive rather than load-bearing; it is kept because the stamping
|
|
11
|
+
* and ordering passes still walk the same tree (Phase 4.1 kept behaviour
|
|
12
|
+
* identical — removing the rewrite is a separate, verifiable change).
|
|
10
13
|
* 2. Deterministic (alphabetical) tool ordering in tools/list, so clients
|
|
11
14
|
* that prompt-cache tools/list get stable hits across restarts.
|
|
12
15
|
* 3. SEP-973 icons metadata on every tool/prompt lacking one.
|
|
13
16
|
* 4. SEP-2549-style cache hints on tools/call results for a fixed
|
|
14
17
|
* allowlist of read-only, idempotent tools (see note below).
|
|
15
18
|
*
|
|
16
|
-
* Wiring: SDK
|
|
19
|
+
* Wiring: the SDK has no plugin hook for this, so applySpecHygiene() must be
|
|
17
20
|
* called once, after all server.registerTool()/registerPrompt() calls and
|
|
18
21
|
* before transport.connect(). It reaches into `server.server` (the
|
|
19
22
|
* underlying Protocol), captures the ListTools/CallTool/ListPrompts handlers
|
|
@@ -22,26 +25,18 @@
|
|
|
22
25
|
* handler for the same method (confirmed from the SDK source: only
|
|
23
26
|
* `assertCanSetRequestHandler`, called by McpServer's own registration path,
|
|
24
27
|
* throws on a pre-existing handler; `setRequestHandler` itself does not).
|
|
28
|
+
* In v2 `setRequestHandler` takes a method string rather than a Zod schema.
|
|
25
29
|
*
|
|
26
|
-
* SEP-2549 note: as specified (2026-07-28
|
|
27
|
-
* ("public"|"private") are defined on
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* namespaced `_meta` key following the SDK's own
|
|
32
|
-
* `io.modelcontextprotocol/<name>` convention (see RELATED_TASK_META_KEY in
|
|
33
|
-
* @modelcontextprotocol/sdk/types.js):
|
|
30
|
+
* SEP-2549 note: as specified (2026-07-28), `ttlMs` / `cacheScope`
|
|
31
|
+
* ("public"|"private") are defined on server/discover results — there is no
|
|
32
|
+
* CacheableResult for tools/call. Since this deliverable asks for tools/call
|
|
33
|
+
* cache hints, the same two field names are carried into a namespaced `_meta`
|
|
34
|
+
* key following the SDK's own `io.modelcontextprotocol/<name>` convention:
|
|
34
35
|
* result._meta["io.modelcontextprotocol/cacheable"] = { ttlMs, cacheScope }
|
|
35
36
|
* This is a documented adaptation, not a key confirmed by the spec text for
|
|
36
37
|
* tools/call — revisit if/when a SEP defines call-result caching explicitly.
|
|
37
38
|
*/
|
|
38
39
|
|
|
39
|
-
import {
|
|
40
|
-
ListToolsRequestSchema,
|
|
41
|
-
CallToolRequestSchema,
|
|
42
|
-
ListPromptsRequestSchema
|
|
43
|
-
} from '@modelcontextprotocol/sdk/types.js';
|
|
44
|
-
|
|
45
40
|
const APPLIED = Symbol('crawlforge.specHygiene.applied');
|
|
46
41
|
|
|
47
42
|
const JSON_SCHEMA_2020_12 = 'https://json-schema.org/draft/2020-12/schema';
|
|
@@ -158,7 +153,7 @@ function wrapToolsCall(innerHandler, cacheableTools) {
|
|
|
158
153
|
* tools/prompts are registered and before transport.connect(). Idempotent:
|
|
159
154
|
* a second call on the same server is a no-op.
|
|
160
155
|
*
|
|
161
|
-
* @param {import('@modelcontextprotocol/
|
|
156
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
162
157
|
* @param {object} [overrides]
|
|
163
158
|
* @param {{src:string,mimeType?:string,sizes?:string[]}} [overrides.icon] — default icon injected into tools/prompts lacking one
|
|
164
159
|
* @param {Record<string,{ttlMs:number,cacheScope:'public'|'private'}>} [overrides.cacheableTools] — tool-name -> cache hint map for tools/call
|
|
@@ -177,16 +172,16 @@ export function applySpecHygiene(server, overrides = {}) {
|
|
|
177
172
|
|
|
178
173
|
const innerToolsList = protocol._requestHandlers.get('tools/list');
|
|
179
174
|
if (innerToolsList) {
|
|
180
|
-
protocol.setRequestHandler(
|
|
175
|
+
protocol.setRequestHandler('tools/list', wrapToolsList(innerToolsList, icon));
|
|
181
176
|
}
|
|
182
177
|
|
|
183
178
|
const innerToolsCall = protocol._requestHandlers.get('tools/call');
|
|
184
179
|
if (innerToolsCall) {
|
|
185
|
-
protocol.setRequestHandler(
|
|
180
|
+
protocol.setRequestHandler('tools/call', wrapToolsCall(innerToolsCall, cacheableTools));
|
|
186
181
|
}
|
|
187
182
|
|
|
188
183
|
const innerPromptsList = protocol._requestHandlers.get('prompts/list');
|
|
189
184
|
if (innerPromptsList) {
|
|
190
|
-
protocol.setRequestHandler(
|
|
185
|
+
protocol.setRequestHandler('prompts/list', wrapPromptsList(innerPromptsList, icon));
|
|
191
186
|
}
|
|
192
187
|
}
|
|
@@ -2,12 +2,11 @@
|
|
|
2
2
|
* stdio transport setup — extracted from server.js runServer().
|
|
3
3
|
* Used when server is launched without the --http flag.
|
|
4
4
|
*/
|
|
5
|
-
|
|
6
|
-
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
5
|
+
import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
7
6
|
|
|
8
7
|
/**
|
|
9
8
|
* Connect the MCP server to stdio transport and log startup message.
|
|
10
|
-
* @param {import('@modelcontextprotocol/
|
|
9
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
11
10
|
*/
|
|
12
11
|
export async function connectStdio(server) {
|
|
13
12
|
const transport = new StdioServerTransport();
|
|
@@ -1,42 +1,62 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Streamable HTTP transport
|
|
2
|
+
* Dual-era Streamable HTTP transport.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* - GET /mcp — SSE stream for server → client notifications
|
|
7
|
-
* - DELETE /mcp — terminate session
|
|
4
|
+
* One endpoint at /mcp serves both protocol eras, routed by the SDK's own
|
|
5
|
+
* classification (`isLegacyRequest`) so this module can never disagree with it:
|
|
8
6
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
7
|
+
* - 2026-07-28 ("modern"): stateless, one server instance per request, each
|
|
8
|
+
* request carrying its own `_meta` envelope (protocol version, clientInfo,
|
|
9
|
+
* clientCapabilities) plus the SEP-2243 `Mcp-Method` / `Mcp-Name` headers.
|
|
10
|
+
* Served by `createMcpHandler(..., { legacy: 'reject' })`, which owns the
|
|
11
|
+
* Content-Type gate (415), the header/body cross-checks (-32020) and
|
|
12
|
+
* `server/discover`.
|
|
13
|
+
* - 2025-era ("legacy"): the sessionful path below — POST /mcp initialize
|
|
14
|
+
* issues an `Mcp-Session-Id`, GET /mcp opens the notification SSE stream,
|
|
15
|
+
* DELETE /mcp terminates the session. One transport + cloned McpServer per
|
|
16
|
+
* session, kept in the `sessions` Map.
|
|
12
17
|
*
|
|
13
18
|
* Auth:
|
|
14
|
-
* - Bearer / X-API-Key required per request
|
|
19
|
+
* - Bearer / X-API-Key required per request on BOTH eras, before any era
|
|
20
|
+
* routing happens (creator mode bypasses, loopback only)
|
|
15
21
|
* - When OAuth is enabled (CRAWLFORGE_OAUTH_ENABLED=true), OAuth bearer
|
|
16
22
|
* tokens are validated by the OAuth provider and mapped server-side to
|
|
17
23
|
* a CrawlForge API key. See src/server/auth/oauth.js.
|
|
18
24
|
*
|
|
19
25
|
* Observability:
|
|
20
26
|
* - GET /metrics returns Prometheus exposition (when observability enabled)
|
|
21
|
-
* - GET /health returns liveness probe
|
|
22
|
-
*
|
|
23
|
-
* Replaces the legacy stateless http.js. Old /mcp endpoint behavior is
|
|
24
|
-
* preserved when CRAWLFORGE_LEGACY_HTTP=true (one-release deprecation window);
|
|
25
|
-
* `http.js`'s connectHttp() forwards straight into this module's legacy mode.
|
|
27
|
+
* - GET /health returns liveness probe + the protocol revisions served
|
|
26
28
|
*/
|
|
27
|
-
|
|
28
|
-
import {
|
|
29
|
-
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
29
|
+
import { McpServer, createMcpHandler, isLegacyRequest, SUPPORTED_PROTOCOL_VERSIONS } from "@modelcontextprotocol/server";
|
|
30
|
+
import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from "@modelcontextprotocol/node";
|
|
30
31
|
import { createServer } from 'node:http';
|
|
31
32
|
import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
32
33
|
import { readFileSync } from 'node:fs';
|
|
33
|
-
import { requestContext } from '../requestContext.js';
|
|
34
|
-
import {
|
|
35
|
-
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
34
|
+
import { requestContext, setServingServer } from '../requestContext.js';
|
|
35
|
+
import { applySpecHygiene } from '../specHygiene.js';
|
|
36
36
|
|
|
37
37
|
const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
|
|
38
38
|
const SERVER_VERSION = pkg.version;
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Protocol revisions this endpoint serves, newest first: the modern era's
|
|
42
|
+
* revisions followed by the 2025-era list the SDK negotiates via `initialize`.
|
|
43
|
+
*
|
|
44
|
+
* The modern list mirrors the SDK's internal SUPPORTED_MODERN_PROTOCOL_VERSIONS,
|
|
45
|
+
* which is deliberately not exported. streamableHttp.test.js pins it against a
|
|
46
|
+
* live `server/discover` result, so an SDK upgrade that adds a revision fails a
|
|
47
|
+
* test rather than drifting silently.
|
|
48
|
+
*/
|
|
49
|
+
const MODERN_PROTOCOL_VERSIONS = Object.freeze(['2026-07-28']);
|
|
50
|
+
const PROTOCOL_VERSIONS = Object.freeze([...MODERN_PROTOCOL_VERSIONS, ...SUPPORTED_PROTOCOL_VERSIONS]);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* SEP-2549 cache hint for the `server/discover` result (2026-07-28 only). The
|
|
54
|
+
* advertisement is the same for every caller and only changes when the server
|
|
55
|
+
* is redeployed, so it is `public`; the 5-minute TTL matches the tools/call
|
|
56
|
+
* hints in specHygiene.js. Without a hint the SDK emits `0` / `'private'`.
|
|
57
|
+
*/
|
|
58
|
+
const DISCOVER_CACHE_HINT = Object.freeze({ ttlMs: 300000, cacheScope: 'public' });
|
|
59
|
+
|
|
40
60
|
/**
|
|
41
61
|
* Build the `tools` array for the Smithery static server card, straight from
|
|
42
62
|
* the live tool registry.
|
|
@@ -59,10 +79,9 @@ function buildToolCards(server) {
|
|
|
59
79
|
.map(([name, tool]) => {
|
|
60
80
|
let inputSchema = { type: 'object', properties: {} };
|
|
61
81
|
try {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
const converted = zodToJsonSchema(zodObject, { $refStrategy: 'none' });
|
|
82
|
+
if (tool?.inputSchema) {
|
|
83
|
+
// The SDK's own conversion, so the card mirrors what tools/list serves.
|
|
84
|
+
const converted = { ...server.toolInputSchemaJson(name) };
|
|
66
85
|
delete converted.$schema;
|
|
67
86
|
inputSchema = converted;
|
|
68
87
|
}
|
|
@@ -89,23 +108,34 @@ function buildToolCards(server) {
|
|
|
89
108
|
* prompt tables — plain config + handler-closure references, no per-connection
|
|
90
109
|
* state — then re-runs the same internal handler-wiring methods McpServer
|
|
91
110
|
* itself calls from registerTool/registerResource/registerPrompt. This
|
|
92
|
-
* depends on
|
|
93
|
-
*
|
|
94
|
-
*
|
|
111
|
+
* depends on the SDK's internal McpServer/Server field names
|
|
112
|
+
* (`_registered*`, `set*RequestHandlers`, `_capabilities`); re-verified
|
|
113
|
+
* against @modelcontextprotocol/server 2.0.0, which still exposes all of
|
|
114
|
+
* them. `_taskStore` is gone — v2 removed experimental tasks (SEP-2663).
|
|
115
|
+
* Re-check on SDK upgrades.
|
|
116
|
+
*
|
|
117
|
+
* The same clone backs a 2025-era session and a single 2026-era request, so
|
|
118
|
+
* both eras serve exactly the same tools — the SDK's "one factory for both
|
|
119
|
+
* legs" rule.
|
|
120
|
+
*
|
|
121
|
+
* applySpecHygiene() runs on the clone because its wrappers live on the
|
|
122
|
+
* template's own Protocol instance, not in the `_registered*` tables the clone
|
|
123
|
+
* copies: without this call an HTTP client got unsorted, icon-less tools/list
|
|
124
|
+
* results and no SEP-2549 cache markers on tools/call, while a stdio client
|
|
125
|
+
* got all three.
|
|
95
126
|
*
|
|
96
|
-
* @param {import('@modelcontextprotocol/
|
|
127
|
+
* @param {import('@modelcontextprotocol/server').McpServer} templateServer
|
|
97
128
|
*/
|
|
98
129
|
function cloneServerForSession(templateServer) {
|
|
99
130
|
const low = templateServer.server;
|
|
100
|
-
// capabilities
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// 'No task store provided for task-capable tool.'
|
|
131
|
+
// capabilities must survive the clone so the session server advertises the
|
|
132
|
+
// same surface as the template. cacheHints only reaches the 2026-era encode
|
|
133
|
+
// seam (it rides a symbol-keyed property that is never serialized), so a
|
|
134
|
+
// 2025-era response is byte-identical with or without it.
|
|
105
135
|
const sessionServer = new McpServer(low._serverInfo, {
|
|
106
136
|
instructions: low._instructions,
|
|
107
137
|
capabilities: low._capabilities,
|
|
108
|
-
|
|
138
|
+
cacheHints: { 'server/discover': DISCOVER_CACHE_HINT }
|
|
109
139
|
});
|
|
110
140
|
|
|
111
141
|
sessionServer._registeredTools = templateServer._registeredTools;
|
|
@@ -118,9 +148,18 @@ function cloneServerForSession(templateServer) {
|
|
|
118
148
|
if (templateServer._promptHandlersInitialized) sessionServer.setPromptRequestHandlers();
|
|
119
149
|
if (templateServer._completionHandlerInitialized) sessionServer.setCompletionRequestHandler();
|
|
120
150
|
|
|
151
|
+
applySpecHygiene(sessionServer);
|
|
152
|
+
|
|
121
153
|
return sessionServer;
|
|
122
154
|
}
|
|
123
155
|
|
|
156
|
+
/** Reads a request body to completion as UTF-8. */
|
|
157
|
+
async function readRequestBody(req) {
|
|
158
|
+
const chunks = [];
|
|
159
|
+
for await (const chunk of req) chunks.push(chunk);
|
|
160
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
161
|
+
}
|
|
162
|
+
|
|
124
163
|
/** Best-effort close — swallows errors so cleanup never throws into a request handler. */
|
|
125
164
|
function safeClose(closable) {
|
|
126
165
|
if (closable && typeof closable.close === 'function') {
|
|
@@ -138,14 +177,14 @@ function sendRpcError(res, status, code, message) {
|
|
|
138
177
|
}
|
|
139
178
|
|
|
140
179
|
/**
|
|
141
|
-
*
|
|
180
|
+
* Dual-era Streamable HTTP transport: stateless 2026-07-28 and sessionful
|
|
181
|
+
* 2025-era traffic on the same /mcp route.
|
|
142
182
|
*
|
|
143
|
-
* @param {import('@modelcontextprotocol/
|
|
183
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
144
184
|
* @param {import('../../core/AuthManager.js').default} authManager
|
|
145
185
|
* @param {import('../../utils/Logger.js').logger} logger
|
|
146
186
|
* @param {object} [options]
|
|
147
187
|
* @param {number} [options.port=3000]
|
|
148
|
-
* @param {boolean} [options.legacy=false] — if true, run in stateless mode (3.1 behavior)
|
|
149
188
|
* @param {object} [options.oauth] — OAuth provider (see src/server/auth/oauth.js)
|
|
150
189
|
* @param {object} [options.metrics] — Prometheus registry (see src/observability/metrics.js)
|
|
151
190
|
*/
|
|
@@ -159,22 +198,44 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
159
198
|
if (authManager.isCreatorMode() && !hostIsLoopback) {
|
|
160
199
|
console.error(`WARNING: creator mode is enabled but the server is bound to ${host} (non-loopback) — per-request auth will NOT be bypassed. Bind to 127.0.0.1 to use creator mode.`);
|
|
161
200
|
}
|
|
162
|
-
const legacy = options.legacy === true;
|
|
163
201
|
const oauthProvider = options.oauth ?? null;
|
|
164
202
|
const metrics = options.metrics ?? null;
|
|
165
203
|
|
|
166
|
-
const mode =
|
|
204
|
+
const mode = 'streamable-stateful';
|
|
167
205
|
const toolCount = Object.keys(server._registeredTools ?? {}).length;
|
|
168
206
|
|
|
169
207
|
// sessionId -> { transport, server }. One StreamableHTTPServerTransport (and
|
|
170
208
|
// therefore one cloned McpServer — see cloneServerForSession) per session.
|
|
209
|
+
// 2025-era only: the modern era is stateless and holds nothing here.
|
|
171
210
|
const sessions = new Map();
|
|
172
211
|
|
|
212
|
+
// 2026-07-28 leg. `legacy: 'reject'` keeps it strict — every 2025-era request
|
|
213
|
+
// is routed to the sessions Map above by isLegacyRequest before it can reach
|
|
214
|
+
// this handler, so the modern leg never has to serve one. The SDK owns the
|
|
215
|
+
// Content-Type gate (415), the Mcp-Method/Mcp-Name cross-checks (-32020 on
|
|
216
|
+
// 400) and `server/discover`; nothing here re-implements them.
|
|
217
|
+
// The factory runs once per request, inside the requestContext.run() below,
|
|
218
|
+
// so the clone it builds can be stamped on the store: that clone — never the
|
|
219
|
+
// template — is the instance this request is actually served from.
|
|
220
|
+
const modernHandler = createMcpHandler((ctx) => {
|
|
221
|
+
const requestServer = cloneServerForSession(server);
|
|
222
|
+
setServingServer(requestServer, ctx?.era ?? 'modern');
|
|
223
|
+
return requestServer;
|
|
224
|
+
}, {
|
|
225
|
+
legacy: 'reject',
|
|
226
|
+
onerror: (err) => logger.warn('2026-era MCP request rejected', { error: err?.message })
|
|
227
|
+
});
|
|
228
|
+
const serveModern = toNodeHandler(modernHandler, {
|
|
229
|
+
onerror: (err) => logger.error('2026-era MCP request failed', { error: err?.message })
|
|
230
|
+
});
|
|
231
|
+
|
|
173
232
|
const httpServer = createServer(async (req, res) => {
|
|
174
233
|
// CORS — Smithery + browser-based MCP clients
|
|
175
234
|
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
176
235
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
|
|
177
|
-
|
|
236
|
+
// MCP-Protocol-Version / Mcp-Method / Mcp-Name are the 2026-07-28 era's
|
|
237
|
+
// request headers; the session id headers are the 2025 era's.
|
|
238
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, MCP-Protocol-Version, Mcp-Method, Mcp-Name, Authorization, X-API-Key, X-Internal-Secret');
|
|
178
239
|
res.setHeader('Access-Control-Expose-Headers', 'Mcp-Session-Id, mcp-session-id');
|
|
179
240
|
|
|
180
241
|
if (req.method === 'OPTIONS') {
|
|
@@ -186,7 +247,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
186
247
|
// Health probe
|
|
187
248
|
if (req.url === '/health') {
|
|
188
249
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
189
|
-
res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode }));
|
|
250
|
+
res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode, protocolVersions: PROTOCOL_VERSIONS }));
|
|
190
251
|
return;
|
|
191
252
|
}
|
|
192
253
|
|
|
@@ -273,38 +334,45 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
273
334
|
internal = authResult.internal === true;
|
|
274
335
|
}
|
|
275
336
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
337
|
+
// Era routing. Only a POST can carry the 2026-07-28 per-request envelope;
|
|
338
|
+
// body-less GET/DELETE are 2025 session operations by construction and
|
|
339
|
+
// isLegacyRequest classifies them as such, so they skip this entirely and
|
|
340
|
+
// keep their existing behaviour byte-for-byte.
|
|
341
|
+
//
|
|
342
|
+
// Reading the body here drains the Node stream, so the parsed value is
|
|
343
|
+
// handed to whichever leg serves the request. A body that is not valid
|
|
344
|
+
// JSON classifies legacy (the SDK's own rule), and the 2025 transport
|
|
345
|
+
// still writes its own parse error — hence the undefined pass-through
|
|
346
|
+
// rather than an answer invented here.
|
|
347
|
+
let parsedBody;
|
|
348
|
+
if (req.method === 'POST') {
|
|
282
349
|
try {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
});
|
|
350
|
+
parsedBody = JSON.parse(await readRequestBody(req));
|
|
351
|
+
} catch {
|
|
352
|
+
parsedBody = undefined;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
if (parsedBody !== undefined) {
|
|
356
|
+
const probe = await toWebRequest(req, parsedBody);
|
|
357
|
+
if (!(await isLegacyRequest(probe, parsedBody))) {
|
|
358
|
+
await requestContext.run({ internal }, () => serveModern(req, res, parsedBody));
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
295
361
|
}
|
|
296
|
-
return;
|
|
297
362
|
}
|
|
298
363
|
|
|
299
|
-
//
|
|
300
|
-
//
|
|
364
|
+
// 2025 era: route by Mcp-Session-Id. A request without the header must be
|
|
365
|
+
// a fresh initialize, which gets its own transport + server pair
|
|
301
366
|
// (independent of any prior session's lifecycle) so reconnects/re-inits
|
|
302
367
|
// never hit a stuck 'already initialized' transport.
|
|
303
368
|
const sessionIdHeader = req.headers['mcp-session-id'];
|
|
304
369
|
const existing = sessionIdHeader ? sessions.get(String(sessionIdHeader)) : undefined;
|
|
305
370
|
|
|
306
371
|
if (existing) {
|
|
307
|
-
await requestContext.run(
|
|
372
|
+
await requestContext.run(
|
|
373
|
+
{ internal, servingServer: existing.server, servingEra: 'legacy' },
|
|
374
|
+
() => existing.transport.handleRequest(req, res, parsedBody)
|
|
375
|
+
);
|
|
308
376
|
return;
|
|
309
377
|
}
|
|
310
378
|
|
|
@@ -321,7 +389,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
321
389
|
}
|
|
322
390
|
|
|
323
391
|
const sessionServer = cloneServerForSession(server);
|
|
324
|
-
const transport = new
|
|
392
|
+
const transport = new NodeStreamableHTTPServerTransport({
|
|
325
393
|
sessionIdGenerator: () => randomUUID(),
|
|
326
394
|
onsessioninitialized: (sid) => {
|
|
327
395
|
sessions.set(sid, { transport, server: sessionServer });
|
|
@@ -337,7 +405,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
337
405
|
|
|
338
406
|
try {
|
|
339
407
|
await sessionServer.connect(transport);
|
|
340
|
-
await requestContext.run(
|
|
408
|
+
await requestContext.run(
|
|
409
|
+
{ internal, servingServer: sessionServer, servingEra: 'legacy' },
|
|
410
|
+
() => transport.handleRequest(req, res, parsedBody)
|
|
411
|
+
);
|
|
341
412
|
} catch (err) {
|
|
342
413
|
logger.error('Streamable HTTP session initialization failed', { error: err?.message });
|
|
343
414
|
safeClose(transport);
|
|
@@ -355,7 +426,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
355
426
|
httpServer.listen(port, host, () => {
|
|
356
427
|
const actual = httpServer.address()?.port ?? port;
|
|
357
428
|
console.error(`CrawlForge MCP Server v${SERVER_VERSION} listening on ${host}:${actual} (Streamable HTTP, ${mode})`);
|
|
358
|
-
console.error(`MCP endpoint: http://${host}:${actual}/mcp`);
|
|
429
|
+
console.error(`MCP endpoint: http://${host}:${actual}/mcp (protocol ${PROTOCOL_VERSIONS.join(', ')})`);
|
|
359
430
|
console.error(`Health check: http://${host}:${actual}/health`);
|
|
360
431
|
if (metrics) console.error(`Metrics: http://${host}:${actual}/metrics`);
|
|
361
432
|
if (oauthProvider) console.error(`OAuth discovery: http://${host}:${actual}/.well-known/oauth-authorization-server`);
|
|
@@ -366,13 +437,17 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
366
437
|
return {
|
|
367
438
|
httpServer,
|
|
368
439
|
sessions,
|
|
369
|
-
/**
|
|
440
|
+
/**
|
|
441
|
+
* Closes every live 2025-era session's transport + server and the modern
|
|
442
|
+
* leg (aborting in-flight exchanges), then the HTTP server.
|
|
443
|
+
*/
|
|
370
444
|
async close() {
|
|
371
445
|
for (const { transport, server: sessionServer } of sessions.values()) {
|
|
372
446
|
safeClose(transport);
|
|
373
447
|
safeClose(sessionServer);
|
|
374
448
|
}
|
|
375
449
|
sessions.clear();
|
|
450
|
+
await modernHandler.close().catch(() => {});
|
|
376
451
|
await new Promise((resolve) => httpServer.close(() => resolve()));
|
|
377
452
|
}
|
|
378
453
|
};
|
package/src/server/withAuth.js
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
* so a valid API key is required for every invocation
|
|
9
9
|
* - try/finally guarantees a single `tool invocation` log line per call
|
|
10
10
|
* - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
|
|
11
|
-
* - outcome ∈ { 'success' | 'error' | 'insufficient_credits' }
|
|
11
|
+
* - outcome ∈ { 'success' | 'error' | 'insufficient_credits' | 'input_required' }
|
|
12
|
+
* - an `input_required` return (Phase 4.4) is a round trip, not an answer: the
|
|
13
|
+
* handler did no work, so it is billed NOTHING and reports no usage. The
|
|
14
|
+
* SDK re-enters the handler with the reply and the terminal entry bills
|
|
15
|
+
* once, so a confirmation costs exactly what the call always cost (G4).
|
|
12
16
|
* - error results get a "Next step:" hint naming the tool to try next
|
|
13
17
|
* (src/server/fallbackHints.js) so a failure is not followed by a blind retry
|
|
14
18
|
* - emits an OTel span via src/observability/tracing.js (no-op if disabled)
|
|
@@ -16,6 +20,7 @@
|
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
import { createHash } from 'node:crypto';
|
|
23
|
+
import { isInputRequiredResult } from '@modelcontextprotocol/server';
|
|
19
24
|
import { recordToolInvocation } from '../observability/tracing.js';
|
|
20
25
|
import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
|
|
21
26
|
import { appendFallbackHint } from './fallbackHints.js';
|
|
@@ -71,7 +76,7 @@ export function hashParams(params) {
|
|
|
71
76
|
*/
|
|
72
77
|
export function makeWithAuth({ authManager, logger, metrics = null, mcpServer = null }) {
|
|
73
78
|
return function withAuth(toolName, handler) {
|
|
74
|
-
const invoke = async (params) => {
|
|
79
|
+
const invoke = async (params, ctx) => {
|
|
75
80
|
const startTime = Date.now();
|
|
76
81
|
const paramHash = hashParams(params);
|
|
77
82
|
const creatorMode = authManager.isCreatorMode();
|
|
@@ -109,7 +114,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
109
114
|
// end user's credits before forwarding — checking the static key's
|
|
110
115
|
// balance here would gate users on an unrelated account).
|
|
111
116
|
if (!billingExempt) {
|
|
112
|
-
const hasCredits = await authManager.checkCredits(creditCost);
|
|
117
|
+
const hasCredits = await authManager.checkCredits(creditCost, ctx);
|
|
118
|
+
// The low-credit warning asks as a round trip (Phase 4.4). Nothing has
|
|
119
|
+
// run, so this costs nothing and reports no usage; the SDK re-enters
|
|
120
|
+
// with the answer.
|
|
121
|
+
if (isInputRequiredResult(hasCredits)) {
|
|
122
|
+
outcome = 'input_required';
|
|
123
|
+
return hasCredits;
|
|
124
|
+
}
|
|
113
125
|
if (!hasCredits) {
|
|
114
126
|
outcome = 'insufficient_credits';
|
|
115
127
|
return {
|
|
@@ -127,7 +139,21 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
127
139
|
}
|
|
128
140
|
|
|
129
141
|
handlerStarted = true;
|
|
130
|
-
const result = await handler(params);
|
|
142
|
+
const result = await handler(params, ctx);
|
|
143
|
+
|
|
144
|
+
// Phase 4.4: a multi-round-trip handler answers `input_required` when it
|
|
145
|
+
// needs the user before it can start. Nothing was fetched, so nothing is
|
|
146
|
+
// owed: no charge, no usage report, and none of the result stages below
|
|
147
|
+
// (there is no result yet to redact, shape or price). The SDK gathers the
|
|
148
|
+
// answer and re-enters this same wrapper; whichever entry finally returns
|
|
149
|
+
// a real result is the one that bills, exactly once. Without this branch
|
|
150
|
+
// an `input_required` is not `isError`, so it books as a success and
|
|
151
|
+
// bills in full on every round — up to eight — for a call that did no
|
|
152
|
+
// work, and a declined confirmation bills too (G4).
|
|
153
|
+
if (isInputRequiredResult(result)) {
|
|
154
|
+
outcome = 'input_required';
|
|
155
|
+
return result;
|
|
156
|
+
}
|
|
131
157
|
|
|
132
158
|
// Tools catch their own failures and return { isError:true } rather than
|
|
133
159
|
// throwing (the shared pattern in server.js). That is still an ERROR
|
|
@@ -275,11 +301,23 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
275
301
|
|
|
276
302
|
// Every invocation runs in its own context so the compliance gate can stamp
|
|
277
303
|
// a refusal where the billing decision can see it. Any outer store (the
|
|
278
|
-
// HTTP transport's `internal` flag) is spread in,
|
|
279
|
-
// callers, who have no transport-provided store,
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
304
|
+
// HTTP transport's `internal` flag, the serving McpServer) is spread in,
|
|
305
|
+
// not replaced — and stdio callers, who have no transport-provided store,
|
|
306
|
+
// get one here.
|
|
307
|
+
//
|
|
308
|
+
// `ctx` is the SDK's per-request context (v2 calls a tool callback with
|
|
309
|
+
// `(args, ctx)`). It is passed straight through to the handler; existing
|
|
310
|
+
// 1-arity handlers ignore it. Its request id is stamped on the context so a
|
|
311
|
+
// server-to-client request sent from inside the tool can ride the same
|
|
312
|
+
// stream as this call — see servingRequestId() in requestContext.js.
|
|
313
|
+
return async (params, ctx) => requestContext.run(
|
|
314
|
+
{
|
|
315
|
+
...(requestContext.getStore() ?? {}),
|
|
316
|
+
preflightRefusal: null,
|
|
317
|
+
actualCost: null,
|
|
318
|
+
servingRequestId: ctx?.mcpReq?.id
|
|
319
|
+
},
|
|
320
|
+
() => invoke(params, ctx)
|
|
283
321
|
);
|
|
284
322
|
};
|
|
285
323
|
}
|