crawlforge-mcp-server 5.10.0 → 6.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/README.md +7 -1
  2. package/package.json +7 -5
  3. package/server.js +210 -251
  4. package/src/cli/commands/login.js +176 -0
  5. package/src/cli/index.js +2 -0
  6. package/src/core/ActionExecutor.js +1 -1
  7. package/src/core/AuthManager.js +19 -6
  8. package/src/core/ChangeTracker.js +1 -1
  9. package/src/core/ElicitationHelper.js +192 -62
  10. package/src/core/SamplingClient.js +8 -2
  11. package/src/core/analysis/ContentAnalyzer.js +1 -1
  12. package/src/core/processing/BrowserProcessor.js +1 -1
  13. package/src/core/processing/ContentProcessor.js +1 -1
  14. package/src/core/processing/PDFProcessor.js +2 -2
  15. package/src/server/registerTool.js +1 -1
  16. package/src/server/requestContext.js +50 -0
  17. package/src/server/specHygiene.js +17 -22
  18. package/src/server/transports/stdio.js +2 -3
  19. package/src/server/transports/streamableHttp.js +142 -67
  20. package/src/server/withAuth.js +47 -9
  21. package/src/tools/advanced/batchScrape/index.js +29 -19
  22. package/src/tools/agent/agent.js +9 -4
  23. package/src/tools/crawl/crawlDeep.js +14 -8
  24. package/src/tools/extract/analyzeContent.js +1 -1
  25. package/src/tools/extract/extractContent.js +1 -1
  26. package/src/tools/extract/extractStructured.js +62 -43
  27. package/src/tools/extract/processDocument.js +1 -1
  28. package/src/tools/extract/summarizeContent.js +1 -1
  29. package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
  30. package/src/tools/research/deepResearch.js +11 -5
  31. package/src/tools/tracking/trackChanges/schema.js +4 -4
  32. package/src/utils/HumanBehaviorSimulator.js +7 -7
  33. package/src/server/taskSupport.js +0 -233
  34. package/src/server/transports/http.js +0 -22
@@ -69,3 +69,53 @@ export function setActualCost(n) {
69
69
  export function reportedActualCost() {
70
70
  return requestContext.getStore()?.actualCost ?? null;
71
71
  }
72
+
73
+ /**
74
+ * Record the McpServer instance that is actually serving this request, and the
75
+ * wire era it speaks.
76
+ *
77
+ * Neither HTTP leg serves from the top-level McpServer that server.js
78
+ * registers everything on: the 2025-era path connects one clone per session,
79
+ * and the modern leg builds a fresh clone per request (see
80
+ * transports/streamableHttp.js). Only a clone is ever `.connect()`ed, so only a
81
+ * clone has a negotiated protocol version, the client's declared capabilities,
82
+ * and a channel to send a server-to-client request on. The template has none of
83
+ * those, which is why anything reading them off it (ElicitationHelper) got
84
+ * `undefined` on every HTTP request.
85
+ *
86
+ * Stdio stamps nothing: there the top-level instance IS the connected one, and
87
+ * the accessors below return null so callers fall back to it.
88
+ *
89
+ * @param {object|null} server the serving McpServer
90
+ * @param {'legacy'|'modern'|null} [era] the wire era it serves
91
+ */
92
+ export function setServingServer(server, era = null) {
93
+ const store = requestContext.getStore();
94
+ if (!store) return;
95
+ store.servingServer = server ?? null;
96
+ store.servingEra = era;
97
+ }
98
+
99
+ /** The McpServer serving this request, or null on stdio / outside a context. */
100
+ export function servingServer() {
101
+ return requestContext.getStore()?.servingServer ?? null;
102
+ }
103
+
104
+ /** The wire era serving this request: 'legacy' | 'modern' | null (stdio). */
105
+ export function servingEra() {
106
+ return requestContext.getStore()?.servingEra ?? null;
107
+ }
108
+
109
+ /**
110
+ * The JSON-RPC id of the request being served, or null when unknown.
111
+ *
112
+ * A server-to-client request sent from inside a tool has to say which inbound
113
+ * request it belongs to: the 2025-era streamable HTTP transport routes an
114
+ * unrelated request to the standalone GET SSE stream and silently DROPS it when
115
+ * the client never opened one, which turns an elicitation prompt into a
116
+ * 60-second stall before it fails open. withAuth stamps this from the SDK's
117
+ * per-request `ctx`; stdio has no streams to pick between and ignores it.
118
+ */
119
+ export function servingRequestId() {
120
+ return requestContext.getStore()?.servingRequestId ?? null;
121
+ }
@@ -1,19 +1,22 @@
1
1
  /**
2
2
  * specHygiene — Phase 6 protocol-hygiene wrapper.
3
3
  *
4
- * Applies three MCP wire-level upgrades to an already-registered McpServer
4
+ * Applies four MCP wire-level upgrades to an already-registered McpServer
5
5
  * without changing how tools/prompts are declared anywhere else:
6
6
  *
7
7
  * 1. JSON Schema 2020-12 dialect stamping on every tool's inputSchema /
8
- * outputSchema (the SDK's zod-to-json-schema conversion still emits
9
- * draft-07-style `definitions` / `#/definitions/...` refs).
8
+ * outputSchema. Under the v2 SDK + zod 4 the conversion already emits
9
+ * 2020-12 with `$defs`, so the `definitions` -> `$defs` rewrite below is
10
+ * now defensive rather than load-bearing; it is kept because the stamping
11
+ * and ordering passes still walk the same tree (Phase 4.1 kept behaviour
12
+ * identical — removing the rewrite is a separate, verifiable change).
10
13
  * 2. Deterministic (alphabetical) tool ordering in tools/list, so clients
11
14
  * that prompt-cache tools/list get stable hits across restarts.
12
15
  * 3. SEP-973 icons metadata on every tool/prompt lacking one.
13
16
  * 4. SEP-2549-style cache hints on tools/call results for a fixed
14
17
  * allowlist of read-only, idempotent tools (see note below).
15
18
  *
16
- * Wiring: SDK 1.30 has no plugin hook for this, so applySpecHygiene() must be
19
+ * Wiring: the SDK has no plugin hook for this, so applySpecHygiene() must be
17
20
  * called once, after all server.registerTool()/registerPrompt() calls and
18
21
  * before transport.connect(). It reaches into `server.server` (the
19
22
  * underlying Protocol), captures the ListTools/CallTool/ListPrompts handlers
@@ -22,26 +25,18 @@
22
25
  * handler for the same method (confirmed from the SDK source: only
23
26
  * `assertCanSetRequestHandler`, called by McpServer's own registration path,
24
27
  * throws on a pre-existing handler; `setRequestHandler` itself does not).
28
+ * In v2 `setRequestHandler` takes a method string rather than a Zod schema.
25
29
  *
26
- * SEP-2549 note: as specified (2026-07-28 RC), `ttlMs` / `cacheScope`
27
- * ("public"|"private") are defined on tools/list, prompts/list,
28
- * resources/list, resources/read and resources/templates/list results —
29
- * there is no CacheableResult for tools/call. Since this deliverable asks
30
- * for tools/call cache hints, the same two field names are carried into a
31
- * namespaced `_meta` key following the SDK's own
32
- * `io.modelcontextprotocol/<name>` convention (see RELATED_TASK_META_KEY in
33
- * @modelcontextprotocol/sdk/types.js):
30
+ * SEP-2549 note: as specified (2026-07-28), `ttlMs` / `cacheScope`
31
+ * ("public"|"private") are defined on server/discover results — there is no
32
+ * CacheableResult for tools/call. Since this deliverable asks for tools/call
33
+ * cache hints, the same two field names are carried into a namespaced `_meta`
34
+ * key following the SDK's own `io.modelcontextprotocol/<name>` convention:
34
35
  * result._meta["io.modelcontextprotocol/cacheable"] = { ttlMs, cacheScope }
35
36
  * This is a documented adaptation, not a key confirmed by the spec text for
36
37
  * tools/call — revisit if/when a SEP defines call-result caching explicitly.
37
38
  */
38
39
 
39
- import {
40
- ListToolsRequestSchema,
41
- CallToolRequestSchema,
42
- ListPromptsRequestSchema
43
- } from '@modelcontextprotocol/sdk/types.js';
44
-
45
40
  const APPLIED = Symbol('crawlforge.specHygiene.applied');
46
41
 
47
42
  const JSON_SCHEMA_2020_12 = 'https://json-schema.org/draft/2020-12/schema';
@@ -158,7 +153,7 @@ function wrapToolsCall(innerHandler, cacheableTools) {
158
153
  * tools/prompts are registered and before transport.connect(). Idempotent:
159
154
  * a second call on the same server is a no-op.
160
155
  *
161
- * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
156
+ * @param {import('@modelcontextprotocol/server').McpServer} server
162
157
  * @param {object} [overrides]
163
158
  * @param {{src:string,mimeType?:string,sizes?:string[]}} [overrides.icon] — default icon injected into tools/prompts lacking one
164
159
  * @param {Record<string,{ttlMs:number,cacheScope:'public'|'private'}>} [overrides.cacheableTools] — tool-name -> cache hint map for tools/call
@@ -177,16 +172,16 @@ export function applySpecHygiene(server, overrides = {}) {
177
172
 
178
173
  const innerToolsList = protocol._requestHandlers.get('tools/list');
179
174
  if (innerToolsList) {
180
- protocol.setRequestHandler(ListToolsRequestSchema, wrapToolsList(innerToolsList, icon));
175
+ protocol.setRequestHandler('tools/list', wrapToolsList(innerToolsList, icon));
181
176
  }
182
177
 
183
178
  const innerToolsCall = protocol._requestHandlers.get('tools/call');
184
179
  if (innerToolsCall) {
185
- protocol.setRequestHandler(CallToolRequestSchema, wrapToolsCall(innerToolsCall, cacheableTools));
180
+ protocol.setRequestHandler('tools/call', wrapToolsCall(innerToolsCall, cacheableTools));
186
181
  }
187
182
 
188
183
  const innerPromptsList = protocol._requestHandlers.get('prompts/list');
189
184
  if (innerPromptsList) {
190
- protocol.setRequestHandler(ListPromptsRequestSchema, wrapPromptsList(innerPromptsList, icon));
185
+ protocol.setRequestHandler('prompts/list', wrapPromptsList(innerPromptsList, icon));
191
186
  }
192
187
  }
@@ -2,12 +2,11 @@
2
2
  * stdio transport setup — extracted from server.js runServer().
3
3
  * Used when server is launched without the --http flag.
4
4
  */
5
-
6
- import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
5
+ import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
7
6
 
8
7
  /**
9
8
  * Connect the MCP server to stdio transport and log startup message.
10
- * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
9
+ * @param {import('@modelcontextprotocol/server').McpServer} server
11
10
  */
12
11
  export async function connectStdio(server) {
13
12
  const transport = new StdioServerTransport();
@@ -1,42 +1,62 @@
1
1
  /**
2
- * Streamable HTTP transport (MCP spec 2025-06-18).
2
+ * Dual-era Streamable HTTP transport.
3
3
  *
4
- * Single endpoint at /mcp:
5
- * - POST /mcp — JSON-RPC request, response as JSON or SSE stream
6
- * - GET /mcp — SSE stream for server → client notifications
7
- * - DELETE /mcp — terminate session
4
+ * One endpoint at /mcp serves both protocol eras, routed by the SDK's own
5
+ * classification (`isLegacyRequest`) so this module can never disagree with it:
8
6
  *
9
- * Session resumption:
10
- * - Server generates a session id and returns it as `Mcp-Session-Id` on init
11
- * - Clients re-send `Mcp-Session-Id` on subsequent requests to resume state
7
+ * - 2026-07-28 ("modern"): stateless, one server instance per request, each
8
+ * request carrying its own `_meta` envelope (protocol version, clientInfo,
9
+ * clientCapabilities) plus the SEP-2243 `Mcp-Method` / `Mcp-Name` headers.
10
+ * Served by `createMcpHandler(..., { legacy: 'reject' })`, which owns the
11
+ * Content-Type gate (415), the header/body cross-checks (-32020) and
12
+ * `server/discover`.
13
+ * - 2025-era ("legacy"): the sessionful path below — POST /mcp initialize
14
+ * issues an `Mcp-Session-Id`, GET /mcp opens the notification SSE stream,
15
+ * DELETE /mcp terminates the session. One transport + cloned McpServer per
16
+ * session, kept in the `sessions` Map.
12
17
  *
13
18
  * Auth:
14
- * - Bearer / X-API-Key required per request (creator mode bypasses)
19
+ * - Bearer / X-API-Key required per request on BOTH eras, before any era
20
+ * routing happens (creator mode bypasses, loopback only)
15
21
  * - When OAuth is enabled (CRAWLFORGE_OAUTH_ENABLED=true), OAuth bearer
16
22
  * tokens are validated by the OAuth provider and mapped server-side to
17
23
  * a CrawlForge API key. See src/server/auth/oauth.js.
18
24
  *
19
25
  * Observability:
20
26
  * - GET /metrics returns Prometheus exposition (when observability enabled)
21
- * - GET /health returns liveness probe
22
- *
23
- * Replaces the legacy stateless http.js. Old /mcp endpoint behavior is
24
- * preserved when CRAWLFORGE_LEGACY_HTTP=true (one-release deprecation window);
25
- * `http.js`'s connectHttp() forwards straight into this module's legacy mode.
27
+ * - GET /health returns liveness probe + the protocol revisions served
26
28
  */
27
-
28
- import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
29
- import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
29
+ import { McpServer, createMcpHandler, isLegacyRequest, SUPPORTED_PROTOCOL_VERSIONS } from "@modelcontextprotocol/server";
30
+ import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from "@modelcontextprotocol/node";
30
31
  import { createServer } from 'node:http';
31
32
  import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
32
33
  import { readFileSync } from 'node:fs';
33
- import { requestContext } from '../requestContext.js';
34
- import { z } from 'zod';
35
- import { zodToJsonSchema } from 'zod-to-json-schema';
34
+ import { requestContext, setServingServer } from '../requestContext.js';
35
+ import { applySpecHygiene } from '../specHygiene.js';
36
36
 
37
37
  const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
38
38
  const SERVER_VERSION = pkg.version;
39
39
 
40
+ /**
41
+ * Protocol revisions this endpoint serves, newest first: the modern era's
42
+ * revisions followed by the 2025-era list the SDK negotiates via `initialize`.
43
+ *
44
+ * The modern list mirrors the SDK's internal SUPPORTED_MODERN_PROTOCOL_VERSIONS,
45
+ * which is deliberately not exported. streamableHttp.test.js pins it against a
46
+ * live `server/discover` result, so an SDK upgrade that adds a revision fails a
47
+ * test rather than drifting silently.
48
+ */
49
+ const MODERN_PROTOCOL_VERSIONS = Object.freeze(['2026-07-28']);
50
+ const PROTOCOL_VERSIONS = Object.freeze([...MODERN_PROTOCOL_VERSIONS, ...SUPPORTED_PROTOCOL_VERSIONS]);
51
+
52
+ /**
53
+ * SEP-2549 cache hint for the `server/discover` result (2026-07-28 only). The
54
+ * advertisement is the same for every caller and only changes when the server
55
+ * is redeployed, so it is `public`; the 5-minute TTL matches the tools/call
56
+ * hints in specHygiene.js. Without a hint the SDK emits `0` / `'private'`.
57
+ */
58
+ const DISCOVER_CACHE_HINT = Object.freeze({ ttlMs: 300000, cacheScope: 'public' });
59
+
40
60
  /**
41
61
  * Build the `tools` array for the Smithery static server card, straight from
42
62
  * the live tool registry.
@@ -59,10 +79,9 @@ function buildToolCards(server) {
59
79
  .map(([name, tool]) => {
60
80
  let inputSchema = { type: 'object', properties: {} };
61
81
  try {
62
- const shape = tool?.inputSchema;
63
- if (shape) {
64
- const zodObject = typeof shape?.safeParse === 'function' ? shape : z.object(shape);
65
- const converted = zodToJsonSchema(zodObject, { $refStrategy: 'none' });
82
+ if (tool?.inputSchema) {
83
+ // The SDK's own conversion, so the card mirrors what tools/list serves.
84
+ const converted = { ...server.toolInputSchemaJson(name) };
66
85
  delete converted.$schema;
67
86
  inputSchema = converted;
68
87
  }
@@ -89,23 +108,34 @@ function buildToolCards(server) {
89
108
  * prompt tables — plain config + handler-closure references, no per-connection
90
109
  * state — then re-runs the same internal handler-wiring methods McpServer
91
110
  * itself calls from registerTool/registerResource/registerPrompt. This
92
- * depends on @modelcontextprotocol/sdk 1.30.0's internal McpServer/Server
93
- * field names (`_registered*`, `set*RequestHandlers`, `_capabilities`,
94
- * `_taskStore`); re-check on SDK upgrades.
111
+ * depends on the SDK's internal McpServer/Server field names
112
+ * (`_registered*`, `set*RequestHandlers`, `_capabilities`); re-verified
113
+ * against @modelcontextprotocol/server 2.0.0, which still exposes all of
114
+ * them. `_taskStore` is gone — v2 removed experimental tasks (SEP-2663).
115
+ * Re-check on SDK upgrades.
116
+ *
117
+ * The same clone backs a 2025-era session and a single 2026-era request, so
118
+ * both eras serve exactly the same tools — the SDK's "one factory for both
119
+ * legs" rule.
120
+ *
121
+ * applySpecHygiene() runs on the clone because its wrappers live on the
122
+ * template's own Protocol instance, not in the `_registered*` tables the clone
123
+ * copies: without this call an HTTP client got unsorted, icon-less tools/list
124
+ * results and no SEP-2549 cache markers on tools/call, while a stdio client
125
+ * got all three.
95
126
  *
96
- * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} templateServer
127
+ * @param {import('@modelcontextprotocol/server').McpServer} templateServer
97
128
  */
98
129
  function cloneServerForSession(templateServer) {
99
130
  const low = templateServer.server;
100
- // capabilities + taskStore must survive the clone: the SDK's Protocol
101
- // constructor wires the tasks/* request handlers only when options.taskStore
102
- // is present, and without it a tools/call on any task-capable tool
103
- // (crawl_deep, batch_scrape, deep_research, agent) throws
104
- // 'No task store provided for task-capable tool.'
131
+ // capabilities must survive the clone so the session server advertises the
132
+ // same surface as the template. cacheHints only reaches the 2026-era encode
133
+ // seam (it rides a symbol-keyed property that is never serialized), so a
134
+ // 2025-era response is byte-identical with or without it.
105
135
  const sessionServer = new McpServer(low._serverInfo, {
106
136
  instructions: low._instructions,
107
137
  capabilities: low._capabilities,
108
- taskStore: low._taskStore
138
+ cacheHints: { 'server/discover': DISCOVER_CACHE_HINT }
109
139
  });
110
140
 
111
141
  sessionServer._registeredTools = templateServer._registeredTools;
@@ -118,9 +148,18 @@ function cloneServerForSession(templateServer) {
118
148
  if (templateServer._promptHandlersInitialized) sessionServer.setPromptRequestHandlers();
119
149
  if (templateServer._completionHandlerInitialized) sessionServer.setCompletionRequestHandler();
120
150
 
151
+ applySpecHygiene(sessionServer);
152
+
121
153
  return sessionServer;
122
154
  }
123
155
 
156
+ /** Reads a request body to completion as UTF-8. */
157
+ async function readRequestBody(req) {
158
+ const chunks = [];
159
+ for await (const chunk of req) chunks.push(chunk);
160
+ return Buffer.concat(chunks).toString('utf8');
161
+ }
162
+
124
163
  /** Best-effort close — swallows errors so cleanup never throws into a request handler. */
125
164
  function safeClose(closable) {
126
165
  if (closable && typeof closable.close === 'function') {
@@ -138,14 +177,14 @@ function sendRpcError(res, status, code, message) {
138
177
  }
139
178
 
140
179
  /**
141
- * Stateful, session-aware Streamable HTTP transport.
180
+ * Dual-era Streamable HTTP transport: stateless 2026-07-28 and sessionful
181
+ * 2025-era traffic on the same /mcp route.
142
182
  *
143
- * @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
183
+ * @param {import('@modelcontextprotocol/server').McpServer} server
144
184
  * @param {import('../../core/AuthManager.js').default} authManager
145
185
  * @param {import('../../utils/Logger.js').logger} logger
146
186
  * @param {object} [options]
147
187
  * @param {number} [options.port=3000]
148
- * @param {boolean} [options.legacy=false] — if true, run in stateless mode (3.1 behavior)
149
188
  * @param {object} [options.oauth] — OAuth provider (see src/server/auth/oauth.js)
150
189
  * @param {object} [options.metrics] — Prometheus registry (see src/observability/metrics.js)
151
190
  */
@@ -159,22 +198,44 @@ export async function connectStreamableHttp(server, authManager, logger, options
159
198
  if (authManager.isCreatorMode() && !hostIsLoopback) {
160
199
  console.error(`WARNING: creator mode is enabled but the server is bound to ${host} (non-loopback) — per-request auth will NOT be bypassed. Bind to 127.0.0.1 to use creator mode.`);
161
200
  }
162
- const legacy = options.legacy === true;
163
201
  const oauthProvider = options.oauth ?? null;
164
202
  const metrics = options.metrics ?? null;
165
203
 
166
- const mode = legacy ? 'legacy-stateless' : 'streamable-stateful';
204
+ const mode = 'streamable-stateful';
167
205
  const toolCount = Object.keys(server._registeredTools ?? {}).length;
168
206
 
169
207
  // sessionId -> { transport, server }. One StreamableHTTPServerTransport (and
170
208
  // therefore one cloned McpServer — see cloneServerForSession) per session.
209
+ // 2025-era only: the modern era is stateless and holds nothing here.
171
210
  const sessions = new Map();
172
211
 
212
+ // 2026-07-28 leg. `legacy: 'reject'` keeps it strict — every 2025-era request
213
+ // is routed to the sessions Map above by isLegacyRequest before it can reach
214
+ // this handler, so the modern leg never has to serve one. The SDK owns the
215
+ // Content-Type gate (415), the Mcp-Method/Mcp-Name cross-checks (-32020 on
216
+ // 400) and `server/discover`; nothing here re-implements them.
217
+ // The factory runs once per request, inside the requestContext.run() below,
218
+ // so the clone it builds can be stamped on the store: that clone — never the
219
+ // template — is the instance this request is actually served from.
220
+ const modernHandler = createMcpHandler((ctx) => {
221
+ const requestServer = cloneServerForSession(server);
222
+ setServingServer(requestServer, ctx?.era ?? 'modern');
223
+ return requestServer;
224
+ }, {
225
+ legacy: 'reject',
226
+ onerror: (err) => logger.warn('2026-era MCP request rejected', { error: err?.message })
227
+ });
228
+ const serveModern = toNodeHandler(modernHandler, {
229
+ onerror: (err) => logger.error('2026-era MCP request failed', { error: err?.message })
230
+ });
231
+
173
232
  const httpServer = createServer(async (req, res) => {
174
233
  // CORS — Smithery + browser-based MCP clients
175
234
  res.setHeader('Access-Control-Allow-Origin', '*');
176
235
  res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
177
- res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, Authorization, X-API-Key, X-Internal-Secret');
236
+ // MCP-Protocol-Version / Mcp-Method / Mcp-Name are the 2026-07-28 era's
237
+ // request headers; the session id headers are the 2025 era's.
238
+ res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, MCP-Protocol-Version, Mcp-Method, Mcp-Name, Authorization, X-API-Key, X-Internal-Secret');
178
239
  res.setHeader('Access-Control-Expose-Headers', 'Mcp-Session-Id, mcp-session-id');
179
240
 
180
241
  if (req.method === 'OPTIONS') {
@@ -186,7 +247,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
186
247
  // Health probe
187
248
  if (req.url === '/health') {
188
249
  res.writeHead(200, { 'Content-Type': 'application/json' });
189
- res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode }));
250
+ res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode, protocolVersions: PROTOCOL_VERSIONS }));
190
251
  return;
191
252
  }
192
253
 
@@ -273,38 +334,45 @@ export async function connectStreamableHttp(server, authManager, logger, options
273
334
  internal = authResult.internal === true;
274
335
  }
275
336
 
276
- if (legacy) {
277
- // Stateless mode: the SDK forbids reusing a transport (or its connected
278
- // Server) across requests, so build a fresh pair per request and always
279
- // end the response, even on failure.
280
- let sessionServer;
281
- let reqTransport;
337
+ // Era routing. Only a POST can carry the 2026-07-28 per-request envelope;
338
+ // body-less GET/DELETE are 2025 session operations by construction and
339
+ // isLegacyRequest classifies them as such, so they skip this entirely and
340
+ // keep their existing behaviour byte-for-byte.
341
+ //
342
+ // Reading the body here drains the Node stream, so the parsed value is
343
+ // handed to whichever leg serves the request. A body that is not valid
344
+ // JSON classifies legacy (the SDK's own rule), and the 2025 transport
345
+ // still writes its own parse error — hence the undefined pass-through
346
+ // rather than an answer invented here.
347
+ let parsedBody;
348
+ if (req.method === 'POST') {
282
349
  try {
283
- sessionServer = cloneServerForSession(server);
284
- reqTransport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
285
- await sessionServer.connect(reqTransport);
286
- await requestContext.run({ internal }, () => reqTransport.handleRequest(req, res));
287
- } catch (err) {
288
- logger.error('Legacy Streamable HTTP request failed', { error: err?.message });
289
- sendRpcError(res, 500, -32603, 'Internal server error');
290
- } finally {
291
- res.on('close', () => {
292
- safeClose(reqTransport);
293
- safeClose(sessionServer);
294
- });
350
+ parsedBody = JSON.parse(await readRequestBody(req));
351
+ } catch {
352
+ parsedBody = undefined;
353
+ }
354
+
355
+ if (parsedBody !== undefined) {
356
+ const probe = await toWebRequest(req, parsedBody);
357
+ if (!(await isLegacyRequest(probe, parsedBody))) {
358
+ await requestContext.run({ internal }, () => serveModern(req, res, parsedBody));
359
+ return;
360
+ }
295
361
  }
296
- return;
297
362
  }
298
363
 
299
- // Stateful mode: route by Mcp-Session-Id. A request without the header
300
- // must be a fresh initialize, which gets its own transport + server pair
364
+ // 2025 era: route by Mcp-Session-Id. A request without the header must be
365
+ // a fresh initialize, which gets its own transport + server pair
301
366
  // (independent of any prior session's lifecycle) so reconnects/re-inits
302
367
  // never hit a stuck 'already initialized' transport.
303
368
  const sessionIdHeader = req.headers['mcp-session-id'];
304
369
  const existing = sessionIdHeader ? sessions.get(String(sessionIdHeader)) : undefined;
305
370
 
306
371
  if (existing) {
307
- await requestContext.run({ internal }, () => existing.transport.handleRequest(req, res));
372
+ await requestContext.run(
373
+ { internal, servingServer: existing.server, servingEra: 'legacy' },
374
+ () => existing.transport.handleRequest(req, res, parsedBody)
375
+ );
308
376
  return;
309
377
  }
310
378
 
@@ -321,7 +389,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
321
389
  }
322
390
 
323
391
  const sessionServer = cloneServerForSession(server);
324
- const transport = new StreamableHTTPServerTransport({
392
+ const transport = new NodeStreamableHTTPServerTransport({
325
393
  sessionIdGenerator: () => randomUUID(),
326
394
  onsessioninitialized: (sid) => {
327
395
  sessions.set(sid, { transport, server: sessionServer });
@@ -337,7 +405,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
337
405
 
338
406
  try {
339
407
  await sessionServer.connect(transport);
340
- await requestContext.run({ internal }, () => transport.handleRequest(req, res));
408
+ await requestContext.run(
409
+ { internal, servingServer: sessionServer, servingEra: 'legacy' },
410
+ () => transport.handleRequest(req, res, parsedBody)
411
+ );
341
412
  } catch (err) {
342
413
  logger.error('Streamable HTTP session initialization failed', { error: err?.message });
343
414
  safeClose(transport);
@@ -355,7 +426,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
355
426
  httpServer.listen(port, host, () => {
356
427
  const actual = httpServer.address()?.port ?? port;
357
428
  console.error(`CrawlForge MCP Server v${SERVER_VERSION} listening on ${host}:${actual} (Streamable HTTP, ${mode})`);
358
- console.error(`MCP endpoint: http://${host}:${actual}/mcp`);
429
+ console.error(`MCP endpoint: http://${host}:${actual}/mcp (protocol ${PROTOCOL_VERSIONS.join(', ')})`);
359
430
  console.error(`Health check: http://${host}:${actual}/health`);
360
431
  if (metrics) console.error(`Metrics: http://${host}:${actual}/metrics`);
361
432
  if (oauthProvider) console.error(`OAuth discovery: http://${host}:${actual}/.well-known/oauth-authorization-server`);
@@ -366,13 +437,17 @@ export async function connectStreamableHttp(server, authManager, logger, options
366
437
  return {
367
438
  httpServer,
368
439
  sessions,
369
- /** Closes every live session's transport + server, then the HTTP server. */
440
+ /**
441
+ * Closes every live 2025-era session's transport + server and the modern
442
+ * leg (aborting in-flight exchanges), then the HTTP server.
443
+ */
370
444
  async close() {
371
445
  for (const { transport, server: sessionServer } of sessions.values()) {
372
446
  safeClose(transport);
373
447
  safeClose(sessionServer);
374
448
  }
375
449
  sessions.clear();
450
+ await modernHandler.close().catch(() => {});
376
451
  await new Promise((resolve) => httpServer.close(() => resolve()));
377
452
  }
378
453
  };
@@ -8,7 +8,11 @@
8
8
  * so a valid API key is required for every invocation
9
9
  * - try/finally guarantees a single `tool invocation` log line per call
10
10
  * - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
11
- * - outcome ∈ { 'success' | 'error' | 'insufficient_credits' }
11
+ * - outcome ∈ { 'success' | 'error' | 'insufficient_credits' | 'input_required' }
12
+ * - an `input_required` return (Phase 4.4) is a round trip, not an answer: the
13
+ * handler did no work, so it is billed NOTHING and reports no usage. The
14
+ * SDK re-enters the handler with the reply and the terminal entry bills
15
+ * once, so a confirmation costs exactly what the call always cost (G4).
12
16
  * - error results get a "Next step:" hint naming the tool to try next
13
17
  * (src/server/fallbackHints.js) so a failure is not followed by a blind retry
14
18
  * - emits an OTel span via src/observability/tracing.js (no-op if disabled)
@@ -16,6 +20,7 @@
16
20
  */
17
21
 
18
22
  import { createHash } from 'node:crypto';
23
+ import { isInputRequiredResult } from '@modelcontextprotocol/server';
19
24
  import { recordToolInvocation } from '../observability/tracing.js';
20
25
  import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
21
26
  import { appendFallbackHint } from './fallbackHints.js';
@@ -71,7 +76,7 @@ export function hashParams(params) {
71
76
  */
72
77
  export function makeWithAuth({ authManager, logger, metrics = null, mcpServer = null }) {
73
78
  return function withAuth(toolName, handler) {
74
- const invoke = async (params) => {
79
+ const invoke = async (params, ctx) => {
75
80
  const startTime = Date.now();
76
81
  const paramHash = hashParams(params);
77
82
  const creatorMode = authManager.isCreatorMode();
@@ -109,7 +114,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
109
114
  // end user's credits before forwarding — checking the static key's
110
115
  // balance here would gate users on an unrelated account).
111
116
  if (!billingExempt) {
112
- const hasCredits = await authManager.checkCredits(creditCost);
117
+ const hasCredits = await authManager.checkCredits(creditCost, ctx);
118
+ // The low-credit warning asks as a round trip (Phase 4.4). Nothing has
119
+ // run, so this costs nothing and reports no usage; the SDK re-enters
120
+ // with the answer.
121
+ if (isInputRequiredResult(hasCredits)) {
122
+ outcome = 'input_required';
123
+ return hasCredits;
124
+ }
113
125
  if (!hasCredits) {
114
126
  outcome = 'insufficient_credits';
115
127
  return {
@@ -127,7 +139,21 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
127
139
  }
128
140
 
129
141
  handlerStarted = true;
130
- const result = await handler(params);
142
+ const result = await handler(params, ctx);
143
+
144
+ // Phase 4.4: a multi-round-trip handler answers `input_required` when it
145
+ // needs the user before it can start. Nothing was fetched, so nothing is
146
+ // owed: no charge, no usage report, and none of the result stages below
147
+ // (there is no result yet to redact, shape or price). The SDK gathers the
148
+ // answer and re-enters this same wrapper; whichever entry finally returns
149
+ // a real result is the one that bills, exactly once. Without this branch
150
+ // an `input_required` is not `isError`, so it books as a success and
151
+ // bills in full on every round — up to eight — for a call that did no
152
+ // work, and a declined confirmation bills too (G4).
153
+ if (isInputRequiredResult(result)) {
154
+ outcome = 'input_required';
155
+ return result;
156
+ }
131
157
 
132
158
  // Tools catch their own failures and return { isError:true } rather than
133
159
  // throwing (the shared pattern in server.js). That is still an ERROR
@@ -275,11 +301,23 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
275
301
 
276
302
  // Every invocation runs in its own context so the compliance gate can stamp
277
303
  // a refusal where the billing decision can see it. Any outer store (the
278
- // HTTP transport's `internal` flag) is spread in, not replaced — and stdio
279
- // callers, who have no transport-provided store, get one here.
280
- return async (params) => requestContext.run(
281
- { ...(requestContext.getStore() ?? {}), preflightRefusal: null, actualCost: null },
282
- () => invoke(params)
304
+ // HTTP transport's `internal` flag, the serving McpServer) is spread in,
305
+ // not replaced — and stdio callers, who have no transport-provided store,
306
+ // get one here.
307
+ //
308
+ // `ctx` is the SDK's per-request context (v2 calls a tool callback with
309
+ // `(args, ctx)`). It is passed straight through to the handler; existing
310
+ // 1-arity handlers ignore it. Its request id is stamped on the context so a
311
+ // server-to-client request sent from inside the tool can ride the same
312
+ // stream as this call — see servingRequestId() in requestContext.js.
313
+ return async (params, ctx) => requestContext.run(
314
+ {
315
+ ...(requestContext.getStore() ?? {}),
316
+ preflightRefusal: null,
317
+ actualCost: null,
318
+ servingRequestId: ctx?.mcpReq?.id
319
+ },
320
+ () => invoke(params, ctx)
283
321
  );
284
322
  };
285
323
  }