crawlforge-mcp-server 5.10.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -5
- package/server.js +201 -249
- package/src/core/ActionExecutor.js +1 -1
- package/src/core/ChangeTracker.js +1 -1
- package/src/core/ElicitationHelper.js +83 -34
- package/src/core/SamplingClient.js +8 -2
- package/src/core/analysis/ContentAnalyzer.js +1 -1
- package/src/core/processing/BrowserProcessor.js +1 -1
- package/src/core/processing/ContentProcessor.js +1 -1
- package/src/core/processing/PDFProcessor.js +2 -2
- package/src/server/registerTool.js +1 -1
- package/src/server/specHygiene.js +17 -22
- package/src/server/transports/stdio.js +2 -3
- package/src/server/transports/streamableHttp.js +128 -66
- package/src/tools/crawl/crawlDeep.js +4 -4
- package/src/tools/extract/analyzeContent.js +1 -1
- package/src/tools/extract/extractContent.js +1 -1
- package/src/tools/extract/processDocument.js +1 -1
- package/src/tools/extract/summarizeContent.js +1 -1
- package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
- package/src/tools/research/deepResearch.js +1 -1
- package/src/tools/tracking/trackChanges/schema.js +4 -4
- package/src/utils/HumanBehaviorSimulator.js +7 -7
- package/src/server/taskSupport.js +0 -233
- package/src/server/transports/http.js +0 -22
|
@@ -1,42 +1,62 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Streamable HTTP transport
|
|
2
|
+
* Dual-era Streamable HTTP transport.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* - GET /mcp — SSE stream for server → client notifications
|
|
7
|
-
* - DELETE /mcp — terminate session
|
|
4
|
+
* One endpoint at /mcp serves both protocol eras, routed by the SDK's own
|
|
5
|
+
* classification (`isLegacyRequest`) so this module can never disagree with it:
|
|
8
6
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
7
|
+
* - 2026-07-28 ("modern"): stateless, one server instance per request, each
|
|
8
|
+
* request carrying its own `_meta` envelope (protocol version, clientInfo,
|
|
9
|
+
* clientCapabilities) plus the SEP-2243 `Mcp-Method` / `Mcp-Name` headers.
|
|
10
|
+
* Served by `createMcpHandler(..., { legacy: 'reject' })`, which owns the
|
|
11
|
+
* Content-Type gate (415), the header/body cross-checks (-32020) and
|
|
12
|
+
* `server/discover`.
|
|
13
|
+
* - 2025-era ("legacy"): the sessionful path below — POST /mcp initialize
|
|
14
|
+
* issues an `Mcp-Session-Id`, GET /mcp opens the notification SSE stream,
|
|
15
|
+
* DELETE /mcp terminates the session. One transport + cloned McpServer per
|
|
16
|
+
* session, kept in the `sessions` Map.
|
|
12
17
|
*
|
|
13
18
|
* Auth:
|
|
14
|
-
* - Bearer / X-API-Key required per request
|
|
19
|
+
* - Bearer / X-API-Key required per request on BOTH eras, before any era
|
|
20
|
+
* routing happens (creator mode bypasses, loopback only)
|
|
15
21
|
* - When OAuth is enabled (CRAWLFORGE_OAUTH_ENABLED=true), OAuth bearer
|
|
16
22
|
* tokens are validated by the OAuth provider and mapped server-side to
|
|
17
23
|
* a CrawlForge API key. See src/server/auth/oauth.js.
|
|
18
24
|
*
|
|
19
25
|
* Observability:
|
|
20
26
|
* - GET /metrics returns Prometheus exposition (when observability enabled)
|
|
21
|
-
* - GET /health returns liveness probe
|
|
22
|
-
*
|
|
23
|
-
* Replaces the legacy stateless http.js. Old /mcp endpoint behavior is
|
|
24
|
-
* preserved when CRAWLFORGE_LEGACY_HTTP=true (one-release deprecation window);
|
|
25
|
-
* `http.js`'s connectHttp() forwards straight into this module's legacy mode.
|
|
27
|
+
* - GET /health returns liveness probe + the protocol revisions served
|
|
26
28
|
*/
|
|
27
|
-
|
|
28
|
-
import {
|
|
29
|
-
import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
29
|
+
import { McpServer, createMcpHandler, isLegacyRequest, SUPPORTED_PROTOCOL_VERSIONS } from "@modelcontextprotocol/server";
|
|
30
|
+
import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from "@modelcontextprotocol/node";
|
|
30
31
|
import { createServer } from 'node:http';
|
|
31
32
|
import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
32
33
|
import { readFileSync } from 'node:fs';
|
|
33
34
|
import { requestContext } from '../requestContext.js';
|
|
34
|
-
import {
|
|
35
|
-
import { zodToJsonSchema } from 'zod-to-json-schema';
|
|
35
|
+
import { applySpecHygiene } from '../specHygiene.js';
|
|
36
36
|
|
|
37
37
|
const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
|
|
38
38
|
const SERVER_VERSION = pkg.version;
|
|
39
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Protocol revisions this endpoint serves, newest first: the modern era's
|
|
42
|
+
* revisions followed by the 2025-era list the SDK negotiates via `initialize`.
|
|
43
|
+
*
|
|
44
|
+
* The modern list mirrors the SDK's internal SUPPORTED_MODERN_PROTOCOL_VERSIONS,
|
|
45
|
+
* which is deliberately not exported. streamableHttp.test.js pins it against a
|
|
46
|
+
* live `server/discover` result, so an SDK upgrade that adds a revision fails a
|
|
47
|
+
* test rather than drifting silently.
|
|
48
|
+
*/
|
|
49
|
+
const MODERN_PROTOCOL_VERSIONS = Object.freeze(['2026-07-28']);
|
|
50
|
+
const PROTOCOL_VERSIONS = Object.freeze([...MODERN_PROTOCOL_VERSIONS, ...SUPPORTED_PROTOCOL_VERSIONS]);
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* SEP-2549 cache hint for the `server/discover` result (2026-07-28 only). The
|
|
54
|
+
* advertisement is the same for every caller and only changes when the server
|
|
55
|
+
* is redeployed, so it is `public`; the 5-minute TTL matches the tools/call
|
|
56
|
+
* hints in specHygiene.js. Without a hint the SDK emits `0` / `'private'`.
|
|
57
|
+
*/
|
|
58
|
+
const DISCOVER_CACHE_HINT = Object.freeze({ ttlMs: 300000, cacheScope: 'public' });
|
|
59
|
+
|
|
40
60
|
/**
|
|
41
61
|
* Build the `tools` array for the Smithery static server card, straight from
|
|
42
62
|
* the live tool registry.
|
|
@@ -59,10 +79,9 @@ function buildToolCards(server) {
|
|
|
59
79
|
.map(([name, tool]) => {
|
|
60
80
|
let inputSchema = { type: 'object', properties: {} };
|
|
61
81
|
try {
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
const
|
|
65
|
-
const converted = zodToJsonSchema(zodObject, { $refStrategy: 'none' });
|
|
82
|
+
if (tool?.inputSchema) {
|
|
83
|
+
// The SDK's own conversion, so the card mirrors what tools/list serves.
|
|
84
|
+
const converted = { ...server.toolInputSchemaJson(name) };
|
|
66
85
|
delete converted.$schema;
|
|
67
86
|
inputSchema = converted;
|
|
68
87
|
}
|
|
@@ -89,23 +108,34 @@ function buildToolCards(server) {
|
|
|
89
108
|
* prompt tables — plain config + handler-closure references, no per-connection
|
|
90
109
|
* state — then re-runs the same internal handler-wiring methods McpServer
|
|
91
110
|
* itself calls from registerTool/registerResource/registerPrompt. This
|
|
92
|
-
* depends on
|
|
93
|
-
*
|
|
94
|
-
*
|
|
111
|
+
* depends on the SDK's internal McpServer/Server field names
|
|
112
|
+
* (`_registered*`, `set*RequestHandlers`, `_capabilities`); re-verified
|
|
113
|
+
* against @modelcontextprotocol/server 2.0.0, which still exposes all of
|
|
114
|
+
* them. `_taskStore` is gone — v2 removed experimental tasks (SEP-2663).
|
|
115
|
+
* Re-check on SDK upgrades.
|
|
116
|
+
*
|
|
117
|
+
* The same clone backs a 2025-era session and a single 2026-era request, so
|
|
118
|
+
* both eras serve exactly the same tools — the SDK's "one factory for both
|
|
119
|
+
* legs" rule.
|
|
120
|
+
*
|
|
121
|
+
* applySpecHygiene() runs on the clone because its wrappers live on the
|
|
122
|
+
* template's own Protocol instance, not in the `_registered*` tables the clone
|
|
123
|
+
* copies: without this call an HTTP client got unsorted, icon-less tools/list
|
|
124
|
+
* results and no SEP-2549 cache markers on tools/call, while a stdio client
|
|
125
|
+
* got all three.
|
|
95
126
|
*
|
|
96
|
-
* @param {import('@modelcontextprotocol/
|
|
127
|
+
* @param {import('@modelcontextprotocol/server').McpServer} templateServer
|
|
97
128
|
*/
|
|
98
129
|
function cloneServerForSession(templateServer) {
|
|
99
130
|
const low = templateServer.server;
|
|
100
|
-
// capabilities
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// 'No task store provided for task-capable tool.'
|
|
131
|
+
// capabilities must survive the clone so the session server advertises the
|
|
132
|
+
// same surface as the template. cacheHints only reaches the 2026-era encode
|
|
133
|
+
// seam (it rides a symbol-keyed property that is never serialized), so a
|
|
134
|
+
// 2025-era response is byte-identical with or without it.
|
|
105
135
|
const sessionServer = new McpServer(low._serverInfo, {
|
|
106
136
|
instructions: low._instructions,
|
|
107
137
|
capabilities: low._capabilities,
|
|
108
|
-
|
|
138
|
+
cacheHints: { 'server/discover': DISCOVER_CACHE_HINT }
|
|
109
139
|
});
|
|
110
140
|
|
|
111
141
|
sessionServer._registeredTools = templateServer._registeredTools;
|
|
@@ -118,9 +148,18 @@ function cloneServerForSession(templateServer) {
|
|
|
118
148
|
if (templateServer._promptHandlersInitialized) sessionServer.setPromptRequestHandlers();
|
|
119
149
|
if (templateServer._completionHandlerInitialized) sessionServer.setCompletionRequestHandler();
|
|
120
150
|
|
|
151
|
+
applySpecHygiene(sessionServer);
|
|
152
|
+
|
|
121
153
|
return sessionServer;
|
|
122
154
|
}
|
|
123
155
|
|
|
156
|
+
/** Reads a request body to completion as UTF-8. */
|
|
157
|
+
async function readRequestBody(req) {
|
|
158
|
+
const chunks = [];
|
|
159
|
+
for await (const chunk of req) chunks.push(chunk);
|
|
160
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
161
|
+
}
|
|
162
|
+
|
|
124
163
|
/** Best-effort close — swallows errors so cleanup never throws into a request handler. */
|
|
125
164
|
function safeClose(closable) {
|
|
126
165
|
if (closable && typeof closable.close === 'function') {
|
|
@@ -138,14 +177,14 @@ function sendRpcError(res, status, code, message) {
|
|
|
138
177
|
}
|
|
139
178
|
|
|
140
179
|
/**
|
|
141
|
-
*
|
|
180
|
+
* Dual-era Streamable HTTP transport: stateless 2026-07-28 and sessionful
|
|
181
|
+
* 2025-era traffic on the same /mcp route.
|
|
142
182
|
*
|
|
143
|
-
* @param {import('@modelcontextprotocol/
|
|
183
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
144
184
|
* @param {import('../../core/AuthManager.js').default} authManager
|
|
145
185
|
* @param {import('../../utils/Logger.js').logger} logger
|
|
146
186
|
* @param {object} [options]
|
|
147
187
|
* @param {number} [options.port=3000]
|
|
148
|
-
* @param {boolean} [options.legacy=false] — if true, run in stateless mode (3.1 behavior)
|
|
149
188
|
* @param {object} [options.oauth] — OAuth provider (see src/server/auth/oauth.js)
|
|
150
189
|
* @param {object} [options.metrics] — Prometheus registry (see src/observability/metrics.js)
|
|
151
190
|
*/
|
|
@@ -159,22 +198,37 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
159
198
|
if (authManager.isCreatorMode() && !hostIsLoopback) {
|
|
160
199
|
console.error(`WARNING: creator mode is enabled but the server is bound to ${host} (non-loopback) — per-request auth will NOT be bypassed. Bind to 127.0.0.1 to use creator mode.`);
|
|
161
200
|
}
|
|
162
|
-
const legacy = options.legacy === true;
|
|
163
201
|
const oauthProvider = options.oauth ?? null;
|
|
164
202
|
const metrics = options.metrics ?? null;
|
|
165
203
|
|
|
166
|
-
const mode =
|
|
204
|
+
const mode = 'streamable-stateful';
|
|
167
205
|
const toolCount = Object.keys(server._registeredTools ?? {}).length;
|
|
168
206
|
|
|
169
207
|
// sessionId -> { transport, server }. One StreamableHTTPServerTransport (and
|
|
170
208
|
// therefore one cloned McpServer — see cloneServerForSession) per session.
|
|
209
|
+
// 2025-era only: the modern era is stateless and holds nothing here.
|
|
171
210
|
const sessions = new Map();
|
|
172
211
|
|
|
212
|
+
// 2026-07-28 leg. `legacy: 'reject'` keeps it strict — every 2025-era request
|
|
213
|
+
// is routed to the sessions Map above by isLegacyRequest before it can reach
|
|
214
|
+
// this handler, so the modern leg never has to serve one. The SDK owns the
|
|
215
|
+
// Content-Type gate (415), the Mcp-Method/Mcp-Name cross-checks (-32020 on
|
|
216
|
+
// 400) and `server/discover`; nothing here re-implements them.
|
|
217
|
+
const modernHandler = createMcpHandler(() => cloneServerForSession(server), {
|
|
218
|
+
legacy: 'reject',
|
|
219
|
+
onerror: (err) => logger.warn('2026-era MCP request rejected', { error: err?.message })
|
|
220
|
+
});
|
|
221
|
+
const serveModern = toNodeHandler(modernHandler, {
|
|
222
|
+
onerror: (err) => logger.error('2026-era MCP request failed', { error: err?.message })
|
|
223
|
+
});
|
|
224
|
+
|
|
173
225
|
const httpServer = createServer(async (req, res) => {
|
|
174
226
|
// CORS — Smithery + browser-based MCP clients
|
|
175
227
|
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
176
228
|
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS');
|
|
177
|
-
|
|
229
|
+
// MCP-Protocol-Version / Mcp-Method / Mcp-Name are the 2026-07-28 era's
|
|
230
|
+
// request headers; the session id headers are the 2025 era's.
|
|
231
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id, mcp-session-id, MCP-Protocol-Version, Mcp-Method, Mcp-Name, Authorization, X-API-Key, X-Internal-Secret');
|
|
178
232
|
res.setHeader('Access-Control-Expose-Headers', 'Mcp-Session-Id, mcp-session-id');
|
|
179
233
|
|
|
180
234
|
if (req.method === 'OPTIONS') {
|
|
@@ -186,7 +240,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
186
240
|
// Health probe
|
|
187
241
|
if (req.url === '/health') {
|
|
188
242
|
res.writeHead(200, { 'Content-Type': 'application/json' });
|
|
189
|
-
res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode }));
|
|
243
|
+
res.end(JSON.stringify({ status: 'ok', version: SERVER_VERSION, mode, protocolVersions: PROTOCOL_VERSIONS }));
|
|
190
244
|
return;
|
|
191
245
|
}
|
|
192
246
|
|
|
@@ -273,38 +327,42 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
273
327
|
internal = authResult.internal === true;
|
|
274
328
|
}
|
|
275
329
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
330
|
+
// Era routing. Only a POST can carry the 2026-07-28 per-request envelope;
|
|
331
|
+
// body-less GET/DELETE are 2025 session operations by construction and
|
|
332
|
+
// isLegacyRequest classifies them as such, so they skip this entirely and
|
|
333
|
+
// keep their existing behaviour byte-for-byte.
|
|
334
|
+
//
|
|
335
|
+
// Reading the body here drains the Node stream, so the parsed value is
|
|
336
|
+
// handed to whichever leg serves the request. A body that is not valid
|
|
337
|
+
// JSON classifies legacy (the SDK's own rule), and the 2025 transport
|
|
338
|
+
// still writes its own parse error — hence the undefined pass-through
|
|
339
|
+
// rather than an answer invented here.
|
|
340
|
+
let parsedBody;
|
|
341
|
+
if (req.method === 'POST') {
|
|
282
342
|
try {
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
});
|
|
343
|
+
parsedBody = JSON.parse(await readRequestBody(req));
|
|
344
|
+
} catch {
|
|
345
|
+
parsedBody = undefined;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
if (parsedBody !== undefined) {
|
|
349
|
+
const probe = await toWebRequest(req, parsedBody);
|
|
350
|
+
if (!(await isLegacyRequest(probe, parsedBody))) {
|
|
351
|
+
await requestContext.run({ internal }, () => serveModern(req, res, parsedBody));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
295
354
|
}
|
|
296
|
-
return;
|
|
297
355
|
}
|
|
298
356
|
|
|
299
|
-
//
|
|
300
|
-
//
|
|
357
|
+
// 2025 era: route by Mcp-Session-Id. A request without the header must be
|
|
358
|
+
// a fresh initialize, which gets its own transport + server pair
|
|
301
359
|
// (independent of any prior session's lifecycle) so reconnects/re-inits
|
|
302
360
|
// never hit a stuck 'already initialized' transport.
|
|
303
361
|
const sessionIdHeader = req.headers['mcp-session-id'];
|
|
304
362
|
const existing = sessionIdHeader ? sessions.get(String(sessionIdHeader)) : undefined;
|
|
305
363
|
|
|
306
364
|
if (existing) {
|
|
307
|
-
await requestContext.run({ internal }, () => existing.transport.handleRequest(req, res));
|
|
365
|
+
await requestContext.run({ internal }, () => existing.transport.handleRequest(req, res, parsedBody));
|
|
308
366
|
return;
|
|
309
367
|
}
|
|
310
368
|
|
|
@@ -321,7 +379,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
321
379
|
}
|
|
322
380
|
|
|
323
381
|
const sessionServer = cloneServerForSession(server);
|
|
324
|
-
const transport = new
|
|
382
|
+
const transport = new NodeStreamableHTTPServerTransport({
|
|
325
383
|
sessionIdGenerator: () => randomUUID(),
|
|
326
384
|
onsessioninitialized: (sid) => {
|
|
327
385
|
sessions.set(sid, { transport, server: sessionServer });
|
|
@@ -337,7 +395,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
337
395
|
|
|
338
396
|
try {
|
|
339
397
|
await sessionServer.connect(transport);
|
|
340
|
-
await requestContext.run({ internal }, () => transport.handleRequest(req, res));
|
|
398
|
+
await requestContext.run({ internal }, () => transport.handleRequest(req, res, parsedBody));
|
|
341
399
|
} catch (err) {
|
|
342
400
|
logger.error('Streamable HTTP session initialization failed', { error: err?.message });
|
|
343
401
|
safeClose(transport);
|
|
@@ -355,7 +413,7 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
355
413
|
httpServer.listen(port, host, () => {
|
|
356
414
|
const actual = httpServer.address()?.port ?? port;
|
|
357
415
|
console.error(`CrawlForge MCP Server v${SERVER_VERSION} listening on ${host}:${actual} (Streamable HTTP, ${mode})`);
|
|
358
|
-
console.error(`MCP endpoint: http://${host}:${actual}/mcp`);
|
|
416
|
+
console.error(`MCP endpoint: http://${host}:${actual}/mcp (protocol ${PROTOCOL_VERSIONS.join(', ')})`);
|
|
359
417
|
console.error(`Health check: http://${host}:${actual}/health`);
|
|
360
418
|
if (metrics) console.error(`Metrics: http://${host}:${actual}/metrics`);
|
|
361
419
|
if (oauthProvider) console.error(`OAuth discovery: http://${host}:${actual}/.well-known/oauth-authorization-server`);
|
|
@@ -366,13 +424,17 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
366
424
|
return {
|
|
367
425
|
httpServer,
|
|
368
426
|
sessions,
|
|
369
|
-
/**
|
|
427
|
+
/**
|
|
428
|
+
* Closes every live 2025-era session's transport + server and the modern
|
|
429
|
+
* leg (aborting in-flight exchanges), then the HTTP server.
|
|
430
|
+
*/
|
|
370
431
|
async close() {
|
|
371
432
|
for (const { transport, server: sessionServer } of sessions.values()) {
|
|
372
433
|
safeClose(transport);
|
|
373
434
|
safeClose(sessionServer);
|
|
374
435
|
}
|
|
375
436
|
sessions.clear();
|
|
437
|
+
await modernHandler.close().catch(() => {});
|
|
376
438
|
await new Promise((resolve) => httpServer.close(() => resolve()));
|
|
377
439
|
}
|
|
378
440
|
};
|
|
@@ -22,7 +22,7 @@ const CrawlDeepSchema = z.object({
|
|
|
22
22
|
dampingFactor: z.number().min(0).max(1).optional().default(0.85),
|
|
23
23
|
maxIterations: z.number().min(1).max(1000).optional().default(100),
|
|
24
24
|
enableCaching: z.boolean().optional().default(true)
|
|
25
|
-
}).optional().
|
|
25
|
+
}).optional().prefault({}),
|
|
26
26
|
// New domain filtering options
|
|
27
27
|
domain_filter: z.object({
|
|
28
28
|
whitelist: z.array(z.union([
|
|
@@ -59,7 +59,7 @@ const CrawlDeepSchema = z.object({
|
|
|
59
59
|
timeout: z.number().optional(),
|
|
60
60
|
maxPages: z.number().optional(),
|
|
61
61
|
concurrency: z.number().optional()
|
|
62
|
-
})).optional().
|
|
62
|
+
})).optional().prefault({})
|
|
63
63
|
}).optional(),
|
|
64
64
|
import_filter_config: z.string().optional(), // JSON string of exported config
|
|
65
65
|
// Session reuse: when enabled, all page fetches share a cookie jar and
|
|
@@ -67,11 +67,11 @@ const CrawlDeepSchema = z.object({
|
|
|
67
67
|
session: z.object({
|
|
68
68
|
enabled: z.boolean(),
|
|
69
69
|
persistCookies: z.boolean().optional().default(true),
|
|
70
|
-
headers: z.record(z.string()).optional().
|
|
70
|
+
headers: z.record(z.string()).optional().prefault({}),
|
|
71
71
|
initialRequest: z.object({
|
|
72
72
|
url: z.string().url(),
|
|
73
73
|
method: z.string().optional().default('GET'),
|
|
74
|
-
headers: z.record(z.string()).optional().
|
|
74
|
+
headers: z.record(z.string()).optional().prefault({}),
|
|
75
75
|
body: z.string().optional()
|
|
76
76
|
}).optional()
|
|
77
77
|
}).optional()
|
|
@@ -26,7 +26,7 @@ const AnalyzeContentSchema = z.object({
|
|
|
26
26
|
includeAdvancedMetrics: z.boolean().default(false),
|
|
27
27
|
groupEntitiesByType: z.boolean().default(true),
|
|
28
28
|
rankByRelevance: z.boolean().default(true)
|
|
29
|
-
}).optional().
|
|
29
|
+
}).optional().prefault({})
|
|
30
30
|
});
|
|
31
31
|
|
|
32
32
|
const AnalyzeContentResult = z.object({
|
|
@@ -45,7 +45,7 @@ const ExtractContentSchema = z.object({
|
|
|
45
45
|
includeRawHTML: z.boolean().default(false),
|
|
46
46
|
includeCleanedHTML: z.boolean().default(false),
|
|
47
47
|
outputFormat: z.enum(['text', 'markdown', 'structured']).default('structured')
|
|
48
|
-
}).optional().
|
|
48
|
+
}).optional().prefault({})
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
const ExtractContentResult = z.object({
|
|
@@ -49,7 +49,7 @@ const ProcessDocumentSchema = z.object({
|
|
|
49
49
|
// Content filtering
|
|
50
50
|
minContentLength: z.number().min(0).default(50),
|
|
51
51
|
removeBoilerplate: z.boolean().default(true)
|
|
52
|
-
}).optional().
|
|
52
|
+
}).optional().prefault({})
|
|
53
53
|
});
|
|
54
54
|
|
|
55
55
|
const ProcessDocumentResult = z.object({
|
|
@@ -28,7 +28,7 @@ const SummarizeContentSchema = z.object({
|
|
|
28
28
|
maxKeywords: z.number().min(1).max(20).default(10),
|
|
29
29
|
preserveStructure: z.boolean().default(false),
|
|
30
30
|
language: z.string().optional()
|
|
31
|
-
}).optional().
|
|
31
|
+
}).optional().prefault({})
|
|
32
32
|
});
|
|
33
33
|
|
|
34
34
|
const SummarizeContentResult = z.object({
|
|
@@ -21,7 +21,7 @@ const GenerateLLMsTxtSchema = z.object({
|
|
|
21
21
|
checkSecurity: z.boolean().optional().default(false).describe('Whether to probe security-sensitive paths (opt-in; sends requests to /admin, /login, etc.)'),
|
|
22
22
|
probeRateLimit: z.boolean().optional().default(false).describe('Whether to send repeated probe requests to estimate rate limits (opt-in; fires ~5 requests)'),
|
|
23
23
|
respectRobots: z.boolean().optional().default(true).describe('Whether to respect robots.txt')
|
|
24
|
-
}).optional().
|
|
24
|
+
}).optional().prefault({}),
|
|
25
25
|
|
|
26
26
|
outputOptions: z.object({
|
|
27
27
|
includeDetailed: z.boolean().optional().default(true).describe('Generate detailed LLMs-full.txt'),
|
|
@@ -31,7 +31,7 @@ const GenerateLLMsTxtSchema = z.object({
|
|
|
31
31
|
customGuidelines: z.array(z.string()).optional().describe('Additional custom guidelines'),
|
|
32
32
|
customRestrictions: z.array(z.string()).optional().describe('Additional restrictions'),
|
|
33
33
|
robotsStyle: z.boolean().optional().default(false).describe('Emit legacy robots.txt-style directives instead of spec-compliant llmstxt.org markdown')
|
|
34
|
-
}).optional().
|
|
34
|
+
}).optional().prefault({}),
|
|
35
35
|
|
|
36
36
|
complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Compliance level for generated guidelines'),
|
|
37
37
|
|
|
@@ -238,7 +238,7 @@ export class DeepResearchTool {
|
|
|
238
238
|
return {
|
|
239
239
|
success: false,
|
|
240
240
|
error: 'Invalid parameters',
|
|
241
|
-
details: validationError.
|
|
241
|
+
details: validationError.issues.map(err => ({
|
|
242
242
|
field: err.path.join('.'),
|
|
243
243
|
message: err.message,
|
|
244
244
|
received: err.received
|
|
@@ -48,7 +48,7 @@ export const TrackChangesSchema = z.object({
|
|
|
48
48
|
moderate: z.number().min(0).max(1).default(0.3),
|
|
49
49
|
major: z.number().min(0).max(1).default(0.7)
|
|
50
50
|
}).optional()
|
|
51
|
-
}).optional().
|
|
51
|
+
}).optional().prefault({}),
|
|
52
52
|
|
|
53
53
|
monitoringOptions: z.object({
|
|
54
54
|
enabled: z.boolean().default(false),
|
|
@@ -59,7 +59,7 @@ export const TrackChangesSchema = z.object({
|
|
|
59
59
|
enableWebhook: z.boolean().default(false),
|
|
60
60
|
webhookUrl: z.string().url().optional(),
|
|
61
61
|
webhookSecret: z.string().optional()
|
|
62
|
-
}).optional().
|
|
62
|
+
}).optional().prefault({}),
|
|
63
63
|
|
|
64
64
|
storageOptions: z.object({
|
|
65
65
|
enableSnapshots: z.boolean().default(true),
|
|
@@ -67,7 +67,7 @@ export const TrackChangesSchema = z.object({
|
|
|
67
67
|
maxHistoryEntries: z.number().min(1).max(1000).default(100),
|
|
68
68
|
compressionEnabled: z.boolean().default(true),
|
|
69
69
|
deltaStorageEnabled: z.boolean().default(true)
|
|
70
|
-
}).optional().
|
|
70
|
+
}).optional().prefault({}),
|
|
71
71
|
|
|
72
72
|
queryOptions: z.object({
|
|
73
73
|
limit: z.number().min(1).max(500).default(50),
|
|
@@ -76,7 +76,7 @@ export const TrackChangesSchema = z.object({
|
|
|
76
76
|
endTime: z.number().optional(),
|
|
77
77
|
includeContent: z.boolean().default(false),
|
|
78
78
|
significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
|
|
79
|
-
}).optional().
|
|
79
|
+
}).optional().prefault({}),
|
|
80
80
|
|
|
81
81
|
notificationOptions: z.object({
|
|
82
82
|
email: z.object({
|
|
@@ -12,7 +12,7 @@ const BehaviorConfigSchema = z.object({
|
|
|
12
12
|
accuracy: z.number().min(0.1).max(1.0).default(0.8), // 0.1 = very inaccurate, 1.0 = perfect
|
|
13
13
|
naturalCurves: z.boolean().default(true),
|
|
14
14
|
randomMicroMovements: z.boolean().default(true)
|
|
15
|
-
}).
|
|
15
|
+
}).prefault({}),
|
|
16
16
|
|
|
17
17
|
typing: z.object({
|
|
18
18
|
enabled: z.boolean().default(true),
|
|
@@ -22,30 +22,30 @@ const BehaviorConfigSchema = z.object({
|
|
|
22
22
|
enabled: z.boolean().default(true),
|
|
23
23
|
frequency: z.number().min(0).max(0.1).default(0.02), // 2% mistake rate
|
|
24
24
|
correctionDelay: z.number().min(100).max(2000).default(500)
|
|
25
|
-
}).
|
|
26
|
-
}).
|
|
25
|
+
}).prefault({})
|
|
26
|
+
}).prefault({}),
|
|
27
27
|
|
|
28
28
|
scrolling: z.object({
|
|
29
29
|
enabled: z.boolean().default(true),
|
|
30
30
|
naturalAcceleration: z.boolean().default(true),
|
|
31
31
|
randomPauses: z.boolean().default(true),
|
|
32
32
|
scrollBackProbability: z.number().min(0).max(1).default(0.1)
|
|
33
|
-
}).
|
|
33
|
+
}).prefault({}),
|
|
34
34
|
|
|
35
35
|
interactions: z.object({
|
|
36
36
|
hoverBeforeClick: z.boolean().default(true),
|
|
37
37
|
clickDelay: z.object({
|
|
38
38
|
min: z.number().default(100),
|
|
39
39
|
max: z.number().default(300)
|
|
40
|
-
}).
|
|
40
|
+
}).prefault({}),
|
|
41
41
|
focusBlurSimulation: z.boolean().default(true),
|
|
42
42
|
idlePeriods: z.object({
|
|
43
43
|
enabled: z.boolean().default(true),
|
|
44
44
|
frequency: z.number().min(0).max(1).default(0.1), // 10% chance
|
|
45
45
|
minDuration: z.number().default(1000),
|
|
46
46
|
maxDuration: z.number().default(5000)
|
|
47
|
-
}).
|
|
48
|
-
}).
|
|
47
|
+
}).prefault({})
|
|
48
|
+
}).prefault({})
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
export class HumanBehaviorSimulator {
|