crawlforge-mcp-server 5.10.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -5
- package/server.js +201 -249
- package/src/core/ActionExecutor.js +1 -1
- package/src/core/ChangeTracker.js +1 -1
- package/src/core/ElicitationHelper.js +83 -34
- package/src/core/SamplingClient.js +8 -2
- package/src/core/analysis/ContentAnalyzer.js +1 -1
- package/src/core/processing/BrowserProcessor.js +1 -1
- package/src/core/processing/ContentProcessor.js +1 -1
- package/src/core/processing/PDFProcessor.js +2 -2
- package/src/server/registerTool.js +1 -1
- package/src/server/specHygiene.js +17 -22
- package/src/server/transports/stdio.js +2 -3
- package/src/server/transports/streamableHttp.js +128 -66
- package/src/tools/crawl/crawlDeep.js +4 -4
- package/src/tools/extract/analyzeContent.js +1 -1
- package/src/tools/extract/extractContent.js +1 -1
- package/src/tools/extract/processDocument.js +1 -1
- package/src/tools/extract/summarizeContent.js +1 -1
- package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
- package/src/tools/research/deepResearch.js +1 -1
- package/src/tools/tracking/trackChanges/schema.js +4 -4
- package/src/utils/HumanBehaviorSimulator.js +7 -7
- package/src/server/taskSupport.js +0 -233
- package/src/server/transports/http.js +0 -22
|
@@ -1,233 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* taskSupport.js — adapter for the MCP `io.modelcontextprotocol/tasks` extension
|
|
3
|
-
* (SDK "experimental" tasks API, @modelcontextprotocol/sdk 1.30.0).
|
|
4
|
-
*
|
|
5
|
-
* Lets long-running tools (crawl_deep, batch_scrape, deep_research, agent) return
|
|
6
|
-
* a task handle immediately and be polled via tasks/get + tasks/result, while a
|
|
7
|
-
* plain (non-task-augmented) tools/call still resolves synchronously via the
|
|
8
|
-
* SDK's own automatic task-polling path (McpServer taskSupport: 'optional').
|
|
9
|
-
*
|
|
10
|
-
* Exports are a frozen contract shared with server.js — do not rename:
|
|
11
|
-
* createTaskStore, TASK_EXECUTION, TASKS_CAPABILITY, makeTaskToolHandler
|
|
12
|
-
*/
|
|
13
|
-
|
|
14
|
-
import { randomUUID } from 'node:crypto';
|
|
15
|
-
import { isTerminal } from '@modelcontextprotocol/sdk/experimental/tasks';
|
|
16
|
-
|
|
17
|
-
const DEFAULT_TTL_MS = 10 * 60 * 1000; // 10 minutes
|
|
18
|
-
const MAX_TTL_MS = 30 * 60 * 1000; // 30 minutes
|
|
19
|
-
const DEFAULT_POLL_INTERVAL_MS = 200;
|
|
20
|
-
|
|
21
|
-
const NOOP_LOGGER = {
|
|
22
|
-
debug() {},
|
|
23
|
-
info() {},
|
|
24
|
-
warn() {},
|
|
25
|
-
error() {}
|
|
26
|
-
};
|
|
27
|
-
|
|
28
|
-
/**
|
|
29
|
-
* Minimal in-memory TaskStore implementing the SDK's TaskStore interface
|
|
30
|
-
* (experimental/tasks/interfaces.d.ts). Not built on top of the SDK's own
|
|
31
|
-
* InMemoryTaskStore: that class has no default/max TTL policy (an
|
|
32
|
-
* unspecified ttl means "unlimited", i.e. never cleaned up) and its cleanup
|
|
33
|
-
* timers are not unref'd, and neither is overridable from outside since its
|
|
34
|
-
* fields are private with no subclass hook.
|
|
35
|
-
*/
|
|
36
|
-
class MemoryTaskStore {
|
|
37
|
-
constructor({ logger = NOOP_LOGGER } = {}) {
|
|
38
|
-
this.logger = logger;
|
|
39
|
-
this.tasks = new Map();
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
// Every task gets an automatic expiry: a requested ttl is honored up to
|
|
43
|
-
// MAX_TTL_MS, and an unspecified/null ("unlimited") ttl falls back to
|
|
44
|
-
// DEFAULT_TTL_MS — this server never lets a task linger forever.
|
|
45
|
-
_clampTtl(ttl) {
|
|
46
|
-
if (ttl === undefined || ttl === null) return DEFAULT_TTL_MS;
|
|
47
|
-
return Math.min(ttl, MAX_TTL_MS);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
_scheduleCleanup(taskId, ttl) {
|
|
51
|
-
const stored = this.tasks.get(taskId);
|
|
52
|
-
if (!stored) return;
|
|
53
|
-
if (stored.cleanupTimer) clearTimeout(stored.cleanupTimer);
|
|
54
|
-
stored.cleanupTimer = setTimeout(() => {
|
|
55
|
-
this.tasks.delete(taskId);
|
|
56
|
-
}, ttl).unref();
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
async createTask(taskParams, requestId, request, sessionId) {
|
|
60
|
-
const ttl = this._clampTtl(taskParams?.ttl);
|
|
61
|
-
const taskId = randomUUID();
|
|
62
|
-
const createdAt = new Date().toISOString();
|
|
63
|
-
const task = {
|
|
64
|
-
taskId,
|
|
65
|
-
status: 'working',
|
|
66
|
-
ttl,
|
|
67
|
-
createdAt,
|
|
68
|
-
lastUpdatedAt: createdAt,
|
|
69
|
-
pollInterval: taskParams?.pollInterval ?? DEFAULT_POLL_INTERVAL_MS
|
|
70
|
-
};
|
|
71
|
-
this.tasks.set(taskId, { task, requestId, request, sessionId, result: undefined, cleanupTimer: null });
|
|
72
|
-
this._scheduleCleanup(taskId, ttl);
|
|
73
|
-
this.logger.debug(`[tasks] created task ${taskId}`, { ttl });
|
|
74
|
-
return task;
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
async getTask(taskId) {
|
|
78
|
-
const stored = this.tasks.get(taskId);
|
|
79
|
-
return stored ? { ...stored.task } : null;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
async storeTaskResult(taskId, status, result) {
|
|
83
|
-
const stored = this.tasks.get(taskId);
|
|
84
|
-
if (!stored) {
|
|
85
|
-
throw new Error(`Task ${taskId} not found`);
|
|
86
|
-
}
|
|
87
|
-
if (isTerminal(stored.task.status)) {
|
|
88
|
-
throw new Error(`Cannot store result for task ${taskId} in terminal status '${stored.task.status}'`);
|
|
89
|
-
}
|
|
90
|
-
stored.result = result;
|
|
91
|
-
stored.task.status = status;
|
|
92
|
-
stored.task.lastUpdatedAt = new Date().toISOString();
|
|
93
|
-
this._scheduleCleanup(taskId, stored.task.ttl);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
async getTaskResult(taskId) {
|
|
97
|
-
const stored = this.tasks.get(taskId);
|
|
98
|
-
if (!stored) {
|
|
99
|
-
throw new Error(`Task ${taskId} not found`);
|
|
100
|
-
}
|
|
101
|
-
if (stored.result === undefined) {
|
|
102
|
-
throw new Error(`Task ${taskId} has no result stored`);
|
|
103
|
-
}
|
|
104
|
-
return stored.result;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
async updateTaskStatus(taskId, status, statusMessage) {
|
|
108
|
-
const stored = this.tasks.get(taskId);
|
|
109
|
-
if (!stored) {
|
|
110
|
-
throw new Error(`Task ${taskId} not found`);
|
|
111
|
-
}
|
|
112
|
-
if (isTerminal(stored.task.status)) {
|
|
113
|
-
throw new Error(`Cannot update task ${taskId} from terminal status '${stored.task.status}'`);
|
|
114
|
-
}
|
|
115
|
-
stored.task.status = status;
|
|
116
|
-
if (statusMessage) stored.task.statusMessage = statusMessage;
|
|
117
|
-
stored.task.lastUpdatedAt = new Date().toISOString();
|
|
118
|
-
if (isTerminal(status)) this._scheduleCleanup(taskId, stored.task.ttl);
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
async listTasks(cursor) {
|
|
122
|
-
const PAGE_SIZE = 50;
|
|
123
|
-
const ids = Array.from(this.tasks.keys());
|
|
124
|
-
let start = 0;
|
|
125
|
-
if (cursor) {
|
|
126
|
-
const idx = ids.indexOf(cursor);
|
|
127
|
-
if (idx < 0) throw new Error(`Invalid cursor: ${cursor}`);
|
|
128
|
-
start = idx + 1;
|
|
129
|
-
}
|
|
130
|
-
const pageIds = ids.slice(start, start + PAGE_SIZE);
|
|
131
|
-
const tasks = pageIds.map((id) => ({ ...this.tasks.get(id).task }));
|
|
132
|
-
const nextCursor = start + PAGE_SIZE < ids.length ? pageIds[pageIds.length - 1] : undefined;
|
|
133
|
-
return { tasks, nextCursor };
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
/** Clears all pending cleanup timers (graceful shutdown / test teardown). */
|
|
137
|
-
destroy() {
|
|
138
|
-
for (const stored of this.tasks.values()) {
|
|
139
|
-
if (stored.cleanupTimer) clearTimeout(stored.cleanupTimer);
|
|
140
|
-
}
|
|
141
|
-
this.tasks.clear();
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* @param {{logger?: object}} [opts]
|
|
147
|
-
* @returns {MemoryTaskStore} an SDK-compatible TaskStore instance
|
|
148
|
-
*/
|
|
149
|
-
export function createTaskStore({ logger } = {}) {
|
|
150
|
-
return new MemoryTaskStore({ logger });
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
/** execution config for tools registered via registerToolTask */
|
|
154
|
-
export const TASK_EXECUTION = { taskSupport: 'optional' };
|
|
155
|
-
|
|
156
|
-
/**
|
|
157
|
-
* Server capabilities object enabling the tasks extension. Pass to
|
|
158
|
-
* server.server.registerCapabilities(TASKS_CAPABILITY) before connecting
|
|
159
|
-
* the transport.
|
|
160
|
-
*/
|
|
161
|
-
export const TASKS_CAPABILITY = {
|
|
162
|
-
tasks: {
|
|
163
|
-
list: {},
|
|
164
|
-
cancel: {},
|
|
165
|
-
requests: {
|
|
166
|
-
tools: {
|
|
167
|
-
call: {}
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
};
|
|
172
|
-
|
|
173
|
-
/**
|
|
174
|
-
* Builds a ToolTaskHandler ({ createTask, getTask, getTaskResult }) suitable
|
|
175
|
-
* for server.experimental.tasks.registerToolTask(name, config, handler).
|
|
176
|
-
*
|
|
177
|
-
* `run` is the existing withAuth-wrapped tool handler: async (args) => CallToolResult.
|
|
178
|
-
* It is started in the background from createTask (never awaited there) so the
|
|
179
|
-
* request returns a task handle immediately.
|
|
180
|
-
*
|
|
181
|
-
* @param {{name: string, run: (args: any) => Promise<any>, taskStore: object, logger?: object}} opts
|
|
182
|
-
*/
|
|
183
|
-
// `taskStore` (the global store) is accepted to keep this signature consistent
|
|
184
|
-
// with createTaskStore's return value, but request handling below uses
|
|
185
|
-
// extra.taskStore — the SDK's request-scoped wrapper, which also emits
|
|
186
|
-
// notifications/tasks/status on every update.
|
|
187
|
-
export function makeTaskToolHandler({ name, run, taskStore, logger = NOOP_LOGGER }) {
|
|
188
|
-
return {
|
|
189
|
-
async createTask(args, extra) {
|
|
190
|
-
const task = await extra.taskStore.createTask({ ttl: extra.taskRequestedTtl });
|
|
191
|
-
logger.debug(`[tasks] ${name}: task ${task.taskId} created`);
|
|
192
|
-
|
|
193
|
-
// Never await run() here — the whole point is to return the task handle now.
|
|
194
|
-
Promise.resolve()
|
|
195
|
-
.then(() => run(args))
|
|
196
|
-
.then(async (result) => {
|
|
197
|
-
try {
|
|
198
|
-
await extra.taskStore.storeTaskResult(task.taskId, 'completed', result);
|
|
199
|
-
logger.debug(`[tasks] ${name}: task ${task.taskId} completed`);
|
|
200
|
-
} catch (storeError) {
|
|
201
|
-
// Task reached a terminal state (e.g. cancelled) before this finished.
|
|
202
|
-
logger.debug(`[tasks] ${name}: dropped late result for task ${task.taskId}: ${storeError.message}`);
|
|
203
|
-
}
|
|
204
|
-
})
|
|
205
|
-
.catch(async (error) => {
|
|
206
|
-
const errorResult = {
|
|
207
|
-
content: [{ type: 'text', text: `Operation failed: ${error instanceof Error ? error.message : String(error)}` }],
|
|
208
|
-
isError: true
|
|
209
|
-
};
|
|
210
|
-
try {
|
|
211
|
-
await extra.taskStore.storeTaskResult(task.taskId, 'failed', errorResult);
|
|
212
|
-
logger.debug(`[tasks] ${name}: task ${task.taskId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
213
|
-
} catch (storeError) {
|
|
214
|
-
logger.debug(`[tasks] ${name}: dropped late failure for task ${task.taskId}: ${storeError.message}`);
|
|
215
|
-
}
|
|
216
|
-
})
|
|
217
|
-
.catch((fatal) => {
|
|
218
|
-
// Last-resort guard: this handler must never produce an unhandled rejection.
|
|
219
|
-
logger.error(`[tasks] ${name}: unexpected error finalizing task ${task.taskId}`, fatal);
|
|
220
|
-
});
|
|
221
|
-
|
|
222
|
-
return { task };
|
|
223
|
-
},
|
|
224
|
-
|
|
225
|
-
async getTask(args, extra) {
|
|
226
|
-
return extra.taskStore.getTask(extra.taskId);
|
|
227
|
-
},
|
|
228
|
-
|
|
229
|
-
async getTaskResult(args, extra) {
|
|
230
|
-
return extra.taskStore.getTaskResult(extra.taskId);
|
|
231
|
-
}
|
|
232
|
-
};
|
|
233
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HTTP transport — back-compat shim.
|
|
3
|
-
*
|
|
4
|
-
* As of v3.2.0 ("Modernize") the canonical HTTP entry point is
|
|
5
|
-
* `connectStreamableHttp` in ./streamableHttp.js. This module is retained
|
|
6
|
-
* so older imports (`./http.js`) keep working; it forwards to the new
|
|
7
|
-
* implementation in stateless ("legacy") mode by default.
|
|
8
|
-
*
|
|
9
|
-
* @deprecated Use connectStreamableHttp from ./streamableHttp.js
|
|
10
|
-
*/
|
|
11
|
-
|
|
12
|
-
import { connectStreamableHttp } from './streamableHttp.js';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* @param {import('@modelcontextprotocol/sdk/server/mcp.js').McpServer} server
|
|
16
|
-
* @param {import('../../core/AuthManager.js').default} authManager
|
|
17
|
-
* @param {import('../../utils/Logger.js').logger} logger
|
|
18
|
-
* @param {number} [port=3000]
|
|
19
|
-
*/
|
|
20
|
-
export async function connectHttp(server, authManager, logger, port = 3000) {
|
|
21
|
-
return connectStreamableHttp(server, authManager, logger, { port, legacy: true });
|
|
22
|
-
}
|