crawlforge-mcp-server 6.0.0 → 6.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -1
- package/package.json +1 -1
- package/server.js +23 -108
- package/src/cli/commands/login.js +176 -0
- package/src/cli/commands/monitor.js +18 -4
- package/src/cli/index.js +2 -0
- package/src/core/AuthManager.js +19 -6
- package/src/core/ElicitationHelper.js +157 -76
- package/src/server/requestContext.js +50 -0
- package/src/server/transports/streamableHttp.js +17 -4
- package/src/server/withAuth.js +47 -9
- package/src/skills/agent-skills/crawlforge-change-tracking/SKILL.md +53 -14
- package/src/tools/advanced/batchScrape/index.js +29 -19
- package/src/tools/agent/agent.js +9 -4
- package/src/tools/crawl/crawlDeep.js +10 -4
- package/src/tools/extract/extractStructured.js +62 -43
- package/src/tools/research/deepResearch.js +10 -4
- package/src/tools/tracking/trackChanges/hosted.js +176 -0
- package/src/tools/tracking/trackChanges/index.js +123 -7
- package/src/tools/tracking/trackChanges/notifier.js +5 -4
- package/src/tools/tracking/trackChanges/schema.js +36 -22
- package/src/core/AlertNotificationSystem.js +0 -602
|
@@ -1,36 +1,56 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* ElicitationHelper — MCP Elicitation for CrawlForge
|
|
3
3
|
*
|
|
4
|
-
* Allows tools to request user confirmation
|
|
5
|
-
*
|
|
6
|
-
* MCP client does not support elicitation.
|
|
4
|
+
* Allows tools to request user confirmation before an expensive or ambiguous
|
|
5
|
+
* operation. Falls back gracefully when the MCP client cannot be asked.
|
|
7
6
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Phase 4.4 moved confirmations from an inline server→client request to the
|
|
8
|
+
* 2026-07-28 MULTI-ROUND-TRIP form: `confirm()` no longer sends anything and no
|
|
9
|
+
* longer awaits. It returns a verdict, and when the user must be asked the
|
|
10
|
+
* verdict carries an `input_required` result for the tool to RETURN. The SDK
|
|
11
|
+
* then either hands it to a 2026-era client or, on a 2025-era connection, runs
|
|
12
|
+
* its own legacy shim (real `elicitation/create` + handler re-entry). One shape
|
|
13
|
+
* serves both eras, which is why nothing here branches on the protocol version
|
|
14
|
+
* any more — the previous era guard reported "unsupported" on 2026-07-28 and
|
|
15
|
+
* every prompt there was silently skipped.
|
|
11
16
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* that same lenient rule, so those clients get their prompt.
|
|
17
|
+
* THE HANDLER IS RE-ENTERED. Everything a tool does above its gate runs a
|
|
18
|
+
* second time when the answer arrives, so a gate belongs above every fetch and
|
|
19
|
+
* every side effect. Billing is not a caller's problem: `withAuth` charges an
|
|
20
|
+
* `input_required` return zero and reports no usage, so a round trip and a
|
|
21
|
+
* declined confirmation are both free (G4).
|
|
18
22
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
23
|
+
* Two properties of the old helper are deliberately preserved:
|
|
24
|
+
*
|
|
25
|
+
* - **Fail-open.** A client that never declared elicitation is not asked, and
|
|
26
|
+
* the operation proceeds. This is not politeness — the SDK answers an
|
|
27
|
+
* `input_required` return on such a connection with `isError: true`
|
|
28
|
+
* ("did not declare the required capability"), so dropping the capability
|
|
29
|
+
* gate would turn a nicety into a failed call. Verified against the SDK.
|
|
30
|
+
* - **We ask at most once.** `inputResponses` is absent on a first entry and
|
|
31
|
+
* present on a retry, so a retry whose answer did not survive the trip
|
|
32
|
+
* (a dropped key, an answer of another kind) proceeds rather than asking
|
|
33
|
+
* again until the shim's round limit fails the call.
|
|
34
|
+
*
|
|
35
|
+
* The one case that cannot be preserved: a client that DECLARES elicitation and
|
|
36
|
+
* then throws answering it now yields an `isError` result from the SDK where the
|
|
37
|
+
* old inline path proceeded. The failure happens inside the SDK after the
|
|
38
|
+
* handler has returned, so nothing here can intercept it. It costs nothing — the
|
|
39
|
+
* handler did no work, so `withAuth` bills zero.
|
|
40
|
+
*
|
|
41
|
+
* Which server instance we ask matters as much as what we ask. server.js
|
|
42
|
+
* constructs this against the top-level template McpServer, but neither HTTP
|
|
43
|
+
* leg serves from it — the 2025-era path connects a clone per session and the
|
|
44
|
+
* modern leg builds one per request, so the template is never `.connect()`ed
|
|
45
|
+
* and reports no client capabilities. The transport stamps the serving clone on
|
|
46
|
+
* the request context; this resolves it from there and falls back to the
|
|
47
|
+
* injected instance, which on stdio IS the connected one. On a 2026-era request
|
|
48
|
+
* there is no connected instance to read at all — capabilities arrive per
|
|
49
|
+
* request in the `_meta` envelope, which is why `ctx` is consulted first.
|
|
27
50
|
*/
|
|
28
51
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
* lexicographic comparison orders them chronologically (the SDK's own rule).
|
|
32
|
-
*/
|
|
33
|
-
const FIRST_MODERN_PROTOCOL_VERSION = '2026-07-28';
|
|
52
|
+
import { inputRequired, inputResponse, CLIENT_CAPABILITIES_META_KEY } from '@modelcontextprotocol/server';
|
|
53
|
+
import { servingRequestId, servingServer } from '../server/requestContext.js';
|
|
34
54
|
|
|
35
55
|
/** The one-boolean schema a confirmation asks with. */
|
|
36
56
|
const CONFIRM_SCHEMA = {
|
|
@@ -70,69 +90,101 @@ export class ElicitationHelper {
|
|
|
70
90
|
}
|
|
71
91
|
|
|
72
92
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
93
|
+
* The McpServer this request is served from: the clone the transport stamped
|
|
94
|
+
* on the request context, else the constructor-injected instance (stdio, and
|
|
95
|
+
* any caller outside a request context).
|
|
96
|
+
* @private
|
|
97
|
+
*/
|
|
98
|
+
get _server() {
|
|
99
|
+
return servingServer() ?? this._mcpServer;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* The client's declared capabilities for the request in flight. A 2026-era
|
|
104
|
+
* request carries them per-request in the `_meta` envelope and has no
|
|
105
|
+
* connected server instance to read; a 2025-era one has them on the serving
|
|
106
|
+
* instance and no envelope.
|
|
107
|
+
* @private
|
|
75
108
|
*/
|
|
76
|
-
|
|
77
|
-
const
|
|
78
|
-
if (
|
|
109
|
+
_clientCapabilities(ctx) {
|
|
110
|
+
const fromEnvelope = ctx?.mcpReq?.envelope?.[CLIENT_CAPABILITIES_META_KEY];
|
|
111
|
+
if (fromEnvelope) return fromEnvelope;
|
|
79
112
|
try {
|
|
80
|
-
|
|
81
|
-
const negotiated = server.getNegotiatedProtocolVersion?.();
|
|
82
|
-
if (typeof negotiated === 'string' && negotiated >= FIRST_MODERN_PROTOCOL_VERSION) return false;
|
|
83
|
-
return formElicitationDeclared(server.getClientCapabilities?.());
|
|
113
|
+
return this._server?.server?.getClientCapabilities?.();
|
|
84
114
|
} catch {
|
|
85
|
-
return
|
|
115
|
+
return undefined;
|
|
86
116
|
}
|
|
87
117
|
}
|
|
88
118
|
|
|
89
119
|
/**
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* @private
|
|
120
|
+
* Whether asking will actually reach the user rather than fail the call.
|
|
121
|
+
* @param {object} [ctx] the SDK per-request context the handler received
|
|
122
|
+
* @returns {boolean}
|
|
94
123
|
*/
|
|
95
|
-
|
|
96
|
-
return this.
|
|
97
|
-
method: 'elicitation/create',
|
|
98
|
-
params: { message, requestedSchema, mode: 'form' },
|
|
99
|
-
});
|
|
124
|
+
supported(ctx) {
|
|
125
|
+
return formElicitationDeclared(this._clientCapabilities(ctx));
|
|
100
126
|
}
|
|
101
127
|
|
|
102
128
|
/**
|
|
103
129
|
* Ask for user confirmation before proceeding with an expensive operation.
|
|
104
|
-
*
|
|
105
|
-
* so tools continue working in non-elicitation clients).
|
|
130
|
+
* SYNCHRONOUS — it performs no I/O. Do not `await` it.
|
|
106
131
|
*
|
|
107
|
-
* @param {
|
|
108
|
-
* @param {
|
|
109
|
-
* @
|
|
132
|
+
* @param {object|undefined} ctx - the SDK per-request context the handler received
|
|
133
|
+
* @param {string} key - stable identifier for this question, unique across tools
|
|
134
|
+
* @param {string} message - human-readable explanation of what requires confirmation
|
|
135
|
+
* @param {object} [details] - additional context (projected cost, URL count, etc.)
|
|
136
|
+
* @returns {{status:'proceed'}|{status:'cancelled'}|{status:'ask', result: object}}
|
|
137
|
+
* `ask` carries an `input_required` result the caller must RETURN verbatim.
|
|
110
138
|
*/
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
return true;
|
|
115
|
-
}
|
|
139
|
+
confirm(ctx, key, message, details = {}) {
|
|
140
|
+
const responses = ctx?.mcpReq?.inputResponses;
|
|
141
|
+
const answered = inputResponse(responses, key);
|
|
116
142
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
143
|
+
if (answered.kind === 'elicit') {
|
|
144
|
+
// Only an explicit accept + confirmed=true proceeds; decline/cancel = stop.
|
|
145
|
+
return answered.action === 'accept' && answered.content?.confirmed === true
|
|
146
|
+
? { status: 'proceed' }
|
|
147
|
+
: { status: 'cancelled' };
|
|
148
|
+
}
|
|
122
149
|
|
|
123
|
-
|
|
150
|
+
// A retry carries an `inputResponses` object even when this key's answer
|
|
151
|
+
// did not survive it. Asking again would burn the shim's rounds and end in
|
|
152
|
+
// a failed call, so one unanswered round trip proceeds instead.
|
|
153
|
+
if (responses !== undefined) {
|
|
154
|
+
this._logger.warn('Elicitation answer did not come back — proceeding without confirmation', { key });
|
|
155
|
+
return { status: 'proceed' };
|
|
156
|
+
}
|
|
124
157
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
this._logger.warn('Elicitation request failed — proceeding without confirmation', { error: err.message });
|
|
129
|
-
return true; // fail-open
|
|
158
|
+
if (!this.supported(ctx)) {
|
|
159
|
+
this._logger.warn('Elicitation not supported by client — proceeding without confirmation', { message });
|
|
160
|
+
return { status: 'proceed' };
|
|
130
161
|
}
|
|
162
|
+
|
|
163
|
+
const detailLines = Object.entries(details)
|
|
164
|
+
.map(([k, v]) => ` ${k}: ${v}`)
|
|
165
|
+
.join('\n');
|
|
166
|
+
const fullMessage = detailLines ? `${message}\n\n${detailLines}` : message;
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
status: 'ask',
|
|
170
|
+
result: inputRequired({
|
|
171
|
+
inputRequests: {
|
|
172
|
+
[key]: inputRequired.elicit({ message: fullMessage, requestedSchema: CONFIRM_SCHEMA }),
|
|
173
|
+
},
|
|
174
|
+
}),
|
|
175
|
+
};
|
|
131
176
|
}
|
|
132
177
|
|
|
133
178
|
/**
|
|
134
179
|
* Ask the user to provide a string value (e.g. missing schema field).
|
|
135
180
|
*
|
|
181
|
+
* Still the 2025-era inline form, and still reached by no tool — this is the
|
|
182
|
+
* repo's one caller-less elicitation path, left as-is under G6 (dead code is
|
|
183
|
+
* reported, not deleted). It therefore keeps the era guard that `confirm()`
|
|
184
|
+
* shed: an inline request throws on a 2026-era connection, so the default is
|
|
185
|
+
* returned there rather than the call being failed. Converting it to a round
|
|
186
|
+
* trip is speculative until something calls it.
|
|
187
|
+
*
|
|
136
188
|
* @param {string} message
|
|
137
189
|
* @param {object} [options]
|
|
138
190
|
* @param {string} [options.fieldName]
|
|
@@ -141,24 +193,43 @@ export class ElicitationHelper {
|
|
|
141
193
|
* @returns {Promise<string|null>} - The user-provided value or null if cancelled/unsupported
|
|
142
194
|
*/
|
|
143
195
|
async requestString(message, { fieldName = 'value', fieldDescription = '', defaultValue } = {}) {
|
|
144
|
-
|
|
196
|
+
const server = this._server?.server;
|
|
197
|
+
const inlineUsable = typeof server?.request === 'function'
|
|
198
|
+
&& !this._modernEra(server)
|
|
199
|
+
&& formElicitationDeclared(this._clientCapabilities());
|
|
200
|
+
|
|
201
|
+
if (!inlineUsable) {
|
|
145
202
|
this._logger.warn('Elicitation not supported — using default value', { fieldName, defaultValue });
|
|
146
203
|
return defaultValue || null;
|
|
147
204
|
}
|
|
148
205
|
|
|
149
206
|
try {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
207
|
+
// relatedRequestId ties the prompt to the tools/call in flight. Without it
|
|
208
|
+
// the 2025-era HTTP transport puts the request on the standalone GET SSE
|
|
209
|
+
// stream and drops it outright when the client never opened one.
|
|
210
|
+
const relatedRequestId = servingRequestId();
|
|
211
|
+
const result = await server.request(
|
|
212
|
+
{
|
|
213
|
+
method: 'elicitation/create',
|
|
214
|
+
params: {
|
|
215
|
+
message,
|
|
216
|
+
requestedSchema: {
|
|
217
|
+
type: 'object',
|
|
218
|
+
properties: {
|
|
219
|
+
[fieldName]: {
|
|
220
|
+
type: 'string',
|
|
221
|
+
title: fieldName,
|
|
222
|
+
description: fieldDescription,
|
|
223
|
+
...(defaultValue ? { default: defaultValue } : {}),
|
|
224
|
+
},
|
|
225
|
+
},
|
|
226
|
+
required: [fieldName],
|
|
227
|
+
},
|
|
228
|
+
mode: 'form',
|
|
158
229
|
},
|
|
159
230
|
},
|
|
160
|
-
|
|
161
|
-
|
|
231
|
+
relatedRequestId === null ? undefined : { relatedRequestId }
|
|
232
|
+
);
|
|
162
233
|
|
|
163
234
|
// The answer is client-supplied and no longer schema-checked by the SDK
|
|
164
235
|
// on this path, so hold it to the type we asked for.
|
|
@@ -171,4 +242,14 @@ export class ElicitationHelper {
|
|
|
171
242
|
return defaultValue || null;
|
|
172
243
|
}
|
|
173
244
|
}
|
|
245
|
+
|
|
246
|
+
/** @private The 2026-07-28 era has no server→client request channel. */
|
|
247
|
+
_modernEra(server) {
|
|
248
|
+
try {
|
|
249
|
+
const negotiated = server?.getNegotiatedProtocolVersion?.();
|
|
250
|
+
return typeof negotiated === 'string' && negotiated >= '2026-07-28';
|
|
251
|
+
} catch {
|
|
252
|
+
return false;
|
|
253
|
+
}
|
|
254
|
+
}
|
|
174
255
|
}
|
|
@@ -69,3 +69,53 @@ export function setActualCost(n) {
|
|
|
69
69
|
export function reportedActualCost() {
|
|
70
70
|
return requestContext.getStore()?.actualCost ?? null;
|
|
71
71
|
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Record the McpServer instance that is actually serving this request, and the
|
|
75
|
+
* wire era it speaks.
|
|
76
|
+
*
|
|
77
|
+
* Neither HTTP leg serves from the top-level McpServer that server.js
|
|
78
|
+
* registers everything on: the 2025-era path connects one clone per session,
|
|
79
|
+
* and the modern leg builds a fresh clone per request (see
|
|
80
|
+
* transports/streamableHttp.js). Only a clone is ever `.connect()`ed, so only a
|
|
81
|
+
* clone has a negotiated protocol version, the client's declared capabilities,
|
|
82
|
+
* and a channel to send a server-to-client request on. The template has none of
|
|
83
|
+
* those, which is why anything reading them off it (ElicitationHelper) got
|
|
84
|
+
* `undefined` on every HTTP request.
|
|
85
|
+
*
|
|
86
|
+
* Stdio stamps nothing: there the top-level instance IS the connected one, and
|
|
87
|
+
* the accessors below return null so callers fall back to it.
|
|
88
|
+
*
|
|
89
|
+
* @param {object|null} server the serving McpServer
|
|
90
|
+
* @param {'legacy'|'modern'|null} [era] the wire era it serves
|
|
91
|
+
*/
|
|
92
|
+
export function setServingServer(server, era = null) {
|
|
93
|
+
const store = requestContext.getStore();
|
|
94
|
+
if (!store) return;
|
|
95
|
+
store.servingServer = server ?? null;
|
|
96
|
+
store.servingEra = era;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The McpServer serving this request, or null on stdio / outside a context. */
|
|
100
|
+
export function servingServer() {
|
|
101
|
+
return requestContext.getStore()?.servingServer ?? null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** The wire era serving this request: 'legacy' | 'modern' | null (stdio). */
|
|
105
|
+
export function servingEra() {
|
|
106
|
+
return requestContext.getStore()?.servingEra ?? null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The JSON-RPC id of the request being served, or null when unknown.
|
|
111
|
+
*
|
|
112
|
+
* A server-to-client request sent from inside a tool has to say which inbound
|
|
113
|
+
* request it belongs to: the 2025-era streamable HTTP transport routes an
|
|
114
|
+
* unrelated request to the standalone GET SSE stream and silently DROPS it when
|
|
115
|
+
* the client never opened one, which turns an elicitation prompt into a
|
|
116
|
+
* 60-second stall before it fails open. withAuth stamps this from the SDK's
|
|
117
|
+
* per-request `ctx`; stdio has no streams to pick between and ignores it.
|
|
118
|
+
*/
|
|
119
|
+
export function servingRequestId() {
|
|
120
|
+
return requestContext.getStore()?.servingRequestId ?? null;
|
|
121
|
+
}
|
|
@@ -31,7 +31,7 @@ import { NodeStreamableHTTPServerTransport, toNodeHandler, toWebRequest } from "
|
|
|
31
31
|
import { createServer } from 'node:http';
|
|
32
32
|
import { createHash, randomUUID, timingSafeEqual } from 'node:crypto';
|
|
33
33
|
import { readFileSync } from 'node:fs';
|
|
34
|
-
import { requestContext } from '../requestContext.js';
|
|
34
|
+
import { requestContext, setServingServer } from '../requestContext.js';
|
|
35
35
|
import { applySpecHygiene } from '../specHygiene.js';
|
|
36
36
|
|
|
37
37
|
const pkg = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
|
|
@@ -214,7 +214,14 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
214
214
|
// this handler, so the modern leg never has to serve one. The SDK owns the
|
|
215
215
|
// Content-Type gate (415), the Mcp-Method/Mcp-Name cross-checks (-32020 on
|
|
216
216
|
// 400) and `server/discover`; nothing here re-implements them.
|
|
217
|
-
|
|
217
|
+
// The factory runs once per request, inside the requestContext.run() below,
|
|
218
|
+
// so the clone it builds can be stamped on the store: that clone — never the
|
|
219
|
+
// template — is the instance this request is actually served from.
|
|
220
|
+
const modernHandler = createMcpHandler((ctx) => {
|
|
221
|
+
const requestServer = cloneServerForSession(server);
|
|
222
|
+
setServingServer(requestServer, ctx?.era ?? 'modern');
|
|
223
|
+
return requestServer;
|
|
224
|
+
}, {
|
|
218
225
|
legacy: 'reject',
|
|
219
226
|
onerror: (err) => logger.warn('2026-era MCP request rejected', { error: err?.message })
|
|
220
227
|
});
|
|
@@ -362,7 +369,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
362
369
|
const existing = sessionIdHeader ? sessions.get(String(sessionIdHeader)) : undefined;
|
|
363
370
|
|
|
364
371
|
if (existing) {
|
|
365
|
-
await requestContext.run(
|
|
372
|
+
await requestContext.run(
|
|
373
|
+
{ internal, servingServer: existing.server, servingEra: 'legacy' },
|
|
374
|
+
() => existing.transport.handleRequest(req, res, parsedBody)
|
|
375
|
+
);
|
|
366
376
|
return;
|
|
367
377
|
}
|
|
368
378
|
|
|
@@ -395,7 +405,10 @@ export async function connectStreamableHttp(server, authManager, logger, options
|
|
|
395
405
|
|
|
396
406
|
try {
|
|
397
407
|
await sessionServer.connect(transport);
|
|
398
|
-
await requestContext.run(
|
|
408
|
+
await requestContext.run(
|
|
409
|
+
{ internal, servingServer: sessionServer, servingEra: 'legacy' },
|
|
410
|
+
() => transport.handleRequest(req, res, parsedBody)
|
|
411
|
+
);
|
|
399
412
|
} catch (err) {
|
|
400
413
|
logger.error('Streamable HTTP session initialization failed', { error: err?.message });
|
|
401
414
|
safeClose(transport);
|
package/src/server/withAuth.js
CHANGED
|
@@ -8,7 +8,11 @@
|
|
|
8
8
|
* so a valid API key is required for every invocation
|
|
9
9
|
* - try/finally guarantees a single `tool invocation` log line per call
|
|
10
10
|
* - log payload: { toolName, paramHash, durationMs, outcome, creditCost, creatorMode }
|
|
11
|
-
* - outcome ∈ { 'success' | 'error' | 'insufficient_credits' }
|
|
11
|
+
* - outcome ∈ { 'success' | 'error' | 'insufficient_credits' | 'input_required' }
|
|
12
|
+
* - an `input_required` return (Phase 4.4) is a round trip, not an answer: the
|
|
13
|
+
* handler did no work, so it is billed NOTHING and reports no usage. The
|
|
14
|
+
* SDK re-enters the handler with the reply and the terminal entry bills
|
|
15
|
+
* once, so a confirmation costs exactly what the call always cost (G4).
|
|
12
16
|
* - error results get a "Next step:" hint naming the tool to try next
|
|
13
17
|
* (src/server/fallbackHints.js) so a failure is not followed by a blind retry
|
|
14
18
|
* - emits an OTel span via src/observability/tracing.js (no-op if disabled)
|
|
@@ -16,6 +20,7 @@
|
|
|
16
20
|
*/
|
|
17
21
|
|
|
18
22
|
import { createHash } from 'node:crypto';
|
|
23
|
+
import { isInputRequiredResult } from '@modelcontextprotocol/server';
|
|
19
24
|
import { recordToolInvocation } from '../observability/tracing.js';
|
|
20
25
|
import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
|
|
21
26
|
import { appendFallbackHint } from './fallbackHints.js';
|
|
@@ -71,7 +76,7 @@ export function hashParams(params) {
|
|
|
71
76
|
*/
|
|
72
77
|
export function makeWithAuth({ authManager, logger, metrics = null, mcpServer = null }) {
|
|
73
78
|
return function withAuth(toolName, handler) {
|
|
74
|
-
const invoke = async (params) => {
|
|
79
|
+
const invoke = async (params, ctx) => {
|
|
75
80
|
const startTime = Date.now();
|
|
76
81
|
const paramHash = hashParams(params);
|
|
77
82
|
const creatorMode = authManager.isCreatorMode();
|
|
@@ -109,7 +114,14 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
109
114
|
// end user's credits before forwarding — checking the static key's
|
|
110
115
|
// balance here would gate users on an unrelated account).
|
|
111
116
|
if (!billingExempt) {
|
|
112
|
-
const hasCredits = await authManager.checkCredits(creditCost);
|
|
117
|
+
const hasCredits = await authManager.checkCredits(creditCost, ctx);
|
|
118
|
+
// The low-credit warning asks as a round trip (Phase 4.4). Nothing has
|
|
119
|
+
// run, so this costs nothing and reports no usage; the SDK re-enters
|
|
120
|
+
// with the answer.
|
|
121
|
+
if (isInputRequiredResult(hasCredits)) {
|
|
122
|
+
outcome = 'input_required';
|
|
123
|
+
return hasCredits;
|
|
124
|
+
}
|
|
113
125
|
if (!hasCredits) {
|
|
114
126
|
outcome = 'insufficient_credits';
|
|
115
127
|
return {
|
|
@@ -127,7 +139,21 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
127
139
|
}
|
|
128
140
|
|
|
129
141
|
handlerStarted = true;
|
|
130
|
-
const result = await handler(params);
|
|
142
|
+
const result = await handler(params, ctx);
|
|
143
|
+
|
|
144
|
+
// Phase 4.4: a multi-round-trip handler answers `input_required` when it
|
|
145
|
+
// needs the user before it can start. Nothing was fetched, so nothing is
|
|
146
|
+
// owed: no charge, no usage report, and none of the result stages below
|
|
147
|
+
// (there is no result yet to redact, shape or price). The SDK gathers the
|
|
148
|
+
// answer and re-enters this same wrapper; whichever entry finally returns
|
|
149
|
+
// a real result is the one that bills, exactly once. Without this branch
|
|
150
|
+
// an `input_required` is not `isError`, so it books as a success and
|
|
151
|
+
// bills in full on every round — up to eight — for a call that did no
|
|
152
|
+
// work, and a declined confirmation bills too (G4).
|
|
153
|
+
if (isInputRequiredResult(result)) {
|
|
154
|
+
outcome = 'input_required';
|
|
155
|
+
return result;
|
|
156
|
+
}
|
|
131
157
|
|
|
132
158
|
// Tools catch their own failures and return { isError:true } rather than
|
|
133
159
|
// throwing (the shared pattern in server.js). That is still an ERROR
|
|
@@ -275,11 +301,23 @@ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer =
|
|
|
275
301
|
|
|
276
302
|
// Every invocation runs in its own context so the compliance gate can stamp
|
|
277
303
|
// a refusal where the billing decision can see it. Any outer store (the
|
|
278
|
-
// HTTP transport's `internal` flag) is spread in,
|
|
279
|
-
// callers, who have no transport-provided store,
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
304
|
+
// HTTP transport's `internal` flag, the serving McpServer) is spread in,
|
|
305
|
+
// not replaced — and stdio callers, who have no transport-provided store,
|
|
306
|
+
// get one here.
|
|
307
|
+
//
|
|
308
|
+
// `ctx` is the SDK's per-request context (v2 calls a tool callback with
|
|
309
|
+
// `(args, ctx)`). It is passed straight through to the handler; existing
|
|
310
|
+
// 1-arity handlers ignore it. Its request id is stamped on the context so a
|
|
311
|
+
// server-to-client request sent from inside the tool can ride the same
|
|
312
|
+
// stream as this call — see servingRequestId() in requestContext.js.
|
|
313
|
+
return async (params, ctx) => requestContext.run(
|
|
314
|
+
{
|
|
315
|
+
...(requestContext.getStore() ?? {}),
|
|
316
|
+
preflightRefusal: null,
|
|
317
|
+
actualCost: null,
|
|
318
|
+
servingRequestId: ctx?.mcpReq?.id
|
|
319
|
+
},
|
|
320
|
+
() => invoke(params, ctx)
|
|
283
321
|
);
|
|
284
322
|
};
|
|
285
323
|
}
|
|
@@ -68,7 +68,14 @@ CLI: `crawlforge track https://example.com --selector ".price" --threshold 1`.
|
|
|
68
68
|
|
|
69
69
|
## Scheduled monitoring & notifications
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
`create_scheduled_monitor` repeats `compare` on a schedule and notifies on
|
|
72
|
+
change. It comes in two kinds.
|
|
73
|
+
|
|
74
|
+
**Local** (default): persisted in `~/.crawlforge/monitors/`; fires in-process
|
|
75
|
+
only while this MCP server runs (missed runs catch up on restart; `crawlforge
|
|
76
|
+
monitor:run-due` from system cron guarantees firing). Notifies by webhook or
|
|
77
|
+
Slack — never email. `goal` (plain-English LLM judge) and
|
|
78
|
+
`notificationThreshold` apply to local monitors only.
|
|
72
79
|
|
|
73
80
|
```json
|
|
74
81
|
{
|
|
@@ -76,19 +83,47 @@ Run continuous monitoring with webhooks instead of polling manually:
|
|
|
76
83
|
"params": {
|
|
77
84
|
"url": "https://example.com/pricing",
|
|
78
85
|
"operation": "create_scheduled_monitor",
|
|
79
|
-
"
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
86
|
+
"scheduledMonitorOptions": { "interval": 1800000, "notificationThreshold": "moderate" },
|
|
87
|
+
"notificationOptions": { "webhook": { "enabled": true, "url": "https://my-site.com/notify" } }
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
**Hosted** (`scheduledMonitorOptions.hosted: true`): registered with
|
|
93
|
+
CrawlForge's hosted monitors API under your API key; nothing is created locally
|
|
94
|
+
and this process never fetches the page. CrawlForge's own scheduler runs every
|
|
95
|
+
check whether or not this process is alive, records it, and notifies by email
|
|
96
|
+
and signed webhook on every changed, new, blocked or errored page. Passing
|
|
97
|
+
`goal` or `notificationThreshold` adds a `warnings` entry.
|
|
98
|
+
|
|
99
|
+
```json
|
|
100
|
+
{
|
|
101
|
+
"tool": "track_changes",
|
|
102
|
+
"params": {
|
|
103
|
+
"url": "https://example.com/pricing",
|
|
104
|
+
"operation": "create_scheduled_monitor",
|
|
105
|
+
"trackingOptions": { "customSelectors": [".price"] },
|
|
106
|
+
"scheduledMonitorOptions": { "hosted": true, "interval": 21600000 },
|
|
107
|
+
"notificationOptions": {
|
|
108
|
+
"email": { "enabled": true, "recipients": ["you@example.com"] },
|
|
109
|
+
"webhook": { "enabled": true, "url": "https://my-site.com/notify" }
|
|
85
110
|
}
|
|
86
111
|
}
|
|
87
112
|
}
|
|
88
113
|
```
|
|
89
114
|
|
|
90
|
-
|
|
91
|
-
`
|
|
115
|
+
Hosted mapping: each `customSelectors` entry becomes a target selector on the
|
|
116
|
+
URL; `schedule` (cron) passes through, otherwise `interval` becomes a cron of
|
|
117
|
+
5–60 minutes dividing 60, whole hours dividing 24, or daily — other values
|
|
118
|
+
round to the nearest and `warnings` says what they became. Up to 5 email
|
|
119
|
+
recipients. `webhook.signingSecret` (16–128 chars) becomes the webhook secret;
|
|
120
|
+
omit it and the response returns a generated `webhookSecret`. The response's
|
|
121
|
+
`monitor` carries the hosted `id`, `nextRunAt`, `estimatedCreditsPerMonth` and
|
|
122
|
+
a `dashboardUrl` for managing it.
|
|
123
|
+
|
|
124
|
+
CLI: `crawlforge monitor:create <url> --every 1800 --webhook <url>` (local) or
|
|
125
|
+
`crawlforge monitor:create <url> --hosted --email you@example.com` (hosted);
|
|
126
|
+
`monitor:list` shows both kinds and `monitor:stop <id>` removes either.
|
|
92
127
|
|
|
93
128
|
## Other operations
|
|
94
129
|
|
|
@@ -99,10 +134,12 @@ CLI (runs until Ctrl+C):
|
|
|
99
134
|
| `monitor` | One monitoring pass. |
|
|
100
135
|
| `get_history` | Retrieve past change records (`queryOptions`). |
|
|
101
136
|
| `get_stats` | Summary statistics for a tracked URL. |
|
|
102
|
-
| `create_scheduled_monitor`
|
|
137
|
+
| `create_scheduled_monitor` | Recurring `compare` + notify; local by default, `scheduledMonitorOptions.hosted: true` for a CrawlForge-run monitor (see above). |
|
|
138
|
+
| `list_scheduled_monitors` | Local monitors (`hosted: false`) then hosted ones (`hosted: true`), with `localCount`/`hostedCount`; `hostedError` if the website is unreachable. |
|
|
139
|
+
| `stop_scheduled_monitor` | By `scheduledMonitorOptions.monitorId`: stops a local monitor, or deletes the hosted one with that id. By `url` alone: stops every local monitor on it and deletes hosted monitors whose targets are all that exact URL. |
|
|
103
140
|
| `get_dashboard` | Aggregate status, recent alerts, trends. |
|
|
104
141
|
| `export_history` | Export change history as `json` or `csv`. |
|
|
105
|
-
| `create_alert_rule` | Conditional alerts (webhook / email
|
|
142
|
+
| `create_alert_rule` | Conditional alerts fired from `compare` (webhook / slack; the email action is not sent by a local process — use a hosted monitor for email). |
|
|
106
143
|
| `generate_trend_report` | Trend analysis over time. |
|
|
107
144
|
| `get_monitoring_templates` | List built-in monitoring presets. |
|
|
108
145
|
|
|
@@ -111,6 +148,8 @@ against the baseline without re-fetching.
|
|
|
111
148
|
|
|
112
149
|
## Cost note
|
|
113
150
|
|
|
114
|
-
`track_changes` = 3 credits per call
|
|
115
|
-
|
|
116
|
-
|
|
151
|
+
`track_changes` = 3 credits per call, except that creating a hosted monitor or
|
|
152
|
+
stopping a hosted-only one charges 0 (3 is the projected ceiling). A typical
|
|
153
|
+
watch is one `create_baseline` plus periodic `compare` calls, or one scheduled
|
|
154
|
+
monitor. Each hosted check bills 3 credits per compared target to your account;
|
|
155
|
+
blocked and errored targets are free.
|