crawlforge-mcp-server 5.9.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -6
- package/server.js +232 -261
- package/src/core/ActionExecutor.js +1 -1
- package/src/core/AuthManager.js +29 -2
- package/src/core/ChangeTracker.js +1 -1
- package/src/core/ElicitationHelper.js +83 -34
- package/src/core/SamplingClient.js +8 -2
- package/src/core/analysis/ContentAnalyzer.js +1 -1
- package/src/core/processing/BrowserProcessor.js +1 -1
- package/src/core/processing/ContentProcessor.js +1 -1
- package/src/core/processing/PDFProcessor.js +2 -2
- package/src/schemas/toolOutputSchemas.js +25 -0
- package/src/server/inlineThreshold.js +5 -0
- package/src/server/redaction.js +469 -0
- package/src/server/registerTool.js +1 -1
- package/src/server/specHygiene.js +17 -22
- package/src/server/transports/stdio.js +2 -3
- package/src/server/transports/streamableHttp.js +128 -66
- package/src/server/withAuth.js +42 -2
- package/src/tools/crawl/crawlDeep.js +4 -4
- package/src/tools/extract/analyzeContent.js +1 -1
- package/src/tools/extract/extractContent.js +1 -1
- package/src/tools/extract/processDocument.js +1 -1
- package/src/tools/extract/summarizeContent.js +1 -1
- package/src/tools/llmstxt/generateLLMsTxt.js +2 -2
- package/src/tools/research/deepResearch.js +1 -1
- package/src/tools/search/batchSearch.js +50 -0
- package/src/tools/search/searchWeb.js +44 -0
- package/src/tools/tracking/trackChanges/schema.js +4 -4
- package/src/utils/HumanBehaviorSimulator.js +7 -7
- package/src/utils/secretMask.js +11 -7
- package/src/server/taskSupport.js +0 -233
- package/src/server/transports/http.js +0 -22
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* redaction — the one declaration (G5) of `redact_pii`: the parameter, the
|
|
3
|
+
* entity and style enums, the per-tool text paths, the credit rule and the
|
|
4
|
+
* stage that applies it (Phase 5, 5.3).
|
|
5
|
+
*
|
|
6
|
+
* Nothing in this server scrubbed personal data out of the page text it hands
|
|
7
|
+
* back. `secretMask.js` covered API keys and tokens in our own LOG lines and
|
|
8
|
+
* nothing else, so a support inbox, a staff phone list or a checkout form's
|
|
9
|
+
* card number travelled straight into the caller's context window. `redact_pii`
|
|
10
|
+
* closes that: opt-in, off by default, and priced at nothing for the regex pass.
|
|
11
|
+
*
|
|
12
|
+
* Applied by withAuth after a successful handler run and BEFORE the
|
|
13
|
+
* inline-threshold stage stores the result. That order is the whole point: a
|
|
14
|
+
* result stored unredacted would be served back verbatim by `read_result`,
|
|
15
|
+
* which is exactly the hole this exists to close.
|
|
16
|
+
*
|
|
17
|
+
* Two passes:
|
|
18
|
+
*
|
|
19
|
+
* fast (default, 0 credits) — `redactPii` from crawlforge-extractors: pure
|
|
20
|
+
* regex over EMAIL, PHONE, FINANCIAL and SECRET, deterministic, no network.
|
|
21
|
+
* model (opt-in, +3 credits once per call) — an Ollama-first NER pass for
|
|
22
|
+
* PERSON and LOCATION, which no regex can decide. The regex pass ALWAYS
|
|
23
|
+
* runs first and the model only adds to it. G3: the model identifies
|
|
24
|
+
* spans, this module replaces them, and a span the model invented — one
|
|
25
|
+
* that is not present verbatim in the input — is discarded. The text it
|
|
26
|
+
* reads is fenced by `fenceUntrusted` and is already regex-redacted, so
|
|
27
|
+
* the fence never wraps unredacted text.
|
|
28
|
+
*
|
|
29
|
+
* An `entities` name the chosen mode cannot serve is a validation error, not
|
|
30
|
+
* a silent omission — see `entityNameIssue`. Asking for a class and being
|
|
31
|
+
* handed a page that still contains it is the failure this feature exists to
|
|
32
|
+
* prevent, so the schema refuses before anything is fetched, at no charge.
|
|
33
|
+
*
|
|
34
|
+
* The model pass is bounded by wall clock and by cost, not by taste: it runs
|
|
35
|
+
* once per TEXT, not once per call, so a 50-URL batch_scrape left unbounded
|
|
36
|
+
* would outlive any client's patience and — where the SamplingClient chain
|
|
37
|
+
* reaches a server-side API key rather than local Ollama — spend real money
|
|
38
|
+
* against a single +3 surcharge. Three caps, so the exposure of one call is a
|
|
39
|
+
* fixed number: at most MODEL_TEXT_CHARS of any one text, MODEL_BUDGET_CHARS
|
|
40
|
+
* across the call, and MODEL_MAX_PASSES completions. A warning says when any
|
|
41
|
+
* of them bit.
|
|
42
|
+
*
|
|
43
|
+
* Known limit, stated rather than hidden: `scrape`'s own query-scoped formats
|
|
44
|
+
* (`highlights`/`question` with `mode: "model"`, Phase 1) run their model step
|
|
45
|
+
* INSIDE the handler, before this stage, so those prompts see the page text as
|
|
46
|
+
* fetched. Everything that leaves the server is redacted; that one internal
|
|
47
|
+
* prompt is not. Closing it means redacting inside unifiedScrape as well,
|
|
48
|
+
* which would put the application logic in two places.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import { z } from 'zod';
|
|
52
|
+
import {
|
|
53
|
+
redactPii,
|
|
54
|
+
REGEX_ENTITIES,
|
|
55
|
+
MODEL_ONLY_ENTITIES,
|
|
56
|
+
DEFAULT_REPLACE_STYLE
|
|
57
|
+
} from 'crawlforge-extractors';
|
|
58
|
+
import { fenceUntrusted } from '../utils/untrustedContent.js';
|
|
59
|
+
|
|
60
|
+
/** What the model-backed PERSON/LOCATION pass adds to a call's price. */
|
|
61
|
+
export const REDACT_PII_MODEL_CREDITS = 3;
|
|
62
|
+
|
|
63
|
+
/** How much of any one text the model pass reads (summarizeContent's cap). */
|
|
64
|
+
const MODEL_TEXT_CHARS = 12000;
|
|
65
|
+
|
|
66
|
+
/** How much text the model pass reads across a whole call. */
|
|
67
|
+
const MODEL_BUDGET_CHARS = 60000;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* How many completions the model pass may make in one call. Derived, not
|
|
71
|
+
* chosen: it is what the character budget already allows at full size, so a
|
|
72
|
+
* result made of many SHORT texts (a 50-URL batch_scrape) cannot turn one
|
|
73
|
+
* +3 surcharge into fifty round-trips.
|
|
74
|
+
*/
|
|
75
|
+
const MODEL_MAX_PASSES = MODEL_BUDGET_CHARS / MODEL_TEXT_CHARS;
|
|
76
|
+
|
|
77
|
+
/** Every entity name the public parameter accepts. */
|
|
78
|
+
export const REDACT_PII_ENTITIES = Object.freeze([...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES]);
|
|
79
|
+
|
|
80
|
+
/** The tag names this module writes, so a model can never re-redact one. */
|
|
81
|
+
const TAG_NAMES = new Set([...REDACT_PII_ENTITIES, 'REDACTED']);
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Why an entity name this mode cannot serve is REJECTED rather than dropped.
|
|
85
|
+
*
|
|
86
|
+
* `redactPii` drops a name it does not handle in silence, and it is right to:
|
|
87
|
+
* a library has no channel to tell a caller anything. A tool does. Under a
|
|
88
|
+
* lenient rule `entities: ["EMAIL", "PERSON"]` with the default `fast` mode
|
|
89
|
+
* returns a page with every personal name still in it, plus a redaction
|
|
90
|
+
* report listing only the emails — a partial version of the exact failure
|
|
91
|
+
* this feature exists to prevent, with no signal that half the request was
|
|
92
|
+
* ignored. So the schema refuses it, before anything is fetched and at no
|
|
93
|
+
* charge, and says which of the two things went wrong.
|
|
94
|
+
*
|
|
95
|
+
* @param {string} name as the caller wrote it
|
|
96
|
+
* @param {unknown} mode the call's `mode`
|
|
97
|
+
* @returns {string|null} the validation message, or null when the name is fine
|
|
98
|
+
*/
|
|
99
|
+
function entityNameIssue(name, mode) {
|
|
100
|
+
const upper = typeof name === 'string' ? name.toUpperCase() : '';
|
|
101
|
+
if (REGEX_ENTITIES.includes(upper)) return null;
|
|
102
|
+
if (MODEL_ONLY_ENTITIES.includes(upper)) {
|
|
103
|
+
if (mode === 'model') return null;
|
|
104
|
+
return `${MODEL_ONLY_ENTITIES.join(' and ')} need mode: "model" — no regex can decide them. Add mode:"model" (+${REDACT_PII_MODEL_CREDITS} credits once per call) or drop "${name}"`;
|
|
105
|
+
}
|
|
106
|
+
return `unknown entity name "${name}" — expected one of ${REGEX_ENTITIES.join(', ')}, or ${MODEL_ONLY_ENTITIES.join('/')} with mode:"model"`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The object form of `redact_pii`; names are matched case-insensitively. */
|
|
110
|
+
const RedactPiiOptionsSchema = z.object({
|
|
111
|
+
entities: z.array(z.string()).optional().describe(`Which classes to redact, case-insensitive: ${REGEX_ENTITIES.join(', ')}, plus ${MODEL_ONLY_ENTITIES.join(' and ')} when mode is "model". Omitted or empty means all four regex classes (and both model classes in "model" mode). An unknown name, or a model-only name without mode:"model", is rejected`),
|
|
112
|
+
replace_style: z.enum(['tag', 'mask', 'remove']).optional().describe('"tag" (default) writes <EMAIL>, "mask" writes [REDACTED], "remove" deletes the value'),
|
|
113
|
+
mode: z.enum(['fast', 'model']).optional().describe('"fast" (default) is regex only and free; "model" adds an Ollama NER pass for PERSON and LOCATION (+3 credits once per call)')
|
|
114
|
+
}).superRefine((value, ctx) => {
|
|
115
|
+
if (!Array.isArray(value.entities)) return;
|
|
116
|
+
value.entities.forEach((name, index) => {
|
|
117
|
+
const message = entityNameIssue(name, value.mode);
|
|
118
|
+
if (message) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['entities', index], message });
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The public parameter, with the text the client sees. Spread into the input
|
|
124
|
+
* schema of each tool in REDACTION_TOOLS.
|
|
125
|
+
*/
|
|
126
|
+
export const REDACT_PII_PARAM = {
|
|
127
|
+
redact_pii: z.union([
|
|
128
|
+
z.boolean(),
|
|
129
|
+
RedactPiiOptionsSchema
|
|
130
|
+
]).optional().describe('Redact personal data from the text this call returns, before it reaches your context window. true means the free regex pass over EMAIL, PHONE, FINANCIAL and SECRET. The result carries redaction:{entities,count}. Default: off')
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Where each tool keeps the text a redaction must cover. `[]` walks an array.
|
|
135
|
+
* Every path listed is redacted, not just the first one that matches — a
|
|
136
|
+
* single unredacted copy (extract_content's `readability.textContent`, say)
|
|
137
|
+
* would defeat the whole feature.
|
|
138
|
+
*/
|
|
139
|
+
export const REDACTION_TOOLS = Object.freeze({
|
|
140
|
+
scrape: [
|
|
141
|
+
'content.markdown', 'content.text', 'content.html', 'content.rawHtml',
|
|
142
|
+
'content.highlights[].text', 'content.answer.text', 'content.answer.evidence[].text'
|
|
143
|
+
],
|
|
144
|
+
extract_content: [
|
|
145
|
+
'content.text', 'content.html', 'content.markdown', 'content.cleanedHTML',
|
|
146
|
+
'readability.content', 'readability.textContent', 'readability.excerpt'
|
|
147
|
+
],
|
|
148
|
+
extract_text: ['text', 'markdown'],
|
|
149
|
+
batch_scrape: ['results[].content.text', 'results[].content.html', 'results[].content.markdown'],
|
|
150
|
+
crawl_deep: ['results[].content'],
|
|
151
|
+
stealth_mode: ['content.text', 'content.html', 'content.markdown'],
|
|
152
|
+
scrape_with_actions: ['content.text', 'content.html', 'content.markdown'],
|
|
153
|
+
process_document: ['content.text', 'content.html', 'content.extractedContent'],
|
|
154
|
+
search_web: [
|
|
155
|
+
'results[].snippet', 'results[].htmlSnippet',
|
|
156
|
+
'results_by_query[].results[].snippet', 'results_by_query[].results[].htmlSnippet'
|
|
157
|
+
]
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Normalise the raw `redact_pii` param into what the two passes need, or null
|
|
162
|
+
* when redaction is off. Reads the RAW value — getToolCost runs before
|
|
163
|
+
* validation and this is the same declaration it prices from — so anything
|
|
164
|
+
* that is neither `true` nor a plain object means off.
|
|
165
|
+
*
|
|
166
|
+
* An absent or empty `entities` means everything the chosen mode can do. An
|
|
167
|
+
* explicit list is honoured exactly: names a regex can decide go to
|
|
168
|
+
* `redactPii`, and PERSON/LOCATION go to the model pass.
|
|
169
|
+
*
|
|
170
|
+
* This is the TOLERANT layer, deliberately. `RedactPiiOptionsSchema` has
|
|
171
|
+
* already rejected an unknown name, and a model-only name outside `model`
|
|
172
|
+
* mode, at no charge — but this function also runs on unvalidated params
|
|
173
|
+
* (getToolCost, and a REST proxy request that arrives pre-validated by the
|
|
174
|
+
* website), so it must resolve rather than throw. Where the strict schema
|
|
175
|
+
* would have refused, this resolves to a selection that redacts nothing by
|
|
176
|
+
* regex, and `applyRedaction` honours that rather than falling back to all.
|
|
177
|
+
*
|
|
178
|
+
* @param {unknown} raw
|
|
179
|
+
* @returns {{ entities: string[], modelEntities: string[], replaceStyle: string, mode: 'fast'|'model' } | null}
|
|
180
|
+
*/
|
|
181
|
+
export function resolveRedaction(raw) {
|
|
182
|
+
if (raw !== true && (raw === null || typeof raw !== 'object' || Array.isArray(raw))) return null;
|
|
183
|
+
const options = raw === true ? {} : raw;
|
|
184
|
+
|
|
185
|
+
const mode = options.mode === 'model' ? 'model' : 'fast';
|
|
186
|
+
const replaceStyle = ['tag', 'mask', 'remove'].includes(options.replace_style)
|
|
187
|
+
? options.replace_style
|
|
188
|
+
: DEFAULT_REPLACE_STYLE;
|
|
189
|
+
|
|
190
|
+
const asked = Array.isArray(options.entities) && options.entities.length > 0
|
|
191
|
+
? options.entities.filter((name) => typeof name === 'string').map((name) => name.toUpperCase())
|
|
192
|
+
: [...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES];
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
entities: REGEX_ENTITIES.filter((name) => asked.includes(name)),
|
|
196
|
+
modelEntities: mode === 'model' ? MODEL_ONLY_ENTITIES.filter((name) => asked.includes(name)) : [],
|
|
197
|
+
replaceStyle,
|
|
198
|
+
mode
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* What redaction adds to a tool's price: nothing for the regex pass, and the
|
|
204
|
+
* model pass's own 3 once per call. Charged once rather than per page for the
|
|
205
|
+
* same reason `batch_scrape` and `crawl_deep` are priced flat — the page count
|
|
206
|
+
* is not knowable before the call, and a projection scaled by `max_pages`
|
|
207
|
+
* would have the credit check reject an ordinary crawl. Projected is the
|
|
208
|
+
* ceiling (G4); withAuth drops it when no model completed.
|
|
209
|
+
*
|
|
210
|
+
* @param {string} toolName
|
|
211
|
+
* @param {object} [params] the call's raw params
|
|
212
|
+
* @returns {0 | 3}
|
|
213
|
+
*/
|
|
214
|
+
export function redactionSurcharge(toolName, params) {
|
|
215
|
+
if (!(toolName in REDACTION_TOOLS)) return 0;
|
|
216
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
217
|
+
return resolved && resolved.modelEntities.length > 0 ? REDACT_PII_MODEL_CREDITS : 0;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── Text routing ─────────────────────────────────────────────────────────────
|
|
221
|
+
|
|
222
|
+
/** "results[].content.text" -> ["results", "[]", "content", "text"]. */
|
|
223
|
+
function segmentsOf(path) {
|
|
224
|
+
const segments = [];
|
|
225
|
+
for (const part of path.split('.')) {
|
|
226
|
+
if (part.endsWith('[]')) {
|
|
227
|
+
segments.push(part.slice(0, -2), '[]');
|
|
228
|
+
} else {
|
|
229
|
+
segments.push(part);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return segments;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Every string the path reaches, each with the setter that writes it back. */
|
|
236
|
+
function collectAt(node, segments, found) {
|
|
237
|
+
if (node === null || typeof node !== 'object') return;
|
|
238
|
+
const [head, ...rest] = segments;
|
|
239
|
+
|
|
240
|
+
if (head === '[]') {
|
|
241
|
+
if (!Array.isArray(node)) return;
|
|
242
|
+
for (let i = 0; i < node.length; i++) {
|
|
243
|
+
if (rest.length === 0) {
|
|
244
|
+
if (typeof node[i] === 'string') found.push({ text: node[i], set: (v) => { node[i] = v; } });
|
|
245
|
+
} else {
|
|
246
|
+
collectAt(node[i], rest, found);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (Array.isArray(node)) return;
|
|
253
|
+
if (rest.length === 0) {
|
|
254
|
+
if (typeof node[head] === 'string') found.push({ text: node[head], set: (v) => { node[head] = v; } });
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
collectAt(node[head], rest, found);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* @param {object} resultObject
|
|
262
|
+
* @param {string[]} paths
|
|
263
|
+
* @returns {Array<{ text: string, set: (value: string) => void }>}
|
|
264
|
+
*/
|
|
265
|
+
export function collectTexts(resultObject, paths) {
|
|
266
|
+
const found = [];
|
|
267
|
+
for (const path of paths) collectAt(resultObject, segmentsOf(path), found);
|
|
268
|
+
return found;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ── The model pass ───────────────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Parse the model's reply into the spans it claims. A line the model wrote
|
|
275
|
+
* that is not present verbatim in the text is dropped: the model identifies,
|
|
276
|
+
* it never rewrites (G3).
|
|
277
|
+
*
|
|
278
|
+
* @param {string} reply
|
|
279
|
+
* @param {string} text the text the model read
|
|
280
|
+
* @param {string[]} wanted the entity names asked for
|
|
281
|
+
* @returns {Array<{ start: number, end: number, entity: string }>}
|
|
282
|
+
*/
|
|
283
|
+
export function spansFromReply(reply, text, wanted) {
|
|
284
|
+
if (typeof reply !== 'string') return [];
|
|
285
|
+
const spans = [];
|
|
286
|
+
for (const line of reply.split('\n')) {
|
|
287
|
+
const match = /^\s*(?:[-*]\s*)?(PERSON|LOCATION)\s*[:\-]\s*(.+?)\s*$/i.exec(line);
|
|
288
|
+
if (!match) continue;
|
|
289
|
+
const entity = match[1].toUpperCase();
|
|
290
|
+
if (!wanted.includes(entity)) continue;
|
|
291
|
+
const value = match[2].replace(/^["'`]|["'`]$/g, '').trim();
|
|
292
|
+
if (value.length < 2 || TAG_NAMES.has(value.toUpperCase())) continue;
|
|
293
|
+
|
|
294
|
+
let from = 0;
|
|
295
|
+
for (;;) {
|
|
296
|
+
const at = text.indexOf(value, from);
|
|
297
|
+
if (at === -1) break;
|
|
298
|
+
spans.push({ start: at, end: at + value.length, entity });
|
|
299
|
+
from = at + value.length;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
// Longest first, so an overlap resolves in favour of the fuller name.
|
|
303
|
+
spans.sort((a, b) => (b.end - b.start) - (a.end - a.start) || a.start - b.start);
|
|
304
|
+
const accepted = [];
|
|
305
|
+
for (const span of spans) {
|
|
306
|
+
if (accepted.some((other) => span.start < other.end && other.start < span.end)) continue;
|
|
307
|
+
accepted.push(span);
|
|
308
|
+
}
|
|
309
|
+
return accepted.sort((a, b) => a.start - b.start);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function replacementFor(entity, style) {
|
|
313
|
+
if (style === 'mask') return '[REDACTED]';
|
|
314
|
+
if (style === 'remove') return '';
|
|
315
|
+
return `<${entity}>`;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Rebuild `text` with the accepted spans replaced; counts them by entity. */
|
|
319
|
+
function applySpans(text, spans, style, counts) {
|
|
320
|
+
let out = '';
|
|
321
|
+
let cursor = 0;
|
|
322
|
+
for (const span of spans) {
|
|
323
|
+
out += text.slice(cursor, span.start) + replacementFor(span.entity, style);
|
|
324
|
+
counts[span.entity] = (counts[span.entity] || 0) + 1;
|
|
325
|
+
cursor = span.end;
|
|
326
|
+
}
|
|
327
|
+
return out + text.slice(cursor);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const PROMPT_SYSTEM = 'You mark personal names and place names in text quoted from a web page. ' +
|
|
331
|
+
'Reply with one entry per line, each "PERSON: <text>" or "LOCATION: <text>", copying the text exactly as it appears. ' +
|
|
332
|
+
'Do not translate, shorten or correct it. If there are none, reply NONE. No other output.';
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* One completion for one text. Returns the reply, or throws — the caller
|
|
336
|
+
* treats a throw as "no LLM route", drops the surcharge and warns.
|
|
337
|
+
* @param {(prompt: string, options: object) => Promise<{ text: string }>} complete
|
|
338
|
+
*/
|
|
339
|
+
async function askModel(complete, text, wanted) {
|
|
340
|
+
const { text: reply } = await complete(
|
|
341
|
+
`${fenceUntrusted(text, 'page text')}\nList every ${wanted.join(' and ')} in the text above.`,
|
|
342
|
+
{ maxTokens: 512, systemPrompt: PROMPT_SYSTEM }
|
|
343
|
+
);
|
|
344
|
+
return reply;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ── The stage ────────────────────────────────────────────────────────────────
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Redact a tool result in place.
|
|
351
|
+
*
|
|
352
|
+
* @param {string} toolName
|
|
353
|
+
* @param {object} resultObject the parsed JSON result; mutated
|
|
354
|
+
* @param {object} params the call's params
|
|
355
|
+
* @param {{ complete?: (prompt: string, options: object) => Promise<{ text: string }> }} deps
|
|
356
|
+
* @returns {Promise<{ redacted: boolean, modelRan: boolean }>}
|
|
357
|
+
*/
|
|
358
|
+
export async function applyRedaction(toolName, resultObject, params, { complete } = {}) {
|
|
359
|
+
const paths = REDACTION_TOOLS[toolName];
|
|
360
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
361
|
+
if (!paths || !resolved) return { redacted: false, modelRan: false };
|
|
362
|
+
if (!resultObject || typeof resultObject !== 'object' || Array.isArray(resultObject)) {
|
|
363
|
+
return { redacted: false, modelRan: false };
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const targets = collectTexts(resultObject, paths);
|
|
367
|
+
const counts = {};
|
|
368
|
+
const warnings = [];
|
|
369
|
+
|
|
370
|
+
// Skip the regex pass entirely when the caller's selection left it nothing
|
|
371
|
+
// to do (`entities: ['PERSON']`). `redactPii` reads an EMPTY array as "all
|
|
372
|
+
// four", so passing one through would redact everything a caller who asked
|
|
373
|
+
// for one model-only class never asked to lose.
|
|
374
|
+
if (resolved.entities.length > 0) {
|
|
375
|
+
for (const target of targets) {
|
|
376
|
+
const { text, redaction } = redactPii(target.text, {
|
|
377
|
+
entities: resolved.entities,
|
|
378
|
+
replaceStyle: resolved.replaceStyle
|
|
379
|
+
});
|
|
380
|
+
for (const [entity, n] of Object.entries(redaction.entities)) counts[entity] = (counts[entity] || 0) + n;
|
|
381
|
+
// What the model pass reads, so the fence never wraps unredacted text.
|
|
382
|
+
target.text = text;
|
|
383
|
+
target.set(text);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
let modelRan = false;
|
|
388
|
+
if (resolved.modelEntities.length > 0 && typeof complete === 'function') {
|
|
389
|
+
let budget = MODEL_BUDGET_CHARS;
|
|
390
|
+
let capped = false;
|
|
391
|
+
let failure = null;
|
|
392
|
+
|
|
393
|
+
let passes = 0;
|
|
394
|
+
for (const target of targets) {
|
|
395
|
+
const live = target.text; // the regex-redacted copy
|
|
396
|
+
if (budget <= 0 || passes >= MODEL_MAX_PASSES) { capped = true; break; }
|
|
397
|
+
const slice = live.slice(0, Math.min(MODEL_TEXT_CHARS, budget));
|
|
398
|
+
if (slice.length === 0) continue;
|
|
399
|
+
if (slice.length < live.length) capped = true;
|
|
400
|
+
budget -= slice.length;
|
|
401
|
+
passes++;
|
|
402
|
+
|
|
403
|
+
let reply;
|
|
404
|
+
try {
|
|
405
|
+
reply = await askModel(complete, slice, resolved.modelEntities);
|
|
406
|
+
} catch (error) {
|
|
407
|
+
failure = error;
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
modelRan = true;
|
|
411
|
+
const spans = spansFromReply(reply, slice, resolved.modelEntities);
|
|
412
|
+
if (spans.length === 0) continue;
|
|
413
|
+
target.set(applySpans(slice, spans, resolved.replaceStyle, counts) + live.slice(slice.length));
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
if (failure && !modelRan) {
|
|
417
|
+
warnings.push(`redact_pii: no model answered, so ${resolved.modelEntities.join(' and ')} were not redacted (${failure.message}); the model surcharge was not charged`);
|
|
418
|
+
} else if (failure) {
|
|
419
|
+
warnings.push(`redact_pii: the model pass stopped early (${failure.message}); the text it did not reach carries the regex redactions only`);
|
|
420
|
+
}
|
|
421
|
+
if (capped) {
|
|
422
|
+
warnings.push(`redact_pii: the model pass read at most ${MODEL_TEXT_CHARS} characters per text, ${MODEL_BUDGET_CHARS} in total and ${MODEL_MAX_PASSES} texts; text beyond that carries the regex redactions only`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const count = Object.values(counts).reduce((sum, n) => sum + n, 0);
|
|
427
|
+
resultObject.redaction = {
|
|
428
|
+
entities: counts,
|
|
429
|
+
count,
|
|
430
|
+
mode: resolved.mode,
|
|
431
|
+
...(resolved.modelEntities.length > 0 ? { model_ran: modelRan } : {})
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// The query-scoped formats index the markdown by offset. Redaction changes
|
|
435
|
+
// its length, so say so rather than let the caller trust a stale offset.
|
|
436
|
+
if (count > 0 && resultObject.content && (resultObject.content.highlights || resultObject.content.answer)) {
|
|
437
|
+
warnings.push('redact_pii: highlight and evidence offsets index the text as it was before redaction');
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
if (warnings.length > 0) {
|
|
441
|
+
resultObject.warnings = [
|
|
442
|
+
...(Array.isArray(resultObject.warnings) ? resultObject.warnings : []),
|
|
443
|
+
...warnings
|
|
444
|
+
];
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
return { redacted: true, modelRan };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The redaction stage as withAuth uses it: a SamplingClient-backed `complete`
|
|
452
|
+
* built only when a model pass was actually asked for, so an ordinary call
|
|
453
|
+
* never loads the client.
|
|
454
|
+
*
|
|
455
|
+
* @param {string} toolName
|
|
456
|
+
* @param {object} resultObject
|
|
457
|
+
* @param {object} params
|
|
458
|
+
* @param {{ mcpServer?: object|null }} [deps]
|
|
459
|
+
*/
|
|
460
|
+
export async function runRedactionStage(toolName, resultObject, params, { mcpServer = null } = {}) {
|
|
461
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
462
|
+
const complete = resolved && resolved.modelEntities.length > 0
|
|
463
|
+
? async (prompt, options) => {
|
|
464
|
+
const { SamplingClient } = await import('../core/SamplingClient.js');
|
|
465
|
+
return new SamplingClient({ mcpServer }).complete(prompt, options);
|
|
466
|
+
}
|
|
467
|
+
: undefined;
|
|
468
|
+
return applyRedaction(toolName, resultObject, params, { complete });
|
|
469
|
+
}
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* validates `structuredContent` against the schema; legacy clients keep
|
|
11
11
|
* reading the JSON-stringified `content` for backward compatibility.
|
|
12
12
|
*
|
|
13
|
-
* @param {import('@modelcontextprotocol/
|
|
13
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
14
14
|
* @param {Function} withAuth — from makeWithAuth() in src/server/withAuth.js
|
|
15
15
|
* @param {Object} descriptor
|
|
16
16
|
* @param {string} descriptor.name — tool name (MCP identifier)
|
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* specHygiene — Phase 6 protocol-hygiene wrapper.
|
|
3
3
|
*
|
|
4
|
-
* Applies
|
|
4
|
+
* Applies four MCP wire-level upgrades to an already-registered McpServer
|
|
5
5
|
* without changing how tools/prompts are declared anywhere else:
|
|
6
6
|
*
|
|
7
7
|
* 1. JSON Schema 2020-12 dialect stamping on every tool's inputSchema /
|
|
8
|
-
* outputSchema
|
|
9
|
-
*
|
|
8
|
+
* outputSchema. Under the v2 SDK + zod 4 the conversion already emits
|
|
9
|
+
* 2020-12 with `$defs`, so the `definitions` -> `$defs` rewrite below is
|
|
10
|
+
* now defensive rather than load-bearing; it is kept because the stamping
|
|
11
|
+
* and ordering passes still walk the same tree (Phase 4.1 kept behaviour
|
|
12
|
+
* identical — removing the rewrite is a separate, verifiable change).
|
|
10
13
|
* 2. Deterministic (alphabetical) tool ordering in tools/list, so clients
|
|
11
14
|
* that prompt-cache tools/list get stable hits across restarts.
|
|
12
15
|
* 3. SEP-973 icons metadata on every tool/prompt lacking one.
|
|
13
16
|
* 4. SEP-2549-style cache hints on tools/call results for a fixed
|
|
14
17
|
* allowlist of read-only, idempotent tools (see note below).
|
|
15
18
|
*
|
|
16
|
-
* Wiring: SDK
|
|
19
|
+
* Wiring: the SDK has no plugin hook for this, so applySpecHygiene() must be
|
|
17
20
|
* called once, after all server.registerTool()/registerPrompt() calls and
|
|
18
21
|
* before transport.connect(). It reaches into `server.server` (the
|
|
19
22
|
* underlying Protocol), captures the ListTools/CallTool/ListPrompts handlers
|
|
@@ -22,26 +25,18 @@
|
|
|
22
25
|
* handler for the same method (confirmed from the SDK source: only
|
|
23
26
|
* `assertCanSetRequestHandler`, called by McpServer's own registration path,
|
|
24
27
|
* throws on a pre-existing handler; `setRequestHandler` itself does not).
|
|
28
|
+
* In v2 `setRequestHandler` takes a method string rather than a Zod schema.
|
|
25
29
|
*
|
|
26
|
-
* SEP-2549 note: as specified (2026-07-28
|
|
27
|
-
* ("public"|"private") are defined on
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* namespaced `_meta` key following the SDK's own
|
|
32
|
-
* `io.modelcontextprotocol/<name>` convention (see RELATED_TASK_META_KEY in
|
|
33
|
-
* @modelcontextprotocol/sdk/types.js):
|
|
30
|
+
* SEP-2549 note: as specified (2026-07-28), `ttlMs` / `cacheScope`
|
|
31
|
+
* ("public"|"private") are defined on server/discover results — there is no
|
|
32
|
+
* CacheableResult for tools/call. Since this deliverable asks for tools/call
|
|
33
|
+
* cache hints, the same two field names are carried into a namespaced `_meta`
|
|
34
|
+
* key following the SDK's own `io.modelcontextprotocol/<name>` convention:
|
|
34
35
|
* result._meta["io.modelcontextprotocol/cacheable"] = { ttlMs, cacheScope }
|
|
35
36
|
* This is a documented adaptation, not a key confirmed by the spec text for
|
|
36
37
|
* tools/call — revisit if/when a SEP defines call-result caching explicitly.
|
|
37
38
|
*/
|
|
38
39
|
|
|
39
|
-
import {
|
|
40
|
-
ListToolsRequestSchema,
|
|
41
|
-
CallToolRequestSchema,
|
|
42
|
-
ListPromptsRequestSchema
|
|
43
|
-
} from '@modelcontextprotocol/sdk/types.js';
|
|
44
|
-
|
|
45
40
|
const APPLIED = Symbol('crawlforge.specHygiene.applied');
|
|
46
41
|
|
|
47
42
|
const JSON_SCHEMA_2020_12 = 'https://json-schema.org/draft/2020-12/schema';
|
|
@@ -158,7 +153,7 @@ function wrapToolsCall(innerHandler, cacheableTools) {
|
|
|
158
153
|
* tools/prompts are registered and before transport.connect(). Idempotent:
|
|
159
154
|
* a second call on the same server is a no-op.
|
|
160
155
|
*
|
|
161
|
-
* @param {import('@modelcontextprotocol/
|
|
156
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
162
157
|
* @param {object} [overrides]
|
|
163
158
|
* @param {{src:string,mimeType?:string,sizes?:string[]}} [overrides.icon] — default icon injected into tools/prompts lacking one
|
|
164
159
|
* @param {Record<string,{ttlMs:number,cacheScope:'public'|'private'}>} [overrides.cacheableTools] — tool-name -> cache hint map for tools/call
|
|
@@ -177,16 +172,16 @@ export function applySpecHygiene(server, overrides = {}) {
|
|
|
177
172
|
|
|
178
173
|
const innerToolsList = protocol._requestHandlers.get('tools/list');
|
|
179
174
|
if (innerToolsList) {
|
|
180
|
-
protocol.setRequestHandler(
|
|
175
|
+
protocol.setRequestHandler('tools/list', wrapToolsList(innerToolsList, icon));
|
|
181
176
|
}
|
|
182
177
|
|
|
183
178
|
const innerToolsCall = protocol._requestHandlers.get('tools/call');
|
|
184
179
|
if (innerToolsCall) {
|
|
185
|
-
protocol.setRequestHandler(
|
|
180
|
+
protocol.setRequestHandler('tools/call', wrapToolsCall(innerToolsCall, cacheableTools));
|
|
186
181
|
}
|
|
187
182
|
|
|
188
183
|
const innerPromptsList = protocol._requestHandlers.get('prompts/list');
|
|
189
184
|
if (innerPromptsList) {
|
|
190
|
-
protocol.setRequestHandler(
|
|
185
|
+
protocol.setRequestHandler('prompts/list', wrapPromptsList(innerPromptsList, icon));
|
|
191
186
|
}
|
|
192
187
|
}
|
|
@@ -2,12 +2,11 @@
|
|
|
2
2
|
* stdio transport setup — extracted from server.js runServer().
|
|
3
3
|
* Used when server is launched without the --http flag.
|
|
4
4
|
*/
|
|
5
|
-
|
|
6
|
-
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
5
|
+
import { StdioServerTransport } from "@modelcontextprotocol/server/stdio";
|
|
7
6
|
|
|
8
7
|
/**
|
|
9
8
|
* Connect the MCP server to stdio transport and log startup message.
|
|
10
|
-
* @param {import('@modelcontextprotocol/
|
|
9
|
+
* @param {import('@modelcontextprotocol/server').McpServer} server
|
|
11
10
|
*/
|
|
12
11
|
export async function connectStdio(server) {
|
|
13
12
|
const transport = new StdioServerTransport();
|