crawlforge-mcp-server 5.9.0 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/server.js +36 -17
- package/src/core/AuthManager.js +29 -2
- package/src/schemas/toolOutputSchemas.js +25 -0
- package/src/server/inlineThreshold.js +5 -0
- package/src/server/redaction.js +469 -0
- package/src/server/withAuth.js +42 -2
- package/src/tools/search/batchSearch.js +50 -0
- package/src/tools/search/searchWeb.js +44 -0
- package/src/utils/secretMask.js +11 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crawlforge-mcp-server",
|
|
3
|
-
"version": "5.
|
|
3
|
+
"version": "5.10.0",
|
|
4
4
|
"mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
|
|
5
5
|
"description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
|
|
6
6
|
"main": "server.js",
|
|
@@ -116,7 +116,7 @@
|
|
|
116
116
|
"cheerio": "^1.1.2",
|
|
117
117
|
"commander": "^14.0.3",
|
|
118
118
|
"compromise": "^14.14.4",
|
|
119
|
-
"crawlforge-extractors": "^1.
|
|
119
|
+
"crawlforge-extractors": "^1.9.0",
|
|
120
120
|
"diff": "^9.0.0",
|
|
121
121
|
"dotenv": "^17.2.1",
|
|
122
122
|
"franc": "^6.2.0",
|
package/server.js
CHANGED
|
@@ -57,6 +57,9 @@ import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
|
|
|
57
57
|
import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
|
|
58
58
|
import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
|
|
59
59
|
import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
|
|
60
|
+
import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
|
|
61
|
+
import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
|
|
62
|
+
import { markPreflightRefusal } from "./src/server/requestContext.js";
|
|
60
63
|
// D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
|
|
61
64
|
import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
|
|
62
65
|
import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
|
|
@@ -109,7 +112,7 @@ const taskStore = createTaskStore({ logger });
|
|
|
109
112
|
// Create the server
|
|
110
113
|
const server = new McpServer({
|
|
111
114
|
name: "crawlforge",
|
|
112
|
-
version: "5.
|
|
115
|
+
version: "5.10.0",
|
|
113
116
|
description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
|
|
114
117
|
homepage: "https://www.crawlforge.dev",
|
|
115
118
|
icon: "https://www.crawlforge.dev/icon.png",
|
|
@@ -199,7 +202,7 @@ const metricsEnabled =
|
|
|
199
202
|
const metrics = metricsEnabled ? createMetricsRegistry() : null;
|
|
200
203
|
|
|
201
204
|
// Tool-handler wrapper: auth + credit tracking + structured invocation logging + observability.
|
|
202
|
-
const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics });
|
|
205
|
+
const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics, mcpServer: server });
|
|
203
206
|
|
|
204
207
|
// Initialize tools
|
|
205
208
|
// search_web falls back to AuthManager's stored key (~/.crawlforge/config.json)
|
|
@@ -391,7 +394,8 @@ registerToolIfEnabled("extract_text", {
|
|
|
391
394
|
remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
|
|
392
395
|
remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
|
|
393
396
|
output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
|
|
394
|
-
...COMPLIANCE_PARAMS
|
|
397
|
+
...COMPLIANCE_PARAMS,
|
|
398
|
+
...REDACT_PII_PARAM
|
|
395
399
|
}
|
|
396
400
|
}, withAuth("extract_text", extractTextHandler));
|
|
397
401
|
|
|
@@ -445,13 +449,14 @@ registerToolIfEnabled("scrape_structured", {
|
|
|
445
449
|
|
|
446
450
|
// Tool: search_web
|
|
447
451
|
registerToolIfEnabled("search_web", {
|
|
448
|
-
description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Cost: 5 credits. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
|
|
452
|
+
description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Pass queries:[...] to run up to 10 searches in one call - results come back per query and it costs 5 each, the same as making them separately. Cost: 5 credits per query. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
|
|
449
453
|
annotations: { title: "Search the Web", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
|
|
450
454
|
// Claude Code tool search loads only names + instructions at session start; this flag
|
|
451
455
|
// ships the full definition too, so the first call needs no ToolSearch round-trip.
|
|
452
456
|
_meta: { "anthropic/alwaysLoad": true },
|
|
453
457
|
inputSchema: {
|
|
454
|
-
query: z.string().describe("Search query string"),
|
|
458
|
+
query: z.string().optional().describe("Search query string. Use this OR queries, not both"),
|
|
459
|
+
...SEARCH_QUERIES_PARAM,
|
|
455
460
|
limit: z.number().min(1).max(100).optional().describe("Maximum number of results to return"),
|
|
456
461
|
offset: z.number().min(0).optional().describe("Number of results to skip for pagination"),
|
|
457
462
|
lang: z.string().optional().describe("Language code for results (e.g. 'en', 'fr')"),
|
|
@@ -494,15 +499,22 @@ registerToolIfEnabled("search_web", {
|
|
|
494
499
|
latitude: z.number().min(-90).max(90),
|
|
495
500
|
longitude: z.number().min(-180).max(180)
|
|
496
501
|
}).optional()
|
|
497
|
-
}).optional().describe("Geo/locale targeting for results")
|
|
502
|
+
}).optional().describe("Geo/locale targeting for results"),
|
|
503
|
+
...REDACT_PII_PARAM
|
|
498
504
|
},
|
|
499
505
|
outputSchema: OUTPUT_SCHEMAS.search_web
|
|
500
|
-
}, withAuth("search_web", async ({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
|
|
506
|
+
}, withAuth("search_web", async ({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
|
|
501
507
|
try {
|
|
502
|
-
if (!query) {
|
|
503
|
-
|
|
508
|
+
if (!query && !queries) {
|
|
509
|
+
// `query` stopped being a required field when `queries` arrived, so the
|
|
510
|
+
// SDK no longer rejects this call for us — it reaches the handler, and
|
|
511
|
+
// without this it would be billed the half-credit error rate for a
|
|
512
|
+
// search that never ran. Same reasoning as ScrapeTemplateTool's
|
|
513
|
+
// badRequest: we fetched nothing, so it costs nothing (G4).
|
|
514
|
+
markPreflightRefusal('BAD_REQUEST');
|
|
515
|
+
return { content: [{ type: "text", text: EXACTLY_ONE_QUERY_MESSAGE }], isError: true };
|
|
504
516
|
}
|
|
505
|
-
const result = await searchWebTool.execute({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
|
|
517
|
+
const result = await searchWebTool.execute({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
|
|
506
518
|
return dualOutput(result);
|
|
507
519
|
} catch (error) {
|
|
508
520
|
return { content: [{ type: "text", text: `Search failed: ${error.message}` }], isError: true };
|
|
@@ -600,7 +612,8 @@ if (toolFilter.isEnabled("crawl_deep")) {
|
|
|
600
612
|
body: z.string().optional()
|
|
601
613
|
}).optional()
|
|
602
614
|
}).optional().describe("Shared cookie-jar/session for login-then-crawl workflows"),
|
|
603
|
-
...MAX_INLINE_CHARS_PARAM
|
|
615
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
616
|
+
...REDACT_PII_PARAM
|
|
604
617
|
},
|
|
605
618
|
outputSchema: OUTPUT_SCHEMAS.crawl_deep,
|
|
606
619
|
execution: TASK_EXECUTION
|
|
@@ -663,7 +676,8 @@ registerToolIfEnabled("extract_content", {
|
|
|
663
676
|
url: z.string().url().describe("The URL to extract content from"),
|
|
664
677
|
options: z.object({}).passthrough().optional().describe("Additional extraction options"),
|
|
665
678
|
...COMPLIANCE_PARAMS,
|
|
666
|
-
...MAX_INLINE_CHARS_PARAM
|
|
679
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
680
|
+
...REDACT_PII_PARAM
|
|
667
681
|
}
|
|
668
682
|
}, withAuth("extract_content", async (params) => {
|
|
669
683
|
try {
|
|
@@ -688,7 +702,8 @@ registerToolIfEnabled("process_document", {
|
|
|
688
702
|
// extractText, outputFormat, etc.) reach the tool instead of being stripped.
|
|
689
703
|
options: z.object({}).passthrough().optional().describe("Additional processing options (maxPages, pageRange:{start,end}, extractText, extractMetadata, outputFormat, ...)"),
|
|
690
704
|
...COMPLIANCE_PARAMS,
|
|
691
|
-
...MAX_INLINE_CHARS_PARAM
|
|
705
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
706
|
+
...REDACT_PII_PARAM
|
|
692
707
|
}
|
|
693
708
|
}, withAuth("process_document", async (params) => {
|
|
694
709
|
try {
|
|
@@ -854,7 +869,8 @@ if (toolFilter.isEnabled("batch_scrape")) {
|
|
|
854
869
|
tags: z.array(z.string()).default([])
|
|
855
870
|
}).optional().describe("Job management options for async processing"),
|
|
856
871
|
...COMPLIANCE_PARAMS,
|
|
857
|
-
...MAX_INLINE_CHARS_PARAM
|
|
872
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
873
|
+
...REDACT_PII_PARAM
|
|
858
874
|
},
|
|
859
875
|
execution: TASK_EXECUTION
|
|
860
876
|
}, makeTaskToolHandler({
|
|
@@ -984,7 +1000,8 @@ registerToolIfEnabled("scrape_with_actions", {
|
|
|
984
1000
|
maxRetries: z.number().min(0).max(3).default(1).describe("Maximum retry attempts on failure"),
|
|
985
1001
|
screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
|
|
986
1002
|
respect_robots: COMPLIANCE_PARAMS.respect_robots,
|
|
987
|
-
...MAX_INLINE_CHARS_PARAM
|
|
1003
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
1004
|
+
...REDACT_PII_PARAM
|
|
988
1005
|
}
|
|
989
1006
|
}, withAuth("scrape_with_actions", async (params) => {
|
|
990
1007
|
try {
|
|
@@ -1091,7 +1108,8 @@ registerToolIfEnabled("scrape", {
|
|
|
1091
1108
|
inputSchema: {
|
|
1092
1109
|
...SCRAPE_INPUT_SHAPE,
|
|
1093
1110
|
...COMPLIANCE_PARAMS,
|
|
1094
|
-
...MAX_INLINE_CHARS_PARAM
|
|
1111
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
1112
|
+
...REDACT_PII_PARAM
|
|
1095
1113
|
},
|
|
1096
1114
|
outputSchema: OUTPUT_SCHEMAS.scrape
|
|
1097
1115
|
}, withAuth("scrape", async (params) => {
|
|
@@ -1409,7 +1427,8 @@ registerToolIfEnabled("stealth_mode", {
|
|
|
1409
1427
|
wait_for: z.number().min(0).max(30000).optional().describe("Extra wait after page load, in ms — for content that renders after DOMContentLoaded"),
|
|
1410
1428
|
verbose: z.boolean().optional().default(false).describe("Return the full generated fingerprint from create_context instead of a summary"),
|
|
1411
1429
|
respect_robots: COMPLIANCE_PARAMS.respect_robots,
|
|
1412
|
-
...MAX_INLINE_CHARS_PARAM
|
|
1430
|
+
...MAX_INLINE_CHARS_PARAM,
|
|
1431
|
+
...REDACT_PII_PARAM
|
|
1413
1432
|
}
|
|
1414
1433
|
}, withAuth("stealth_mode", async ({ operation, stealthConfig, contextId, urlToTest, url, formats, wait_for, verbose, engine, respect_robots }) => {
|
|
1415
1434
|
try {
|
package/src/core/AuthManager.js
CHANGED
|
@@ -14,6 +14,8 @@ import { logger } from '../utils/Logger.js';
|
|
|
14
14
|
import { maskSecrets } from '../utils/secretMask.js';
|
|
15
15
|
import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
|
|
16
16
|
import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
|
|
17
|
+
import { searchQueryCount } from '../tools/search/batchSearch.js';
|
|
18
|
+
import { redactionSurcharge, REDACT_PII_MODEL_CREDITS } from '../server/redaction.js';
|
|
17
19
|
// Stamped on every usage report so support can tell which client version
|
|
18
20
|
// made a call; it read '3.0.3' from 3.0.3 to 5.6.10 (the website stores it
|
|
19
21
|
// per record from its Phase 1.3 on).
|
|
@@ -626,6 +628,11 @@ class AuthManager {
|
|
|
626
628
|
return costs.search_web;
|
|
627
629
|
}
|
|
628
630
|
|
|
631
|
+
// redact_pii's regex pass is free; its opt-in model pass adds 3 once per
|
|
632
|
+
// call, on every tool that offers the param. The rule lives with the
|
|
633
|
+
// schema that declares it (src/server/redaction.js).
|
|
634
|
+
const redaction = redactionSurcharge(tool, params);
|
|
635
|
+
|
|
629
636
|
// The query-scoped formats (highlights, question) add 1 once per call,
|
|
630
637
|
// and mode:"model" adds 3 once; escalate:true adds the stealth browser's
|
|
631
638
|
// own 5, since that is the ceiling the call may reach. The base price in
|
|
@@ -633,10 +640,17 @@ class AuthManager {
|
|
|
633
640
|
// that declare the params.
|
|
634
641
|
if (tool === 'scrape') {
|
|
635
642
|
const { query, model } = scrapeFormatSurcharge(params?.formats);
|
|
636
|
-
return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate);
|
|
643
|
+
return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// search_web's batch form (5.1) runs one backend search per query, so it
|
|
647
|
+
// is priced as that many search_web calls. The count rule lives with the
|
|
648
|
+
// schema that declares `queries` (src/tools/search/batchSearch.js).
|
|
649
|
+
if (tool === 'search_web') {
|
|
650
|
+
return costs.search_web * searchQueryCount(params?.queries) + redaction;
|
|
637
651
|
}
|
|
638
652
|
|
|
639
|
-
return costs[tool] ?? 1;
|
|
653
|
+
return (costs[tool] ?? 1) + redaction;
|
|
640
654
|
}
|
|
641
655
|
|
|
642
656
|
/**
|
|
@@ -677,6 +691,13 @@ class AuthManager {
|
|
|
677
691
|
note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
|
|
678
692
|
break;
|
|
679
693
|
}
|
|
694
|
+
case 'search_web': {
|
|
695
|
+
const queryCount = searchQueryCount(params?.queries);
|
|
696
|
+
note = queryCount > 1
|
|
697
|
+
? `5 per query, ${queryCount} queries — exactly what ${queryCount} separate calls cost. A query that fails before it reaches a backend is not billed.`
|
|
698
|
+
: 'Fixed cost per invocation.';
|
|
699
|
+
break;
|
|
700
|
+
}
|
|
680
701
|
case 'extract_with_llm':
|
|
681
702
|
note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
|
|
682
703
|
break;
|
|
@@ -708,6 +729,12 @@ class AuthManager {
|
|
|
708
729
|
note = 'Fixed cost per invocation.';
|
|
709
730
|
}
|
|
710
731
|
|
|
732
|
+
// redact_pii applies to nine different tools, so its note is appended
|
|
733
|
+
// rather than owned by any one case.
|
|
734
|
+
if (redactionSurcharge(toolName, params) > 0) {
|
|
735
|
+
note += ` redact_pii mode:"model" adds ${REDACT_PII_MODEL_CREDITS} once for the PERSON/LOCATION pass; the charge drops back when no LLM route answered.`;
|
|
736
|
+
}
|
|
737
|
+
|
|
711
738
|
return { projected, note };
|
|
712
739
|
}
|
|
713
740
|
|
|
@@ -42,6 +42,16 @@ const resultHandleShape = {
|
|
|
42
42
|
expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
|
|
43
43
|
};
|
|
44
44
|
|
|
45
|
+
// ── redact_pii (Phase 5, 5.3) ───────────────────────────────────────────────
|
|
46
|
+
|
|
47
|
+
// What the redaction stage reports back on every tool that offers redact_pii.
|
|
48
|
+
const redactionShape = z.object({
|
|
49
|
+
entities: z.record(z.number()).optional().describe('How many spans were replaced, by entity class; a class with no hits is omitted'),
|
|
50
|
+
count: z.number().optional().describe('Total spans replaced'),
|
|
51
|
+
mode: z.enum(['fast', 'model']).optional().describe('"fast" is the free regex pass; "model" added an Ollama NER pass for PERSON and LOCATION'),
|
|
52
|
+
model_ran: z.boolean().optional().describe('mode "model" only: whether a model actually answered. False means no LLM route existed and the model surcharge was not charged')
|
|
53
|
+
}).passthrough().optional().describe('Present when redact_pii was set: what was redacted from the text of this result');
|
|
54
|
+
|
|
45
55
|
// ── scrape ──────────────────────────────────────────────────────────────────
|
|
46
56
|
|
|
47
57
|
const scrapeLinkShape = z.object({
|
|
@@ -131,6 +141,7 @@ const scrapeShape = {
|
|
|
131
141
|
}).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
|
|
132
142
|
content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
|
|
133
143
|
warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
|
|
144
|
+
redaction: redactionShape,
|
|
134
145
|
...resultHandleShape,
|
|
135
146
|
_cost: costShape
|
|
136
147
|
};
|
|
@@ -272,8 +283,20 @@ const searchWebResultShape = z.object({
|
|
|
272
283
|
metadata: z.record(z.unknown()).optional()
|
|
273
284
|
}).passthrough();
|
|
274
285
|
|
|
286
|
+
// The batch form (5.1): one entry per query, in the order they were given.
|
|
287
|
+
// Each carries the same fields a single-query call returns, plus the query
|
|
288
|
+
// itself — or `error` when that one query failed and the others did not.
|
|
289
|
+
const searchWebBatchEntryShape = z.object({
|
|
290
|
+
query: z.string().optional(),
|
|
291
|
+
error: z.string().optional().describe('Present when this query failed; the other queries in the batch are unaffected'),
|
|
292
|
+
results: z.array(searchWebResultShape).optional()
|
|
293
|
+
}).passthrough();
|
|
294
|
+
|
|
275
295
|
const searchWebShape = {
|
|
276
296
|
query: z.string().optional(),
|
|
297
|
+
queries: z.array(z.string()).optional().describe('Batch form: the queries that ran, in order'),
|
|
298
|
+
count: z.number().optional().describe('Batch form: how many queries ran'),
|
|
299
|
+
results_by_query: z.array(searchWebBatchEntryShape).optional().describe('Batch form: one entry per query, in order'),
|
|
277
300
|
effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
|
|
278
301
|
expanded_queries: z.array(z.string()).optional(),
|
|
279
302
|
results: z.array(searchWebResultShape).optional(),
|
|
@@ -302,6 +325,7 @@ const searchWebShape = {
|
|
|
302
325
|
query_expansion: z.record(z.unknown()).nullable().optional(),
|
|
303
326
|
localization_applied: z.boolean().optional()
|
|
304
327
|
}).passthrough().optional(),
|
|
328
|
+
redaction: redactionShape,
|
|
305
329
|
_cost: costShape
|
|
306
330
|
};
|
|
307
331
|
|
|
@@ -380,6 +404,7 @@ const crawlDeepShape = {
|
|
|
380
404
|
}).passthrough().optional(),
|
|
381
405
|
crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
|
|
382
406
|
cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
|
|
407
|
+
redaction: redactionShape,
|
|
383
408
|
...resultHandleShape,
|
|
384
409
|
_cost: costShape
|
|
385
410
|
};
|
|
@@ -135,6 +135,11 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
|
|
|
135
135
|
shaped[key] = value;
|
|
136
136
|
}
|
|
137
137
|
}
|
|
138
|
+
// redact_pii's report is an object, so the scalar filter above drops it —
|
|
139
|
+
// and the caller was charged for it. Carry it through like warnings.
|
|
140
|
+
if (resultObject.redaction && typeof resultObject.redaction === 'object') {
|
|
141
|
+
shaped.redaction = resultObject.redaction;
|
|
142
|
+
}
|
|
138
143
|
Object.assign(shaped, {
|
|
139
144
|
preview,
|
|
140
145
|
result_handle: handle,
|
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* redaction — the one declaration (G5) of `redact_pii`: the parameter, the
|
|
3
|
+
* entity and style enums, the per-tool text paths, the credit rule and the
|
|
4
|
+
* stage that applies it (Phase 5, 5.3).
|
|
5
|
+
*
|
|
6
|
+
* Nothing in this server scrubbed personal data out of the page text it hands
|
|
7
|
+
* back. `secretMask.js` covered API keys and tokens in our own LOG lines and
|
|
8
|
+
* nothing else, so a support inbox, a staff phone list or a checkout form's
|
|
9
|
+
* card number travelled straight into the caller's context window. `redact_pii`
|
|
10
|
+
* closes that: opt-in, off by default, and priced at nothing for the regex pass.
|
|
11
|
+
*
|
|
12
|
+
* Applied by withAuth after a successful handler run and BEFORE the
|
|
13
|
+
* inline-threshold stage stores the result. That order is the whole point: a
|
|
14
|
+
* result stored unredacted would be served back verbatim by `read_result`,
|
|
15
|
+
* which is exactly the hole this exists to close.
|
|
16
|
+
*
|
|
17
|
+
* Two passes:
|
|
18
|
+
*
|
|
19
|
+
* fast (default, 0 credits) — `redactPii` from crawlforge-extractors: pure
|
|
20
|
+
* regex over EMAIL, PHONE, FINANCIAL and SECRET, deterministic, no network.
|
|
21
|
+
* model (opt-in, +3 credits once per call) — an Ollama-first NER pass for
|
|
22
|
+
* PERSON and LOCATION, which no regex can decide. The regex pass ALWAYS
|
|
23
|
+
* runs first and the model only adds to it. G3: the model identifies
|
|
24
|
+
* spans, this module replaces them, and a span the model invented — one
|
|
25
|
+
* that is not present verbatim in the input — is discarded. The text it
|
|
26
|
+
* reads is fenced by `fenceUntrusted` and is already regex-redacted, so
|
|
27
|
+
* the fence never wraps unredacted text.
|
|
28
|
+
*
|
|
29
|
+
* An `entities` name the chosen mode cannot serve is a validation error, not
|
|
30
|
+
* a silent omission — see `entityNameIssue`. Asking for a class and being
|
|
31
|
+
* handed a page that still contains it is the failure this feature exists to
|
|
32
|
+
* prevent, so the schema refuses before anything is fetched, at no charge.
|
|
33
|
+
*
|
|
34
|
+
* The model pass is bounded by wall clock and by cost, not by taste: it runs
|
|
35
|
+
* once per TEXT, not once per call, so a 50-URL batch_scrape left unbounded
|
|
36
|
+
* would outlive any client's patience and — where the SamplingClient chain
|
|
37
|
+
* reaches a server-side API key rather than local Ollama — spend real money
|
|
38
|
+
* against a single +3 surcharge. Three caps, so the exposure of one call is a
|
|
39
|
+
* fixed number: at most MODEL_TEXT_CHARS of any one text, MODEL_BUDGET_CHARS
|
|
40
|
+
* across the call, and MODEL_MAX_PASSES completions. A warning says when any
|
|
41
|
+
* of them bit.
|
|
42
|
+
*
|
|
43
|
+
* Known limit, stated rather than hidden: `scrape`'s own query-scoped formats
|
|
44
|
+
* (`highlights`/`question` with `mode: "model"`, Phase 1) run their model step
|
|
45
|
+
* INSIDE the handler, before this stage, so those prompts see the page text as
|
|
46
|
+
* fetched. Everything that leaves the server is redacted; that one internal
|
|
47
|
+
* prompt is not. Closing it means redacting inside unifiedScrape as well,
|
|
48
|
+
* which would put the application logic in two places.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import { z } from 'zod';
|
|
52
|
+
import {
|
|
53
|
+
redactPii,
|
|
54
|
+
REGEX_ENTITIES,
|
|
55
|
+
MODEL_ONLY_ENTITIES,
|
|
56
|
+
DEFAULT_REPLACE_STYLE
|
|
57
|
+
} from 'crawlforge-extractors';
|
|
58
|
+
import { fenceUntrusted } from '../utils/untrustedContent.js';
|
|
59
|
+
|
|
60
|
+
/** What the model-backed PERSON/LOCATION pass adds to a call's price. */
|
|
61
|
+
export const REDACT_PII_MODEL_CREDITS = 3;
|
|
62
|
+
|
|
63
|
+
/** How much of any one text the model pass reads (summarizeContent's cap). */
|
|
64
|
+
const MODEL_TEXT_CHARS = 12000;
|
|
65
|
+
|
|
66
|
+
/** How much text the model pass reads across a whole call. */
|
|
67
|
+
const MODEL_BUDGET_CHARS = 60000;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* How many completions the model pass may make in one call. Derived, not
|
|
71
|
+
* chosen: it is what the character budget already allows at full size, so a
|
|
72
|
+
* result made of many SHORT texts (a 50-URL batch_scrape) cannot turn one
|
|
73
|
+
* +3 surcharge into fifty round-trips.
|
|
74
|
+
*/
|
|
75
|
+
const MODEL_MAX_PASSES = MODEL_BUDGET_CHARS / MODEL_TEXT_CHARS;
|
|
76
|
+
|
|
77
|
+
/** Every entity name the public parameter accepts. */
|
|
78
|
+
export const REDACT_PII_ENTITIES = Object.freeze([...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES]);
|
|
79
|
+
|
|
80
|
+
/** The tag names this module writes, so a model can never re-redact one. */
|
|
81
|
+
const TAG_NAMES = new Set([...REDACT_PII_ENTITIES, 'REDACTED']);
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Why an entity name this mode cannot serve is REJECTED rather than dropped.
|
|
85
|
+
*
|
|
86
|
+
* `redactPii` drops a name it does not handle in silence, and it is right to:
|
|
87
|
+
* a library has no channel to tell a caller anything. A tool does. Under a
|
|
88
|
+
* lenient rule `entities: ["EMAIL", "PERSON"]` with the default `fast` mode
|
|
89
|
+
* returns a page with every personal name still in it, plus a redaction
|
|
90
|
+
* report listing only the emails — a partial version of the exact failure
|
|
91
|
+
* this feature exists to prevent, with no signal that half the request was
|
|
92
|
+
* ignored. So the schema refuses it, before anything is fetched and at no
|
|
93
|
+
* charge, and says which of the two things went wrong.
|
|
94
|
+
*
|
|
95
|
+
* @param {string} name as the caller wrote it
|
|
96
|
+
* @param {unknown} mode the call's `mode`
|
|
97
|
+
* @returns {string|null} the validation message, or null when the name is fine
|
|
98
|
+
*/
|
|
99
|
+
function entityNameIssue(name, mode) {
|
|
100
|
+
const upper = typeof name === 'string' ? name.toUpperCase() : '';
|
|
101
|
+
if (REGEX_ENTITIES.includes(upper)) return null;
|
|
102
|
+
if (MODEL_ONLY_ENTITIES.includes(upper)) {
|
|
103
|
+
if (mode === 'model') return null;
|
|
104
|
+
return `${MODEL_ONLY_ENTITIES.join(' and ')} need mode: "model" — no regex can decide them. Add mode:"model" (+${REDACT_PII_MODEL_CREDITS} credits once per call) or drop "${name}"`;
|
|
105
|
+
}
|
|
106
|
+
return `unknown entity name "${name}" — expected one of ${REGEX_ENTITIES.join(', ')}, or ${MODEL_ONLY_ENTITIES.join('/')} with mode:"model"`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The object form of `redact_pii`; names are matched case-insensitively. */
|
|
110
|
+
const RedactPiiOptionsSchema = z.object({
|
|
111
|
+
entities: z.array(z.string()).optional().describe(`Which classes to redact, case-insensitive: ${REGEX_ENTITIES.join(', ')}, plus ${MODEL_ONLY_ENTITIES.join(' and ')} when mode is "model". Omitted or empty means all four regex classes (and both model classes in "model" mode). An unknown name, or a model-only name without mode:"model", is rejected`),
|
|
112
|
+
replace_style: z.enum(['tag', 'mask', 'remove']).optional().describe('"tag" (default) writes <EMAIL>, "mask" writes [REDACTED], "remove" deletes the value'),
|
|
113
|
+
mode: z.enum(['fast', 'model']).optional().describe('"fast" (default) is regex only and free; "model" adds an Ollama NER pass for PERSON and LOCATION (+3 credits once per call)')
|
|
114
|
+
}).superRefine((value, ctx) => {
|
|
115
|
+
if (!Array.isArray(value.entities)) return;
|
|
116
|
+
value.entities.forEach((name, index) => {
|
|
117
|
+
const message = entityNameIssue(name, value.mode);
|
|
118
|
+
if (message) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['entities', index], message });
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The public parameter, with the text the client sees. Spread into the input
|
|
124
|
+
* schema of each tool in REDACTION_TOOLS.
|
|
125
|
+
*/
|
|
126
|
+
export const REDACT_PII_PARAM = {
|
|
127
|
+
redact_pii: z.union([
|
|
128
|
+
z.boolean(),
|
|
129
|
+
RedactPiiOptionsSchema
|
|
130
|
+
]).optional().describe('Redact personal data from the text this call returns, before it reaches your context window. true means the free regex pass over EMAIL, PHONE, FINANCIAL and SECRET. The result carries redaction:{entities,count}. Default: off')
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Where each tool keeps the text a redaction must cover. `[]` walks an array.
|
|
135
|
+
* Every path listed is redacted, not just the first one that matches — a
|
|
136
|
+
* single unredacted copy (extract_content's `readability.textContent`, say)
|
|
137
|
+
* would defeat the whole feature.
|
|
138
|
+
*/
|
|
139
|
+
export const REDACTION_TOOLS = Object.freeze({
|
|
140
|
+
scrape: [
|
|
141
|
+
'content.markdown', 'content.text', 'content.html', 'content.rawHtml',
|
|
142
|
+
'content.highlights[].text', 'content.answer.text', 'content.answer.evidence[].text'
|
|
143
|
+
],
|
|
144
|
+
extract_content: [
|
|
145
|
+
'content.text', 'content.html', 'content.markdown', 'content.cleanedHTML',
|
|
146
|
+
'readability.content', 'readability.textContent', 'readability.excerpt'
|
|
147
|
+
],
|
|
148
|
+
extract_text: ['text', 'markdown'],
|
|
149
|
+
batch_scrape: ['results[].content.text', 'results[].content.html', 'results[].content.markdown'],
|
|
150
|
+
crawl_deep: ['results[].content'],
|
|
151
|
+
stealth_mode: ['content.text', 'content.html', 'content.markdown'],
|
|
152
|
+
scrape_with_actions: ['content.text', 'content.html', 'content.markdown'],
|
|
153
|
+
process_document: ['content.text', 'content.html', 'content.extractedContent'],
|
|
154
|
+
search_web: [
|
|
155
|
+
'results[].snippet', 'results[].htmlSnippet',
|
|
156
|
+
'results_by_query[].results[].snippet', 'results_by_query[].results[].htmlSnippet'
|
|
157
|
+
]
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Normalise the raw `redact_pii` param into what the two passes need, or null
|
|
162
|
+
* when redaction is off. Reads the RAW value — getToolCost runs before
|
|
163
|
+
* validation and this is the same declaration it prices from — so anything
|
|
164
|
+
* that is neither `true` nor a plain object means off.
|
|
165
|
+
*
|
|
166
|
+
* An absent or empty `entities` means everything the chosen mode can do. An
|
|
167
|
+
* explicit list is honoured exactly: names a regex can decide go to
|
|
168
|
+
* `redactPii`, and PERSON/LOCATION go to the model pass.
|
|
169
|
+
*
|
|
170
|
+
* This is the TOLERANT layer, deliberately. `RedactPiiOptionsSchema` has
|
|
171
|
+
* already rejected an unknown name, and a model-only name outside `model`
|
|
172
|
+
* mode, at no charge — but this function also runs on unvalidated params
|
|
173
|
+
* (getToolCost, and a REST proxy request that arrives pre-validated by the
|
|
174
|
+
* website), so it must resolve rather than throw. Where the strict schema
|
|
175
|
+
* would have refused, this resolves to a selection that redacts nothing by
|
|
176
|
+
* regex, and `applyRedaction` honours that rather than falling back to all.
|
|
177
|
+
*
|
|
178
|
+
* @param {unknown} raw
|
|
179
|
+
* @returns {{ entities: string[], modelEntities: string[], replaceStyle: string, mode: 'fast'|'model' } | null}
|
|
180
|
+
*/
|
|
181
|
+
export function resolveRedaction(raw) {
|
|
182
|
+
if (raw !== true && (raw === null || typeof raw !== 'object' || Array.isArray(raw))) return null;
|
|
183
|
+
const options = raw === true ? {} : raw;
|
|
184
|
+
|
|
185
|
+
const mode = options.mode === 'model' ? 'model' : 'fast';
|
|
186
|
+
const replaceStyle = ['tag', 'mask', 'remove'].includes(options.replace_style)
|
|
187
|
+
? options.replace_style
|
|
188
|
+
: DEFAULT_REPLACE_STYLE;
|
|
189
|
+
|
|
190
|
+
const asked = Array.isArray(options.entities) && options.entities.length > 0
|
|
191
|
+
? options.entities.filter((name) => typeof name === 'string').map((name) => name.toUpperCase())
|
|
192
|
+
: [...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES];
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
entities: REGEX_ENTITIES.filter((name) => asked.includes(name)),
|
|
196
|
+
modelEntities: mode === 'model' ? MODEL_ONLY_ENTITIES.filter((name) => asked.includes(name)) : [],
|
|
197
|
+
replaceStyle,
|
|
198
|
+
mode
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* What redaction adds to a tool's price: nothing for the regex pass, and the
|
|
204
|
+
* model pass's own 3 once per call. Charged once rather than per page for the
|
|
205
|
+
* same reason `batch_scrape` and `crawl_deep` are priced flat — the page count
|
|
206
|
+
* is not knowable before the call, and a projection scaled by `max_pages`
|
|
207
|
+
* would have the credit check reject an ordinary crawl. Projected is the
|
|
208
|
+
* ceiling (G4); withAuth drops it when no model completed.
|
|
209
|
+
*
|
|
210
|
+
* @param {string} toolName
|
|
211
|
+
* @param {object} [params] the call's raw params
|
|
212
|
+
* @returns {0 | 3}
|
|
213
|
+
*/
|
|
214
|
+
export function redactionSurcharge(toolName, params) {
|
|
215
|
+
if (!(toolName in REDACTION_TOOLS)) return 0;
|
|
216
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
217
|
+
return resolved && resolved.modelEntities.length > 0 ? REDACT_PII_MODEL_CREDITS : 0;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── Text routing ─────────────────────────────────────────────────────────────
|
|
221
|
+
|
|
222
|
+
/** "results[].content.text" -> ["results", "[]", "content", "text"]. */
|
|
223
|
+
function segmentsOf(path) {
|
|
224
|
+
const segments = [];
|
|
225
|
+
for (const part of path.split('.')) {
|
|
226
|
+
if (part.endsWith('[]')) {
|
|
227
|
+
segments.push(part.slice(0, -2), '[]');
|
|
228
|
+
} else {
|
|
229
|
+
segments.push(part);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
return segments;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Every string the path reaches, each with the setter that writes it back. */
|
|
236
|
+
function collectAt(node, segments, found) {
|
|
237
|
+
if (node === null || typeof node !== 'object') return;
|
|
238
|
+
const [head, ...rest] = segments;
|
|
239
|
+
|
|
240
|
+
if (head === '[]') {
|
|
241
|
+
if (!Array.isArray(node)) return;
|
|
242
|
+
for (let i = 0; i < node.length; i++) {
|
|
243
|
+
if (rest.length === 0) {
|
|
244
|
+
if (typeof node[i] === 'string') found.push({ text: node[i], set: (v) => { node[i] = v; } });
|
|
245
|
+
} else {
|
|
246
|
+
collectAt(node[i], rest, found);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
if (Array.isArray(node)) return;
|
|
253
|
+
if (rest.length === 0) {
|
|
254
|
+
if (typeof node[head] === 'string') found.push({ text: node[head], set: (v) => { node[head] = v; } });
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
collectAt(node[head], rest, found);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* @param {object} resultObject
|
|
262
|
+
* @param {string[]} paths
|
|
263
|
+
* @returns {Array<{ text: string, set: (value: string) => void }>}
|
|
264
|
+
*/
|
|
265
|
+
export function collectTexts(resultObject, paths) {
|
|
266
|
+
const found = [];
|
|
267
|
+
for (const path of paths) collectAt(resultObject, segmentsOf(path), found);
|
|
268
|
+
return found;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ── The model pass ───────────────────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Parse the model's reply into the spans it claims. A line the model wrote
|
|
275
|
+
* that is not present verbatim in the text is dropped: the model identifies,
|
|
276
|
+
* it never rewrites (G3).
|
|
277
|
+
*
|
|
278
|
+
* @param {string} reply
|
|
279
|
+
* @param {string} text the text the model read
|
|
280
|
+
* @param {string[]} wanted the entity names asked for
|
|
281
|
+
* @returns {Array<{ start: number, end: number, entity: string }>}
|
|
282
|
+
*/
|
|
283
|
+
export function spansFromReply(reply, text, wanted) {
|
|
284
|
+
if (typeof reply !== 'string') return [];
|
|
285
|
+
const spans = [];
|
|
286
|
+
for (const line of reply.split('\n')) {
|
|
287
|
+
const match = /^\s*(?:[-*]\s*)?(PERSON|LOCATION)\s*[:\-]\s*(.+?)\s*$/i.exec(line);
|
|
288
|
+
if (!match) continue;
|
|
289
|
+
const entity = match[1].toUpperCase();
|
|
290
|
+
if (!wanted.includes(entity)) continue;
|
|
291
|
+
const value = match[2].replace(/^["'`]|["'`]$/g, '').trim();
|
|
292
|
+
if (value.length < 2 || TAG_NAMES.has(value.toUpperCase())) continue;
|
|
293
|
+
|
|
294
|
+
let from = 0;
|
|
295
|
+
for (;;) {
|
|
296
|
+
const at = text.indexOf(value, from);
|
|
297
|
+
if (at === -1) break;
|
|
298
|
+
spans.push({ start: at, end: at + value.length, entity });
|
|
299
|
+
from = at + value.length;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
// Longest first, so an overlap resolves in favour of the fuller name.
|
|
303
|
+
spans.sort((a, b) => (b.end - b.start) - (a.end - a.start) || a.start - b.start);
|
|
304
|
+
const accepted = [];
|
|
305
|
+
for (const span of spans) {
|
|
306
|
+
if (accepted.some((other) => span.start < other.end && other.start < span.end)) continue;
|
|
307
|
+
accepted.push(span);
|
|
308
|
+
}
|
|
309
|
+
return accepted.sort((a, b) => a.start - b.start);
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
function replacementFor(entity, style) {
|
|
313
|
+
if (style === 'mask') return '[REDACTED]';
|
|
314
|
+
if (style === 'remove') return '';
|
|
315
|
+
return `<${entity}>`;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** Rebuild `text` with the accepted spans replaced; counts them by entity. */
|
|
319
|
+
function applySpans(text, spans, style, counts) {
|
|
320
|
+
let out = '';
|
|
321
|
+
let cursor = 0;
|
|
322
|
+
for (const span of spans) {
|
|
323
|
+
out += text.slice(cursor, span.start) + replacementFor(span.entity, style);
|
|
324
|
+
counts[span.entity] = (counts[span.entity] || 0) + 1;
|
|
325
|
+
cursor = span.end;
|
|
326
|
+
}
|
|
327
|
+
return out + text.slice(cursor);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
const PROMPT_SYSTEM = 'You mark personal names and place names in text quoted from a web page. ' +
|
|
331
|
+
'Reply with one entry per line, each "PERSON: <text>" or "LOCATION: <text>", copying the text exactly as it appears. ' +
|
|
332
|
+
'Do not translate, shorten or correct it. If there are none, reply NONE. No other output.';
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* One completion for one text. Returns the reply, or throws — the caller
|
|
336
|
+
* treats a throw as "no LLM route", drops the surcharge and warns.
|
|
337
|
+
* @param {(prompt: string, options: object) => Promise<{ text: string }>} complete
|
|
338
|
+
*/
|
|
339
|
+
async function askModel(complete, text, wanted) {
|
|
340
|
+
const { text: reply } = await complete(
|
|
341
|
+
`${fenceUntrusted(text, 'page text')}\nList every ${wanted.join(' and ')} in the text above.`,
|
|
342
|
+
{ maxTokens: 512, systemPrompt: PROMPT_SYSTEM }
|
|
343
|
+
);
|
|
344
|
+
return reply;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
// ── The stage ────────────────────────────────────────────────────────────────
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Redact a tool result in place.
|
|
351
|
+
*
|
|
352
|
+
* @param {string} toolName
|
|
353
|
+
* @param {object} resultObject the parsed JSON result; mutated
|
|
354
|
+
* @param {object} params the call's params
|
|
355
|
+
* @param {{ complete?: (prompt: string, options: object) => Promise<{ text: string }> }} deps
|
|
356
|
+
* @returns {Promise<{ redacted: boolean, modelRan: boolean }>}
|
|
357
|
+
*/
|
|
358
|
+
export async function applyRedaction(toolName, resultObject, params, { complete } = {}) {
|
|
359
|
+
const paths = REDACTION_TOOLS[toolName];
|
|
360
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
361
|
+
if (!paths || !resolved) return { redacted: false, modelRan: false };
|
|
362
|
+
if (!resultObject || typeof resultObject !== 'object' || Array.isArray(resultObject)) {
|
|
363
|
+
return { redacted: false, modelRan: false };
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const targets = collectTexts(resultObject, paths);
|
|
367
|
+
const counts = {};
|
|
368
|
+
const warnings = [];
|
|
369
|
+
|
|
370
|
+
// Skip the regex pass entirely when the caller's selection left it nothing
|
|
371
|
+
// to do (`entities: ['PERSON']`). `redactPii` reads an EMPTY array as "all
|
|
372
|
+
// four", so passing one through would redact everything a caller who asked
|
|
373
|
+
// for one model-only class never asked to lose.
|
|
374
|
+
if (resolved.entities.length > 0) {
|
|
375
|
+
for (const target of targets) {
|
|
376
|
+
const { text, redaction } = redactPii(target.text, {
|
|
377
|
+
entities: resolved.entities,
|
|
378
|
+
replaceStyle: resolved.replaceStyle
|
|
379
|
+
});
|
|
380
|
+
for (const [entity, n] of Object.entries(redaction.entities)) counts[entity] = (counts[entity] || 0) + n;
|
|
381
|
+
// What the model pass reads, so the fence never wraps unredacted text.
|
|
382
|
+
target.text = text;
|
|
383
|
+
target.set(text);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
let modelRan = false;
|
|
388
|
+
if (resolved.modelEntities.length > 0 && typeof complete === 'function') {
|
|
389
|
+
let budget = MODEL_BUDGET_CHARS;
|
|
390
|
+
let capped = false;
|
|
391
|
+
let failure = null;
|
|
392
|
+
|
|
393
|
+
let passes = 0;
|
|
394
|
+
for (const target of targets) {
|
|
395
|
+
const live = target.text; // the regex-redacted copy
|
|
396
|
+
if (budget <= 0 || passes >= MODEL_MAX_PASSES) { capped = true; break; }
|
|
397
|
+
const slice = live.slice(0, Math.min(MODEL_TEXT_CHARS, budget));
|
|
398
|
+
if (slice.length === 0) continue;
|
|
399
|
+
if (slice.length < live.length) capped = true;
|
|
400
|
+
budget -= slice.length;
|
|
401
|
+
passes++;
|
|
402
|
+
|
|
403
|
+
let reply;
|
|
404
|
+
try {
|
|
405
|
+
reply = await askModel(complete, slice, resolved.modelEntities);
|
|
406
|
+
} catch (error) {
|
|
407
|
+
failure = error;
|
|
408
|
+
break;
|
|
409
|
+
}
|
|
410
|
+
modelRan = true;
|
|
411
|
+
const spans = spansFromReply(reply, slice, resolved.modelEntities);
|
|
412
|
+
if (spans.length === 0) continue;
|
|
413
|
+
target.set(applySpans(slice, spans, resolved.replaceStyle, counts) + live.slice(slice.length));
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
if (failure && !modelRan) {
|
|
417
|
+
warnings.push(`redact_pii: no model answered, so ${resolved.modelEntities.join(' and ')} were not redacted (${failure.message}); the model surcharge was not charged`);
|
|
418
|
+
} else if (failure) {
|
|
419
|
+
warnings.push(`redact_pii: the model pass stopped early (${failure.message}); the text it did not reach carries the regex redactions only`);
|
|
420
|
+
}
|
|
421
|
+
if (capped) {
|
|
422
|
+
warnings.push(`redact_pii: the model pass read at most ${MODEL_TEXT_CHARS} characters per text, ${MODEL_BUDGET_CHARS} in total and ${MODEL_MAX_PASSES} texts; text beyond that carries the regex redactions only`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
const count = Object.values(counts).reduce((sum, n) => sum + n, 0);
|
|
427
|
+
resultObject.redaction = {
|
|
428
|
+
entities: counts,
|
|
429
|
+
count,
|
|
430
|
+
mode: resolved.mode,
|
|
431
|
+
...(resolved.modelEntities.length > 0 ? { model_ran: modelRan } : {})
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// The query-scoped formats index the markdown by offset. Redaction changes
|
|
435
|
+
// its length, so say so rather than let the caller trust a stale offset.
|
|
436
|
+
if (count > 0 && resultObject.content && (resultObject.content.highlights || resultObject.content.answer)) {
|
|
437
|
+
warnings.push('redact_pii: highlight and evidence offsets index the text as it was before redaction');
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
if (warnings.length > 0) {
|
|
441
|
+
resultObject.warnings = [
|
|
442
|
+
...(Array.isArray(resultObject.warnings) ? resultObject.warnings : []),
|
|
443
|
+
...warnings
|
|
444
|
+
];
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
return { redacted: true, modelRan };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The redaction stage as withAuth uses it: a SamplingClient-backed `complete`
|
|
452
|
+
* built only when a model pass was actually asked for, so an ordinary call
|
|
453
|
+
* never loads the client.
|
|
454
|
+
*
|
|
455
|
+
* @param {string} toolName
|
|
456
|
+
* @param {object} resultObject
|
|
457
|
+
* @param {object} params
|
|
458
|
+
* @param {{ mcpServer?: object|null }} [deps]
|
|
459
|
+
*/
|
|
460
|
+
export async function runRedactionStage(toolName, resultObject, params, { mcpServer = null } = {}) {
|
|
461
|
+
const resolved = resolveRedaction(params?.redact_pii);
|
|
462
|
+
const complete = resolved && resolved.modelEntities.length > 0
|
|
463
|
+
? async (prompt, options) => {
|
|
464
|
+
const { SamplingClient } = await import('../core/SamplingClient.js');
|
|
465
|
+
return new SamplingClient({ mcpServer }).complete(prompt, options);
|
|
466
|
+
}
|
|
467
|
+
: undefined;
|
|
468
|
+
return applyRedaction(toolName, resultObject, params, { complete });
|
|
469
|
+
}
|
package/src/server/withAuth.js
CHANGED
|
@@ -20,6 +20,7 @@ import { recordToolInvocation } from '../observability/tracing.js';
|
|
|
20
20
|
import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
|
|
21
21
|
import { appendFallbackHint } from './fallbackHints.js';
|
|
22
22
|
import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
|
|
23
|
+
import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
|
|
23
24
|
import { getResultStore } from '../core/ResultStore.js';
|
|
24
25
|
|
|
25
26
|
/**
|
|
@@ -36,6 +37,24 @@ function shapeLargeResult(toolName, result, params) {
|
|
|
36
37
|
if (result.structuredContent) result.structuredContent = shaped;
|
|
37
38
|
}
|
|
38
39
|
|
|
40
|
+
/**
|
|
41
|
+
* Redact the result's page text in place (Phase 5, 5.3). Runs BEFORE
|
|
42
|
+
* shapeLargeResult: a result stored unredacted would be served straight back
|
|
43
|
+
* out by read_result, which is the hole redact_pii exists to close.
|
|
44
|
+
*
|
|
45
|
+
* @returns {Promise<boolean>} whether a model pass actually completed
|
|
46
|
+
*/
|
|
47
|
+
async function redactResult(toolName, result, params, mcpServer) {
|
|
48
|
+
if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return false;
|
|
49
|
+
let parsed;
|
|
50
|
+
try { parsed = JSON.parse(result.content[0].text); } catch { return false; }
|
|
51
|
+
const { redacted, modelRan } = await runRedactionStage(toolName, parsed, params, { mcpServer });
|
|
52
|
+
if (!redacted) return false;
|
|
53
|
+
result.content[0].text = JSON.stringify(parsed, null, 2);
|
|
54
|
+
if (result.structuredContent) result.structuredContent = parsed;
|
|
55
|
+
return modelRan;
|
|
56
|
+
}
|
|
57
|
+
|
|
39
58
|
export function hashParams(params) {
|
|
40
59
|
try {
|
|
41
60
|
return createHash('sha256').update(JSON.stringify(params ?? {})).digest('hex').slice(0, 12);
|
|
@@ -50,7 +69,7 @@ export function hashParams(params) {
|
|
|
50
69
|
* @param {object} deps.logger
|
|
51
70
|
* @param {object} [deps.metrics] — optional Prometheus registry (see src/observability/metrics.js)
|
|
52
71
|
*/
|
|
53
|
-
export function makeWithAuth({ authManager, logger, metrics = null }) {
|
|
72
|
+
export function makeWithAuth({ authManager, logger, metrics = null, mcpServer = null }) {
|
|
54
73
|
return function withAuth(toolName, handler) {
|
|
55
74
|
const invoke = async (params) => {
|
|
56
75
|
const startTime = Date.now();
|
|
@@ -62,6 +81,13 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
|
|
|
62
81
|
const internal = isInternalRequest();
|
|
63
82
|
const billingExempt = creatorMode || internal;
|
|
64
83
|
const creditCost = billingExempt ? 0 : authManager.getToolCost(toolName, params);
|
|
84
|
+
// redact_pii's model pass runs at this seam, AFTER the handler, so the
|
|
85
|
+
// handler's own setActualCost report cannot know about it. Split the
|
|
86
|
+
// projection: the tool prices its own work, and the model surcharge is
|
|
87
|
+
// added back only when a model actually completed (G4).
|
|
88
|
+
const redactionCost = billingExempt ? 0 : Math.min(redactionSurcharge(toolName, params), creditCost);
|
|
89
|
+
const toolCost = creditCost - redactionCost;
|
|
90
|
+
let redactionModelRan = false;
|
|
65
91
|
let outcome = 'pending';
|
|
66
92
|
let thrown = null;
|
|
67
93
|
// Only bill the error-path half-charge once the handler has actually run.
|
|
@@ -73,7 +99,8 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
|
|
|
73
99
|
// lower the charge, never raise it.
|
|
74
100
|
const billable = () => {
|
|
75
101
|
const reported = reportedActualCost();
|
|
76
|
-
|
|
102
|
+
const spent = reported == null ? toolCost : Math.min(reported, toolCost);
|
|
103
|
+
return spent + (redactionModelRan ? redactionCost : 0);
|
|
77
104
|
};
|
|
78
105
|
|
|
79
106
|
try {
|
|
@@ -116,6 +143,19 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
|
|
|
116
143
|
// result did real work and bills for it.
|
|
117
144
|
const refused = isErrorResult && preflightRefusal() !== null;
|
|
118
145
|
|
|
146
|
+
// Phase 5 (5.3): redact_pii scrubs the page text this call returns.
|
|
147
|
+
// It runs BEFORE the inline-threshold stage below, so what gets
|
|
148
|
+
// stored — and what read_result later serves — is already redacted.
|
|
149
|
+
// Internal (website REST proxy) requests are redacted too: the text
|
|
150
|
+
// is the same text, and the website has no copy of it to scrub.
|
|
151
|
+
if (!isErrorResult && toolName in REDACTION_TOOLS) {
|
|
152
|
+
try {
|
|
153
|
+
redactionModelRan = await redactResult(toolName, result, params, mcpServer);
|
|
154
|
+
} catch {
|
|
155
|
+
/* redaction must never break the request path */
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
119
159
|
// Phase 2: a result over max_inline_chars is stored and returned as
|
|
120
160
|
// a preview plus a result_handle for read_result. Internal (website
|
|
121
161
|
// REST proxy) requests are left whole — the website applies its own
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* batchSearch.js — the one declaration of `search_web`'s `queries` param
|
|
3
|
+
* (Phase 5, 5.1).
|
|
4
|
+
*
|
|
5
|
+
* `search_web` takes one query. A caller comparing five phrasings, or
|
|
6
|
+
* checking a term across five sites, pays five round-trips for what is one
|
|
7
|
+
* unit of work. `queries` accepts 1-10 of them in a single call; each still
|
|
8
|
+
* runs through the SAME `execute()` pipeline — expansion, cache, dedupe,
|
|
9
|
+
* rank — so a batch entry and a single call are the same result.
|
|
10
|
+
*
|
|
11
|
+
* The batch wrapper sits ABOVE the provider short-circuit in `execute()`, so
|
|
12
|
+
* the SearXNG branch (which is not part of the adapter factory) inherits
|
|
13
|
+
* `queries` without a second implementation.
|
|
14
|
+
*
|
|
15
|
+
* The field and the price rule live here so `server.js`, the tool module and
|
|
16
|
+
* `AuthManager.getToolCost` read one declaration (G5) — the same arrangement
|
|
17
|
+
* `escalation.js` has for `scrape`.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { z } from 'zod';
|
|
21
|
+
|
|
22
|
+
/** How many queries one call may carry. */
|
|
23
|
+
export const MAX_SEARCH_QUERIES = 10;
|
|
24
|
+
|
|
25
|
+
/** search_web's price for ONE query, as AuthManager's table spells it. */
|
|
26
|
+
export const SEARCH_WEB_CREDITS = 5;
|
|
27
|
+
|
|
28
|
+
/** The public field, with the text the client sees. */
|
|
29
|
+
export const SEARCH_QUERIES_PARAM = {
|
|
30
|
+
queries: z.array(z.string().min(1)).min(1).max(MAX_SEARCH_QUERIES).optional().describe(`Run 1-${MAX_SEARCH_QUERIES} searches in one call instead of ${MAX_SEARCH_QUERIES} round-trips; every other parameter applies to each. Results come back in results_by_query, one entry per query, in order. Costs 5 per query. Use this OR query, not both`)
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/** Rejected when a call carries both `query` and `queries`, or neither. */
|
|
34
|
+
export const EXACTLY_ONE_QUERY_MESSAGE =
|
|
35
|
+
`search_web needs exactly one of "query" (a single search) or "queries" (1-${MAX_SEARCH_QUERIES} searches in one call), not both and not neither`;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* How many searches this call will run, for pricing. Reads the RAW param —
|
|
39
|
+
* getToolCost runs before validation — so anything that is not a non-empty
|
|
40
|
+
* array prices as the single-query form, and a caller who sends more than the
|
|
41
|
+
* schema allows never produces a silly projection (the schema rejects that
|
|
42
|
+
* call before it is billed at all).
|
|
43
|
+
*
|
|
44
|
+
* @param {unknown} queries the raw `queries` param
|
|
45
|
+
* @returns {number} 1..MAX_SEARCH_QUERIES
|
|
46
|
+
*/
|
|
47
|
+
export function searchQueryCount(queries) {
|
|
48
|
+
if (!Array.isArray(queries) || queries.length === 0) return 1;
|
|
49
|
+
return Math.min(queries.length, MAX_SEARCH_QUERIES);
|
|
50
|
+
}
|
|
@@ -8,6 +8,10 @@ import { SearchResultCache } from './ranking/SearchResultCache.js';
|
|
|
8
8
|
import LocalizationManager from '../../core/LocalizationManager.js';
|
|
9
9
|
import { isCreatorModeVerified } from '../../core/creatorMode.js';
|
|
10
10
|
import { searchViaSearxng } from './providers/searxng.js';
|
|
11
|
+
import { MAX_SEARCH_QUERIES, SEARCH_WEB_CREDITS, EXACTLY_ONE_QUERY_MESSAGE } from './batchSearch.js';
|
|
12
|
+
import { setActualCost } from '../../server/requestContext.js';
|
|
13
|
+
|
|
14
|
+
const BatchQueriesSchema = z.array(z.string().min(1)).min(1).max(MAX_SEARCH_QUERIES);
|
|
11
15
|
|
|
12
16
|
const SearchWebSchema = z.object({
|
|
13
17
|
query: z.string().min(1),
|
|
@@ -126,7 +130,47 @@ export class SearchWebTool {
|
|
|
126
130
|
});
|
|
127
131
|
}
|
|
128
132
|
|
|
133
|
+
/**
|
|
134
|
+
* Run 1-10 queries through the single-query pipeline and key the results by
|
|
135
|
+
* query (5.1). Sequential: each entry is a separate billed backend search,
|
|
136
|
+
* and firing ten at once at the provider buys nothing but rate limits.
|
|
137
|
+
*
|
|
138
|
+
* @param {object} params the call's params, `queries` included
|
|
139
|
+
* @returns {Promise<{queries: string[], count: number, results_by_query: object[]}>}
|
|
140
|
+
*/
|
|
141
|
+
async _executeBatch(params) {
|
|
142
|
+
const { queries, query: _ignored, ...rest } = params;
|
|
143
|
+
const parsed = BatchQueriesSchema.parse(queries);
|
|
144
|
+
|
|
145
|
+
const results_by_query = [];
|
|
146
|
+
let ran = 0;
|
|
147
|
+
for (const query of parsed) {
|
|
148
|
+
try {
|
|
149
|
+
results_by_query.push({ query, ...await this.execute({ ...rest, query }) });
|
|
150
|
+
ran++;
|
|
151
|
+
} catch (error) {
|
|
152
|
+
// One failed query does not sink the other nine.
|
|
153
|
+
results_by_query.push({ query, error: error.message });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// The projection is 5 per query; a query that failed before it reached a
|
|
158
|
+
// backend did no work and must not be billed for one (G4). The projection
|
|
159
|
+
// is the ceiling, so this only ever lowers the charge.
|
|
160
|
+
setActualCost(SEARCH_WEB_CREDITS * ran);
|
|
161
|
+
|
|
162
|
+
return { queries: parsed, count: parsed.length, results_by_query };
|
|
163
|
+
}
|
|
164
|
+
|
|
129
165
|
async execute(params) {
|
|
166
|
+
// Batch form (5.1). Handled above the provider short-circuit below, so
|
|
167
|
+
// the SearXNG branch runs each query through the same path the default
|
|
168
|
+
// provider does.
|
|
169
|
+
const hasQuery = params?.query !== undefined;
|
|
170
|
+
const hasQueries = params?.queries !== undefined;
|
|
171
|
+
if (hasQuery === hasQueries) throw new Error(EXACTLY_ONE_QUERY_MESSAGE);
|
|
172
|
+
if (hasQueries) return await this._executeBatch(params);
|
|
173
|
+
|
|
130
174
|
try {
|
|
131
175
|
const validated = SearchWebSchema.parse(params);
|
|
132
176
|
|
package/src/utils/secretMask.js
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
* logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import { redactPii } from 'crawlforge-extractors';
|
|
13
|
+
|
|
12
14
|
const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
|
|
13
15
|
|
|
14
16
|
const MASK = '[REDACTED]';
|
|
@@ -97,16 +99,18 @@ export function maskError(error) {
|
|
|
97
99
|
|
|
98
100
|
/**
|
|
99
101
|
* Heuristic: redact strings that look like API keys / tokens embedded in text.
|
|
102
|
+
*
|
|
103
|
+
* The patterns moved to crawlforge-extractors' `redactPii` (Phase 5, 5.3), so
|
|
104
|
+
* the two surfaces run one implementation and page text gets the same
|
|
105
|
+
* treatment log lines already got. The SECRET class there is this heuristic,
|
|
106
|
+
* ported verbatim and asserted byte-for-byte against it upstream: the label
|
|
107
|
+
* survives and only the value is replaced, which is what keeps an error
|
|
108
|
+
* message saying WHICH credential the request carried.
|
|
109
|
+
*
|
|
100
110
|
* @param {string} str
|
|
101
111
|
* @returns {string}
|
|
102
112
|
*/
|
|
103
113
|
function redactSecretsFromString(str) {
|
|
104
114
|
if (typeof str !== 'string') return str;
|
|
105
|
-
return str
|
|
106
|
-
.replace(/(Bearer\s+)\S+/gi, `$1${MASK}`)
|
|
107
|
-
.replace(/(api[_-]?key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
|
|
108
|
-
.replace(/(x-api-key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
|
|
109
|
-
.replace(/(password\s*[:=]\s*)\S+/gi, `$1${MASK}`)
|
|
110
|
-
.replace(/(secret\s*[:=]\s*)\S+/gi, `$1${MASK}`)
|
|
111
|
-
.replace(/(token\s*[:=]\s*)\S+/gi, `$1${MASK}`);
|
|
115
|
+
return redactPii(str, { entities: ['SECRET'], replaceStyle: 'mask' }).text;
|
|
112
116
|
}
|