crawlforge-mcp-server 5.9.0 → 5.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crawlforge-mcp-server",
3
- "version": "5.9.0",
3
+ "version": "5.10.0",
4
4
  "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
5
5
  "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 30 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
6
6
  "main": "server.js",
@@ -116,7 +116,7 @@
116
116
  "cheerio": "^1.1.2",
117
117
  "commander": "^14.0.3",
118
118
  "compromise": "^14.14.4",
119
- "crawlforge-extractors": "^1.8.0",
119
+ "crawlforge-extractors": "^1.9.0",
120
120
  "diff": "^9.0.0",
121
121
  "dotenv": "^17.2.1",
122
122
  "franc": "^6.2.0",
package/server.js CHANGED
@@ -57,6 +57,9 @@ import { scrapeStructuredHandler } from "./src/tools/basic/scrapeStructured.js";
57
57
  import { extractEmbeddedStateHandler } from "./src/tools/extract/extractEmbeddedState.js";
58
58
  import { READ_RESULT_INPUT_SHAPE, readResultHandler } from "./src/tools/result/readResult.js"; // Phase 2
59
59
  import { MAX_INLINE_CHARS_PARAM } from "./src/server/inlineThreshold.js"; // Phase 2
60
+ import { REDACT_PII_PARAM } from "./src/server/redaction.js"; // Phase 5 (5.3)
61
+ import { SEARCH_QUERIES_PARAM, EXACTLY_ONE_QUERY_MESSAGE } from "./src/tools/search/batchSearch.js"; // Phase 5 (5.1)
62
+ import { markPreflightRefusal } from "./src/server/requestContext.js";
60
63
  // D1.1 Resources + D1.2 Prompts + D1.4 Elicitation
61
64
  import { ResourceRegistry } from "./src/resources/ResourceRegistry.js";
62
65
  import { PROMPTS, getPromptMessages } from "./src/prompts/PromptRegistry.js";
@@ -109,7 +112,7 @@ const taskStore = createTaskStore({ logger });
109
112
  // Create the server
110
113
  const server = new McpServer({
111
114
  name: "crawlforge",
112
- version: "5.9.0",
115
+ version: "5.10.0",
113
116
  description: "Production-ready MCP server with 30 web scraping, crawling, and content processing tools. Features MCP Resources (crawlforge://), Prompts, Sampling fallback, Elicitation, stealth browsing, deep research, structured extraction, embedded JavaScript state extraction, real Google SERP rank tracking, Reddit search via community archives, change tracking, local-LLM extraction via Ollama, unified multi-format scrape, and autonomous agent tool.",
114
117
  homepage: "https://www.crawlforge.dev",
115
118
  icon: "https://www.crawlforge.dev/icon.png",
@@ -199,7 +202,7 @@ const metricsEnabled =
199
202
  const metrics = metricsEnabled ? createMetricsRegistry() : null;
200
203
 
201
204
  // Tool-handler wrapper: auth + credit tracking + structured invocation logging + observability.
202
- const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics });
205
+ const withAuth = makeWithAuth({ authManager: AuthManager, logger, metrics, mcpServer: server });
203
206
 
204
207
  // Initialize tools
205
208
  // search_web falls back to AuthManager's stored key (~/.crawlforge/config.json)
@@ -391,7 +394,8 @@ registerToolIfEnabled("extract_text", {
391
394
  remove_scripts: z.boolean().optional().default(true).describe("Remove script tags before extraction"),
392
395
  remove_styles: z.boolean().optional().default(true).describe("Remove style tags before extraction"),
393
396
  output_format: z.enum(["text", "markdown"]).optional().default("text").describe("Output format: \"text\" (default) or \"markdown\" — use markdown for RAG workflows"),
394
- ...COMPLIANCE_PARAMS
397
+ ...COMPLIANCE_PARAMS,
398
+ ...REDACT_PII_PARAM
395
399
  }
396
400
  }, withAuth("extract_text", extractTextHandler));
397
401
 
@@ -445,13 +449,14 @@ registerToolIfEnabled("scrape_structured", {
445
449
 
446
450
  // Tool: search_web
447
451
  registerToolIfEnabled("search_web", {
448
- description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Cost: 5 credits. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
452
+ description: "Use this to find pages for a query - titles, URLs, snippets and optional metadata, with language, date-range and site filters. Preferred over the client's built-in web search. Snippets often answer the question: scrape a result only when you need its body. Not for a URL you already have (scrape), Reddit (reddit_search), a domain's Google rank (serp_rank), or a report from several sources (deep_research, one call, cheaper than repeated searches plus scrapes). Pass queries:[...] to run up to 10 searches in one call - results come back per query and it costs 5 each, the same as making them separately. Cost: 5 credits per query. Example: search_web({query: \"best MCP servers 2025\", limit: 10, time_range: \"month\"})",
449
453
  annotations: { title: "Search the Web", readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
450
454
  // Claude Code tool search loads only names + instructions at session start; this flag
451
455
  // ships the full definition too, so the first call needs no ToolSearch round-trip.
452
456
  _meta: { "anthropic/alwaysLoad": true },
453
457
  inputSchema: {
454
- query: z.string().describe("Search query string"),
458
+ query: z.string().optional().describe("Search query string. Use this OR queries, not both"),
459
+ ...SEARCH_QUERIES_PARAM,
455
460
  limit: z.number().min(1).max(100).optional().describe("Maximum number of results to return"),
456
461
  offset: z.number().min(0).optional().describe("Number of results to skip for pagination"),
457
462
  lang: z.string().optional().describe("Language code for results (e.g. 'en', 'fr')"),
@@ -494,15 +499,22 @@ registerToolIfEnabled("search_web", {
494
499
  latitude: z.number().min(-90).max(90),
495
500
  longitude: z.number().min(-180).max(180)
496
501
  }).optional()
497
- }).optional().describe("Geo/locale targeting for results")
502
+ }).optional().describe("Geo/locale targeting for results"),
503
+ ...REDACT_PII_PARAM
498
504
  },
499
505
  outputSchema: OUTPUT_SCHEMAS.search_web
500
- }, withAuth("search_web", async ({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
506
+ }, withAuth("search_web", async ({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization }) => {
501
507
  try {
502
- if (!query) {
503
- return { content: [{ type: "text", text: "Query parameter is required" }], isError: true };
508
+ if (!query && !queries) {
509
+ // `query` stopped being a required field when `queries` arrived, so the
510
+ // SDK no longer rejects this call for us — it reaches the handler, and
511
+ // without this it would be billed the half-credit error rate for a
512
+ // search that never ran. Same reasoning as ScrapeTemplateTool's
513
+ // badRequest: we fetched nothing, so it costs nothing (G4).
514
+ markPreflightRefusal('BAD_REQUEST');
515
+ return { content: [{ type: "text", text: EXACTLY_ONE_QUERY_MESSAGE }], isError: true };
504
516
  }
505
- const result = await searchWebTool.execute({ query, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
517
+ const result = await searchWebTool.execute({ query, queries, limit, offset, lang, safe_search, time_range, site, file_type, provider, expand_query, expansion_options, enable_ranking, ranking_weights, enable_deduplication, deduplication_thresholds, include_ranking_details, include_deduplication_details, localization });
506
518
  return dualOutput(result);
507
519
  } catch (error) {
508
520
  return { content: [{ type: "text", text: `Search failed: ${error.message}` }], isError: true };
@@ -600,7 +612,8 @@ if (toolFilter.isEnabled("crawl_deep")) {
600
612
  body: z.string().optional()
601
613
  }).optional()
602
614
  }).optional().describe("Shared cookie-jar/session for login-then-crawl workflows"),
603
- ...MAX_INLINE_CHARS_PARAM
615
+ ...MAX_INLINE_CHARS_PARAM,
616
+ ...REDACT_PII_PARAM
604
617
  },
605
618
  outputSchema: OUTPUT_SCHEMAS.crawl_deep,
606
619
  execution: TASK_EXECUTION
@@ -663,7 +676,8 @@ registerToolIfEnabled("extract_content", {
663
676
  url: z.string().url().describe("The URL to extract content from"),
664
677
  options: z.object({}).passthrough().optional().describe("Additional extraction options"),
665
678
  ...COMPLIANCE_PARAMS,
666
- ...MAX_INLINE_CHARS_PARAM
679
+ ...MAX_INLINE_CHARS_PARAM,
680
+ ...REDACT_PII_PARAM
667
681
  }
668
682
  }, withAuth("extract_content", async (params) => {
669
683
  try {
@@ -688,7 +702,8 @@ registerToolIfEnabled("process_document", {
688
702
  // extractText, outputFormat, etc.) reach the tool instead of being stripped.
689
703
  options: z.object({}).passthrough().optional().describe("Additional processing options (maxPages, pageRange:{start,end}, extractText, extractMetadata, outputFormat, ...)"),
690
704
  ...COMPLIANCE_PARAMS,
691
- ...MAX_INLINE_CHARS_PARAM
705
+ ...MAX_INLINE_CHARS_PARAM,
706
+ ...REDACT_PII_PARAM
692
707
  }
693
708
  }, withAuth("process_document", async (params) => {
694
709
  try {
@@ -854,7 +869,8 @@ if (toolFilter.isEnabled("batch_scrape")) {
854
869
  tags: z.array(z.string()).default([])
855
870
  }).optional().describe("Job management options for async processing"),
856
871
  ...COMPLIANCE_PARAMS,
857
- ...MAX_INLINE_CHARS_PARAM
872
+ ...MAX_INLINE_CHARS_PARAM,
873
+ ...REDACT_PII_PARAM
858
874
  },
859
875
  execution: TASK_EXECUTION
860
876
  }, makeTaskToolHandler({
@@ -984,7 +1000,8 @@ registerToolIfEnabled("scrape_with_actions", {
984
1000
  maxRetries: z.number().min(0).max(3).default(1).describe("Maximum retry attempts on failure"),
985
1001
  screenshotOnError: z.boolean().default(true).describe("Capture screenshot when an error occurs"),
986
1002
  respect_robots: COMPLIANCE_PARAMS.respect_robots,
987
- ...MAX_INLINE_CHARS_PARAM
1003
+ ...MAX_INLINE_CHARS_PARAM,
1004
+ ...REDACT_PII_PARAM
988
1005
  }
989
1006
  }, withAuth("scrape_with_actions", async (params) => {
990
1007
  try {
@@ -1091,7 +1108,8 @@ registerToolIfEnabled("scrape", {
1091
1108
  inputSchema: {
1092
1109
  ...SCRAPE_INPUT_SHAPE,
1093
1110
  ...COMPLIANCE_PARAMS,
1094
- ...MAX_INLINE_CHARS_PARAM
1111
+ ...MAX_INLINE_CHARS_PARAM,
1112
+ ...REDACT_PII_PARAM
1095
1113
  },
1096
1114
  outputSchema: OUTPUT_SCHEMAS.scrape
1097
1115
  }, withAuth("scrape", async (params) => {
@@ -1409,7 +1427,8 @@ registerToolIfEnabled("stealth_mode", {
1409
1427
  wait_for: z.number().min(0).max(30000).optional().describe("Extra wait after page load, in ms — for content that renders after DOMContentLoaded"),
1410
1428
  verbose: z.boolean().optional().default(false).describe("Return the full generated fingerprint from create_context instead of a summary"),
1411
1429
  respect_robots: COMPLIANCE_PARAMS.respect_robots,
1412
- ...MAX_INLINE_CHARS_PARAM
1430
+ ...MAX_INLINE_CHARS_PARAM,
1431
+ ...REDACT_PII_PARAM
1413
1432
  }
1414
1433
  }, withAuth("stealth_mode", async ({ operation, stealthConfig, contextId, urlToTest, url, formats, wait_for, verbose, engine, respect_robots }) => {
1415
1434
  try {
@@ -14,6 +14,8 @@ import { logger } from '../utils/Logger.js';
14
14
  import { maskSecrets } from '../utils/secretMask.js';
15
15
  import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
16
16
  import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
17
+ import { searchQueryCount } from '../tools/search/batchSearch.js';
18
+ import { redactionSurcharge, REDACT_PII_MODEL_CREDITS } from '../server/redaction.js';
17
19
  // Stamped on every usage report so support can tell which client version
18
20
  // made a call; it read '3.0.3' from 3.0.3 to 5.6.10 (the website stores it
19
21
  // per record from its Phase 1.3 on).
@@ -626,6 +628,11 @@ class AuthManager {
626
628
  return costs.search_web;
627
629
  }
628
630
 
631
+ // redact_pii's regex pass is free; its opt-in model pass adds 3 once per
632
+ // call, on every tool that offers the param. The rule lives with the
633
+ // schema that declares it (src/server/redaction.js).
634
+ const redaction = redactionSurcharge(tool, params);
635
+
629
636
  // The query-scoped formats (highlights, question) add 1 once per call,
630
637
  // and mode:"model" adds 3 once; escalate:true adds the stealth browser's
631
638
  // own 5, since that is the ceiling the call may reach. The base price in
@@ -633,10 +640,17 @@ class AuthManager {
633
640
  // that declare the params.
634
641
  if (tool === 'scrape') {
635
642
  const { query, model } = scrapeFormatSurcharge(params?.formats);
636
- return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate);
643
+ return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
644
+ }
645
+
646
+ // search_web's batch form (5.1) runs one backend search per query, so it
647
+ // is priced as that many search_web calls. The count rule lives with the
648
+ // schema that declares `queries` (src/tools/search/batchSearch.js).
649
+ if (tool === 'search_web') {
650
+ return costs.search_web * searchQueryCount(params?.queries) + redaction;
637
651
  }
638
652
 
639
- return costs[tool] ?? 1;
653
+ return (costs[tool] ?? 1) + redaction;
640
654
  }
641
655
 
642
656
  /**
@@ -677,6 +691,13 @@ class AuthManager {
677
691
  note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
678
692
  break;
679
693
  }
694
+ case 'search_web': {
695
+ const queryCount = searchQueryCount(params?.queries);
696
+ note = queryCount > 1
697
+ ? `5 per query, ${queryCount} queries — exactly what ${queryCount} separate calls cost. A query that fails before it reaches a backend is not billed.`
698
+ : 'Fixed cost per invocation.';
699
+ break;
700
+ }
680
701
  case 'extract_with_llm':
681
702
  note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
682
703
  break;
@@ -708,6 +729,12 @@ class AuthManager {
708
729
  note = 'Fixed cost per invocation.';
709
730
  }
710
731
 
732
+ // redact_pii applies to nine different tools, so its note is appended
733
+ // rather than owned by any one case.
734
+ if (redactionSurcharge(toolName, params) > 0) {
735
+ note += ` redact_pii mode:"model" adds ${REDACT_PII_MODEL_CREDITS} once for the PERSON/LOCATION pass; the charge drops back when no LLM route answered.`;
736
+ }
737
+
711
738
  return { projected, note };
712
739
  }
713
740
 
@@ -42,6 +42,16 @@ const resultHandleShape = {
42
42
  expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
43
43
  };
44
44
 
45
+ // ── redact_pii (Phase 5, 5.3) ───────────────────────────────────────────────
46
+
47
+ // What the redaction stage reports back on every tool that offers redact_pii.
48
+ const redactionShape = z.object({
49
+ entities: z.record(z.number()).optional().describe('How many spans were replaced, by entity class; a class with no hits is omitted'),
50
+ count: z.number().optional().describe('Total spans replaced'),
51
+ mode: z.enum(['fast', 'model']).optional().describe('"fast" is the free regex pass; "model" added an Ollama NER pass for PERSON and LOCATION'),
52
+ model_ran: z.boolean().optional().describe('mode "model" only: whether a model actually answered. False means no LLM route existed and the model surcharge was not charged')
53
+ }).passthrough().optional().describe('Present when redact_pii was set: what was redacted from the text of this result');
54
+
45
55
  // ── scrape ──────────────────────────────────────────────────────────────────
46
56
 
47
57
  const scrapeLinkShape = z.object({
@@ -131,6 +141,7 @@ const scrapeShape = {
131
141
  }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
132
142
  content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
133
143
  warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
144
+ redaction: redactionShape,
134
145
  ...resultHandleShape,
135
146
  _cost: costShape
136
147
  };
@@ -272,8 +283,20 @@ const searchWebResultShape = z.object({
272
283
  metadata: z.record(z.unknown()).optional()
273
284
  }).passthrough();
274
285
 
286
+ // The batch form (5.1): one entry per query, in the order they were given.
287
+ // Each carries the same fields a single-query call returns, plus the query
288
+ // itself — or `error` when that one query failed and the others did not.
289
+ const searchWebBatchEntryShape = z.object({
290
+ query: z.string().optional(),
291
+ error: z.string().optional().describe('Present when this query failed; the other queries in the batch are unaffected'),
292
+ results: z.array(searchWebResultShape).optional()
293
+ }).passthrough();
294
+
275
295
  const searchWebShape = {
276
296
  query: z.string().optional(),
297
+ queries: z.array(z.string()).optional().describe('Batch form: the queries that ran, in order'),
298
+ count: z.number().optional().describe('Batch form: how many queries ran'),
299
+ results_by_query: z.array(searchWebBatchEntryShape).optional().describe('Batch form: one entry per query, in order'),
277
300
  effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
278
301
  expanded_queries: z.array(z.string()).optional(),
279
302
  results: z.array(searchWebResultShape).optional(),
@@ -302,6 +325,7 @@ const searchWebShape = {
302
325
  query_expansion: z.record(z.unknown()).nullable().optional(),
303
326
  localization_applied: z.boolean().optional()
304
327
  }).passthrough().optional(),
328
+ redaction: redactionShape,
305
329
  _cost: costShape
306
330
  };
307
331
 
@@ -380,6 +404,7 @@ const crawlDeepShape = {
380
404
  }).passthrough().optional(),
381
405
  crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
382
406
  cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
407
+ redaction: redactionShape,
383
408
  ...resultHandleShape,
384
409
  _cost: costShape
385
410
  };
@@ -135,6 +135,11 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
135
135
  shaped[key] = value;
136
136
  }
137
137
  }
138
+ // redact_pii's report is an object, so the scalar filter above drops it —
139
+ // and the caller was charged for it. Carry it through like warnings.
140
+ if (resultObject.redaction && typeof resultObject.redaction === 'object') {
141
+ shaped.redaction = resultObject.redaction;
142
+ }
138
143
  Object.assign(shaped, {
139
144
  preview,
140
145
  result_handle: handle,
@@ -0,0 +1,469 @@
1
+ /**
2
+ * redaction — the one declaration (G5) of `redact_pii`: the parameter, the
3
+ * entity and style enums, the per-tool text paths, the credit rule and the
4
+ * stage that applies it (Phase 5, 5.3).
5
+ *
6
+ * Nothing in this server scrubbed personal data out of the page text it hands
7
+ * back. `secretMask.js` covered API keys and tokens in our own LOG lines and
8
+ * nothing else, so a support inbox, a staff phone list or a checkout form's
9
+ * card number travelled straight into the caller's context window. `redact_pii`
10
+ * closes that: opt-in, off by default, and priced at nothing for the regex pass.
11
+ *
12
+ * Applied by withAuth after a successful handler run and BEFORE the
13
+ * inline-threshold stage stores the result. That order is the whole point: a
14
+ * result stored unredacted would be served back verbatim by `read_result`,
15
+ * which is exactly the hole this exists to close.
16
+ *
17
+ * Two passes:
18
+ *
19
+ * fast (default, 0 credits) — `redactPii` from crawlforge-extractors: pure
20
+ * regex over EMAIL, PHONE, FINANCIAL and SECRET, deterministic, no network.
21
+ * model (opt-in, +3 credits once per call) — an Ollama-first NER pass for
22
+ * PERSON and LOCATION, which no regex can decide. The regex pass ALWAYS
23
+ * runs first and the model only adds to it. G3: the model identifies
24
+ * spans, this module replaces them, and a span the model invented — one
25
+ * that is not present verbatim in the input — is discarded. The text it
26
+ * reads is fenced by `fenceUntrusted` and is already regex-redacted, so
27
+ * the fence never wraps unredacted text.
28
+ *
29
+ * An `entities` name the chosen mode cannot serve is a validation error, not
30
+ * a silent omission — see `entityNameIssue`. Asking for a class and being
31
+ * handed a page that still contains it is the failure this feature exists to
32
+ * prevent, so the schema refuses before anything is fetched, at no charge.
33
+ *
34
+ * The model pass is bounded by wall clock and by cost, not by taste: it runs
35
+ * once per TEXT, not once per call, so a 50-URL batch_scrape left unbounded
36
+ * would outlive any client's patience and — where the SamplingClient chain
37
+ * reaches a server-side API key rather than local Ollama — spend real money
38
+ * against a single +3 surcharge. Three caps, so the exposure of one call is a
39
+ * fixed number: at most MODEL_TEXT_CHARS of any one text, MODEL_BUDGET_CHARS
40
+ * across the call, and MODEL_MAX_PASSES completions. A warning says when any
41
+ * of them bit.
42
+ *
43
+ * Known limit, stated rather than hidden: `scrape`'s own query-scoped formats
44
+ * (`highlights`/`question` with `mode: "model"`, Phase 1) run their model step
45
+ * INSIDE the handler, before this stage, so those prompts see the page text as
46
+ * fetched. Everything that leaves the server is redacted; that one internal
47
+ * prompt is not. Closing it means redacting inside unifiedScrape as well,
48
+ * which would put the application logic in two places.
49
+ */
50
+
51
+ import { z } from 'zod';
52
+ import {
53
+ redactPii,
54
+ REGEX_ENTITIES,
55
+ MODEL_ONLY_ENTITIES,
56
+ DEFAULT_REPLACE_STYLE
57
+ } from 'crawlforge-extractors';
58
+ import { fenceUntrusted } from '../utils/untrustedContent.js';
59
+
60
+ /** What the model-backed PERSON/LOCATION pass adds to a call's price. */
61
+ export const REDACT_PII_MODEL_CREDITS = 3;
62
+
63
+ /** How much of any one text the model pass reads (summarizeContent's cap). */
64
+ const MODEL_TEXT_CHARS = 12000;
65
+
66
+ /** How much text the model pass reads across a whole call. */
67
+ const MODEL_BUDGET_CHARS = 60000;
68
+
69
+ /**
70
+ * How many completions the model pass may make in one call. Derived, not
71
+ * chosen: it is what the character budget already allows at full size, so a
72
+ * result made of many SHORT texts (a 50-URL batch_scrape) cannot turn one
73
+ * +3 surcharge into fifty round-trips.
74
+ */
75
+ const MODEL_MAX_PASSES = MODEL_BUDGET_CHARS / MODEL_TEXT_CHARS;
76
+
77
+ /** Every entity name the public parameter accepts. */
78
+ export const REDACT_PII_ENTITIES = Object.freeze([...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES]);
79
+
80
+ /** The tag names this module writes, so a model can never re-redact one. */
81
+ const TAG_NAMES = new Set([...REDACT_PII_ENTITIES, 'REDACTED']);
82
+
83
+ /**
84
+ * Why an entity name this mode cannot serve is REJECTED rather than dropped.
85
+ *
86
+ * `redactPii` drops a name it does not handle in silence, and it is right to:
87
+ * a library has no channel to tell a caller anything. A tool does. Under a
88
+ * lenient rule `entities: ["EMAIL", "PERSON"]` with the default `fast` mode
89
+ * returns a page with every personal name still in it, plus a redaction
90
+ * report listing only the emails — a partial version of the exact failure
91
+ * this feature exists to prevent, with no signal that half the request was
92
+ * ignored. So the schema refuses it, before anything is fetched and at no
93
+ * charge, and says which of the two things went wrong.
94
+ *
95
+ * @param {string} name as the caller wrote it
96
+ * @param {unknown} mode the call's `mode`
97
+ * @returns {string|null} the validation message, or null when the name is fine
98
+ */
99
+ function entityNameIssue(name, mode) {
100
+ const upper = typeof name === 'string' ? name.toUpperCase() : '';
101
+ if (REGEX_ENTITIES.includes(upper)) return null;
102
+ if (MODEL_ONLY_ENTITIES.includes(upper)) {
103
+ if (mode === 'model') return null;
104
+ return `${MODEL_ONLY_ENTITIES.join(' and ')} need mode: "model" — no regex can decide them. Add mode:"model" (+${REDACT_PII_MODEL_CREDITS} credits once per call) or drop "${name}"`;
105
+ }
106
+ return `unknown entity name "${name}" — expected one of ${REGEX_ENTITIES.join(', ')}, or ${MODEL_ONLY_ENTITIES.join('/')} with mode:"model"`;
107
+ }
108
+
109
+ /** The object form of `redact_pii`; names are matched case-insensitively. */
110
+ const RedactPiiOptionsSchema = z.object({
111
+ entities: z.array(z.string()).optional().describe(`Which classes to redact, case-insensitive: ${REGEX_ENTITIES.join(', ')}, plus ${MODEL_ONLY_ENTITIES.join(' and ')} when mode is "model". Omitted or empty means all four regex classes (and both model classes in "model" mode). An unknown name, or a model-only name without mode:"model", is rejected`),
112
+ replace_style: z.enum(['tag', 'mask', 'remove']).optional().describe('"tag" (default) writes <EMAIL>, "mask" writes [REDACTED], "remove" deletes the value'),
113
+ mode: z.enum(['fast', 'model']).optional().describe('"fast" (default) is regex only and free; "model" adds an Ollama NER pass for PERSON and LOCATION (+3 credits once per call)')
114
+ }).superRefine((value, ctx) => {
115
+ if (!Array.isArray(value.entities)) return;
116
+ value.entities.forEach((name, index) => {
117
+ const message = entityNameIssue(name, value.mode);
118
+ if (message) ctx.addIssue({ code: z.ZodIssueCode.custom, path: ['entities', index], message });
119
+ });
120
+ });
121
+
122
+ /**
123
+ * The public parameter, with the text the client sees. Spread into the input
124
+ * schema of each tool in REDACTION_TOOLS.
125
+ */
126
+ export const REDACT_PII_PARAM = {
127
+ redact_pii: z.union([
128
+ z.boolean(),
129
+ RedactPiiOptionsSchema
130
+ ]).optional().describe('Redact personal data from the text this call returns, before it reaches your context window. true means the free regex pass over EMAIL, PHONE, FINANCIAL and SECRET. The result carries redaction:{entities,count}. Default: off')
131
+ };
132
+
133
+ /**
134
+ * Where each tool keeps the text a redaction must cover. `[]` walks an array.
135
+ * Every path listed is redacted, not just the first one that matches — a
136
+ * single unredacted copy (extract_content's `readability.textContent`, say)
137
+ * would defeat the whole feature.
138
+ */
139
+ export const REDACTION_TOOLS = Object.freeze({
140
+ scrape: [
141
+ 'content.markdown', 'content.text', 'content.html', 'content.rawHtml',
142
+ 'content.highlights[].text', 'content.answer.text', 'content.answer.evidence[].text'
143
+ ],
144
+ extract_content: [
145
+ 'content.text', 'content.html', 'content.markdown', 'content.cleanedHTML',
146
+ 'readability.content', 'readability.textContent', 'readability.excerpt'
147
+ ],
148
+ extract_text: ['text', 'markdown'],
149
+ batch_scrape: ['results[].content.text', 'results[].content.html', 'results[].content.markdown'],
150
+ crawl_deep: ['results[].content'],
151
+ stealth_mode: ['content.text', 'content.html', 'content.markdown'],
152
+ scrape_with_actions: ['content.text', 'content.html', 'content.markdown'],
153
+ process_document: ['content.text', 'content.html', 'content.extractedContent'],
154
+ search_web: [
155
+ 'results[].snippet', 'results[].htmlSnippet',
156
+ 'results_by_query[].results[].snippet', 'results_by_query[].results[].htmlSnippet'
157
+ ]
158
+ });
159
+
160
+ /**
161
+ * Normalise the raw `redact_pii` param into what the two passes need, or null
162
+ * when redaction is off. Reads the RAW value — getToolCost runs before
163
+ * validation and this is the same declaration it prices from — so anything
164
+ * that is neither `true` nor a plain object means off.
165
+ *
166
+ * An absent or empty `entities` means everything the chosen mode can do. An
167
+ * explicit list is honoured exactly: names a regex can decide go to
168
+ * `redactPii`, and PERSON/LOCATION go to the model pass.
169
+ *
170
+ * This is the TOLERANT layer, deliberately. `RedactPiiOptionsSchema` has
171
+ * already rejected an unknown name, and a model-only name outside `model`
172
+ * mode, at no charge — but this function also runs on unvalidated params
173
+ * (getToolCost, and a REST proxy request that arrives pre-validated by the
174
+ * website), so it must resolve rather than throw. Where the strict schema
175
+ * would have refused, this resolves to a selection that redacts nothing by
176
+ * regex, and `applyRedaction` honours that rather than falling back to all.
177
+ *
178
+ * @param {unknown} raw
179
+ * @returns {{ entities: string[], modelEntities: string[], replaceStyle: string, mode: 'fast'|'model' } | null}
180
+ */
181
+ export function resolveRedaction(raw) {
182
+ if (raw !== true && (raw === null || typeof raw !== 'object' || Array.isArray(raw))) return null;
183
+ const options = raw === true ? {} : raw;
184
+
185
+ const mode = options.mode === 'model' ? 'model' : 'fast';
186
+ const replaceStyle = ['tag', 'mask', 'remove'].includes(options.replace_style)
187
+ ? options.replace_style
188
+ : DEFAULT_REPLACE_STYLE;
189
+
190
+ const asked = Array.isArray(options.entities) && options.entities.length > 0
191
+ ? options.entities.filter((name) => typeof name === 'string').map((name) => name.toUpperCase())
192
+ : [...REGEX_ENTITIES, ...MODEL_ONLY_ENTITIES];
193
+
194
+ return {
195
+ entities: REGEX_ENTITIES.filter((name) => asked.includes(name)),
196
+ modelEntities: mode === 'model' ? MODEL_ONLY_ENTITIES.filter((name) => asked.includes(name)) : [],
197
+ replaceStyle,
198
+ mode
199
+ };
200
+ }
201
+
202
+ /**
203
+ * What redaction adds to a tool's price: nothing for the regex pass, and the
204
+ * model pass's own 3 once per call. Charged once rather than per page for the
205
+ * same reason `batch_scrape` and `crawl_deep` are priced flat — the page count
206
+ * is not knowable before the call, and a projection scaled by `max_pages`
207
+ * would have the credit check reject an ordinary crawl. Projected is the
208
+ * ceiling (G4); withAuth drops it when no model completed.
209
+ *
210
+ * @param {string} toolName
211
+ * @param {object} [params] the call's raw params
212
+ * @returns {0 | 3}
213
+ */
214
+ export function redactionSurcharge(toolName, params) {
215
+ if (!(toolName in REDACTION_TOOLS)) return 0;
216
+ const resolved = resolveRedaction(params?.redact_pii);
217
+ return resolved && resolved.modelEntities.length > 0 ? REDACT_PII_MODEL_CREDITS : 0;
218
+ }
219
+
220
+ // ── Text routing ─────────────────────────────────────────────────────────────
221
+
222
+ /** "results[].content.text" -> ["results", "[]", "content", "text"]. */
223
+ function segmentsOf(path) {
224
+ const segments = [];
225
+ for (const part of path.split('.')) {
226
+ if (part.endsWith('[]')) {
227
+ segments.push(part.slice(0, -2), '[]');
228
+ } else {
229
+ segments.push(part);
230
+ }
231
+ }
232
+ return segments;
233
+ }
234
+
235
+ /** Every string the path reaches, each with the setter that writes it back. */
236
+ function collectAt(node, segments, found) {
237
+ if (node === null || typeof node !== 'object') return;
238
+ const [head, ...rest] = segments;
239
+
240
+ if (head === '[]') {
241
+ if (!Array.isArray(node)) return;
242
+ for (let i = 0; i < node.length; i++) {
243
+ if (rest.length === 0) {
244
+ if (typeof node[i] === 'string') found.push({ text: node[i], set: (v) => { node[i] = v; } });
245
+ } else {
246
+ collectAt(node[i], rest, found);
247
+ }
248
+ }
249
+ return;
250
+ }
251
+
252
+ if (Array.isArray(node)) return;
253
+ if (rest.length === 0) {
254
+ if (typeof node[head] === 'string') found.push({ text: node[head], set: (v) => { node[head] = v; } });
255
+ return;
256
+ }
257
+ collectAt(node[head], rest, found);
258
+ }
259
+
260
+ /**
261
+ * @param {object} resultObject
262
+ * @param {string[]} paths
263
+ * @returns {Array<{ text: string, set: (value: string) => void }>}
264
+ */
265
+ export function collectTexts(resultObject, paths) {
266
+ const found = [];
267
+ for (const path of paths) collectAt(resultObject, segmentsOf(path), found);
268
+ return found;
269
+ }
270
+
271
+ // ── The model pass ───────────────────────────────────────────────────────────
272
+
273
+ /**
274
+ * Parse the model's reply into the spans it claims. A line the model wrote
275
+ * that is not present verbatim in the text is dropped: the model identifies,
276
+ * it never rewrites (G3).
277
+ *
278
+ * @param {string} reply
279
+ * @param {string} text the text the model read
280
+ * @param {string[]} wanted the entity names asked for
281
+ * @returns {Array<{ start: number, end: number, entity: string }>}
282
+ */
283
+ export function spansFromReply(reply, text, wanted) {
284
+ if (typeof reply !== 'string') return [];
285
+ const spans = [];
286
+ for (const line of reply.split('\n')) {
287
+ const match = /^\s*(?:[-*]\s*)?(PERSON|LOCATION)\s*[:\-]\s*(.+?)\s*$/i.exec(line);
288
+ if (!match) continue;
289
+ const entity = match[1].toUpperCase();
290
+ if (!wanted.includes(entity)) continue;
291
+ const value = match[2].replace(/^["'`]|["'`]$/g, '').trim();
292
+ if (value.length < 2 || TAG_NAMES.has(value.toUpperCase())) continue;
293
+
294
+ let from = 0;
295
+ for (;;) {
296
+ const at = text.indexOf(value, from);
297
+ if (at === -1) break;
298
+ spans.push({ start: at, end: at + value.length, entity });
299
+ from = at + value.length;
300
+ }
301
+ }
302
+ // Longest first, so an overlap resolves in favour of the fuller name.
303
+ spans.sort((a, b) => (b.end - b.start) - (a.end - a.start) || a.start - b.start);
304
+ const accepted = [];
305
+ for (const span of spans) {
306
+ if (accepted.some((other) => span.start < other.end && other.start < span.end)) continue;
307
+ accepted.push(span);
308
+ }
309
+ return accepted.sort((a, b) => a.start - b.start);
310
+ }
311
+
312
+ function replacementFor(entity, style) {
313
+ if (style === 'mask') return '[REDACTED]';
314
+ if (style === 'remove') return '';
315
+ return `<${entity}>`;
316
+ }
317
+
318
+ /** Rebuild `text` with the accepted spans replaced; counts them by entity. */
319
+ function applySpans(text, spans, style, counts) {
320
+ let out = '';
321
+ let cursor = 0;
322
+ for (const span of spans) {
323
+ out += text.slice(cursor, span.start) + replacementFor(span.entity, style);
324
+ counts[span.entity] = (counts[span.entity] || 0) + 1;
325
+ cursor = span.end;
326
+ }
327
+ return out + text.slice(cursor);
328
+ }
329
+
330
+ const PROMPT_SYSTEM = 'You mark personal names and place names in text quoted from a web page. ' +
331
+ 'Reply with one entry per line, each "PERSON: <text>" or "LOCATION: <text>", copying the text exactly as it appears. ' +
332
+ 'Do not translate, shorten or correct it. If there are none, reply NONE. No other output.';
333
+
334
+ /**
335
+ * One completion for one text. Returns the reply, or throws — the caller
336
+ * treats a throw as "no LLM route", drops the surcharge and warns.
337
+ * @param {(prompt: string, options: object) => Promise<{ text: string }>} complete
338
+ */
339
+ async function askModel(complete, text, wanted) {
340
+ const { text: reply } = await complete(
341
+ `${fenceUntrusted(text, 'page text')}\nList every ${wanted.join(' and ')} in the text above.`,
342
+ { maxTokens: 512, systemPrompt: PROMPT_SYSTEM }
343
+ );
344
+ return reply;
345
+ }
346
+
347
+ // ── The stage ────────────────────────────────────────────────────────────────
348
+
349
+ /**
350
+ * Redact a tool result in place.
351
+ *
352
+ * @param {string} toolName
353
+ * @param {object} resultObject the parsed JSON result; mutated
354
+ * @param {object} params the call's params
355
+ * @param {{ complete?: (prompt: string, options: object) => Promise<{ text: string }> }} deps
356
+ * @returns {Promise<{ redacted: boolean, modelRan: boolean }>}
357
+ */
358
+ export async function applyRedaction(toolName, resultObject, params, { complete } = {}) {
359
+ const paths = REDACTION_TOOLS[toolName];
360
+ const resolved = resolveRedaction(params?.redact_pii);
361
+ if (!paths || !resolved) return { redacted: false, modelRan: false };
362
+ if (!resultObject || typeof resultObject !== 'object' || Array.isArray(resultObject)) {
363
+ return { redacted: false, modelRan: false };
364
+ }
365
+
366
+ const targets = collectTexts(resultObject, paths);
367
+ const counts = {};
368
+ const warnings = [];
369
+
370
+ // Skip the regex pass entirely when the caller's selection left it nothing
371
+ // to do (`entities: ['PERSON']`). `redactPii` reads an EMPTY array as "all
372
+ // four", so passing one through would redact everything a caller who asked
373
+ // for one model-only class never asked to lose.
374
+ if (resolved.entities.length > 0) {
375
+ for (const target of targets) {
376
+ const { text, redaction } = redactPii(target.text, {
377
+ entities: resolved.entities,
378
+ replaceStyle: resolved.replaceStyle
379
+ });
380
+ for (const [entity, n] of Object.entries(redaction.entities)) counts[entity] = (counts[entity] || 0) + n;
381
+ // What the model pass reads, so the fence never wraps unredacted text.
382
+ target.text = text;
383
+ target.set(text);
384
+ }
385
+ }
386
+
387
+ let modelRan = false;
388
+ if (resolved.modelEntities.length > 0 && typeof complete === 'function') {
389
+ let budget = MODEL_BUDGET_CHARS;
390
+ let capped = false;
391
+ let failure = null;
392
+
393
+ let passes = 0;
394
+ for (const target of targets) {
395
+ const live = target.text; // the regex-redacted copy
396
+ if (budget <= 0 || passes >= MODEL_MAX_PASSES) { capped = true; break; }
397
+ const slice = live.slice(0, Math.min(MODEL_TEXT_CHARS, budget));
398
+ if (slice.length === 0) continue;
399
+ if (slice.length < live.length) capped = true;
400
+ budget -= slice.length;
401
+ passes++;
402
+
403
+ let reply;
404
+ try {
405
+ reply = await askModel(complete, slice, resolved.modelEntities);
406
+ } catch (error) {
407
+ failure = error;
408
+ break;
409
+ }
410
+ modelRan = true;
411
+ const spans = spansFromReply(reply, slice, resolved.modelEntities);
412
+ if (spans.length === 0) continue;
413
+ target.set(applySpans(slice, spans, resolved.replaceStyle, counts) + live.slice(slice.length));
414
+ }
415
+
416
+ if (failure && !modelRan) {
417
+ warnings.push(`redact_pii: no model answered, so ${resolved.modelEntities.join(' and ')} were not redacted (${failure.message}); the model surcharge was not charged`);
418
+ } else if (failure) {
419
+ warnings.push(`redact_pii: the model pass stopped early (${failure.message}); the text it did not reach carries the regex redactions only`);
420
+ }
421
+ if (capped) {
422
+ warnings.push(`redact_pii: the model pass read at most ${MODEL_TEXT_CHARS} characters per text, ${MODEL_BUDGET_CHARS} in total and ${MODEL_MAX_PASSES} texts; text beyond that carries the regex redactions only`);
423
+ }
424
+ }
425
+
426
+ const count = Object.values(counts).reduce((sum, n) => sum + n, 0);
427
+ resultObject.redaction = {
428
+ entities: counts,
429
+ count,
430
+ mode: resolved.mode,
431
+ ...(resolved.modelEntities.length > 0 ? { model_ran: modelRan } : {})
432
+ };
433
+
434
+ // The query-scoped formats index the markdown by offset. Redaction changes
435
+ // its length, so say so rather than let the caller trust a stale offset.
436
+ if (count > 0 && resultObject.content && (resultObject.content.highlights || resultObject.content.answer)) {
437
+ warnings.push('redact_pii: highlight and evidence offsets index the text as it was before redaction');
438
+ }
439
+
440
+ if (warnings.length > 0) {
441
+ resultObject.warnings = [
442
+ ...(Array.isArray(resultObject.warnings) ? resultObject.warnings : []),
443
+ ...warnings
444
+ ];
445
+ }
446
+
447
+ return { redacted: true, modelRan };
448
+ }
449
+
450
+ /**
451
+ * The redaction stage as withAuth uses it: a SamplingClient-backed `complete`
452
+ * built only when a model pass was actually asked for, so an ordinary call
453
+ * never loads the client.
454
+ *
455
+ * @param {string} toolName
456
+ * @param {object} resultObject
457
+ * @param {object} params
458
+ * @param {{ mcpServer?: object|null }} [deps]
459
+ */
460
+ export async function runRedactionStage(toolName, resultObject, params, { mcpServer = null } = {}) {
461
+ const resolved = resolveRedaction(params?.redact_pii);
462
+ const complete = resolved && resolved.modelEntities.length > 0
463
+ ? async (prompt, options) => {
464
+ const { SamplingClient } = await import('../core/SamplingClient.js');
465
+ return new SamplingClient({ mcpServer }).complete(prompt, options);
466
+ }
467
+ : undefined;
468
+ return applyRedaction(toolName, resultObject, params, { complete });
469
+ }
@@ -20,6 +20,7 @@ import { recordToolInvocation } from '../observability/tracing.js';
20
20
  import { isInternalRequest, preflightRefusal, reportedActualCost, requestContext } from './requestContext.js';
21
21
  import { appendFallbackHint } from './fallbackHints.js';
22
22
  import { INLINE_THRESHOLD_TOOLS, applyInlineThreshold } from './inlineThreshold.js';
23
+ import { REDACTION_TOOLS, redactionSurcharge, runRedactionStage } from './redaction.js';
23
24
  import { getResultStore } from '../core/ResultStore.js';
24
25
 
25
26
  /**
@@ -36,6 +37,24 @@ function shapeLargeResult(toolName, result, params) {
36
37
  if (result.structuredContent) result.structuredContent = shaped;
37
38
  }
38
39
 
40
+ /**
41
+ * Redact the result's page text in place (Phase 5, 5.3). Runs BEFORE
42
+ * shapeLargeResult: a result stored unredacted would be served straight back
43
+ * out by read_result, which is the hole redact_pii exists to close.
44
+ *
45
+ * @returns {Promise<boolean>} whether a model pass actually completed
46
+ */
47
+ async function redactResult(toolName, result, params, mcpServer) {
48
+ if (!result || !Array.isArray(result.content) || result.content[0]?.type !== 'text') return false;
49
+ let parsed;
50
+ try { parsed = JSON.parse(result.content[0].text); } catch { return false; }
51
+ const { redacted, modelRan } = await runRedactionStage(toolName, parsed, params, { mcpServer });
52
+ if (!redacted) return false;
53
+ result.content[0].text = JSON.stringify(parsed, null, 2);
54
+ if (result.structuredContent) result.structuredContent = parsed;
55
+ return modelRan;
56
+ }
57
+
39
58
  export function hashParams(params) {
40
59
  try {
41
60
  return createHash('sha256').update(JSON.stringify(params ?? {})).digest('hex').slice(0, 12);
@@ -50,7 +69,7 @@ export function hashParams(params) {
50
69
  * @param {object} deps.logger
51
70
  * @param {object} [deps.metrics] — optional Prometheus registry (see src/observability/metrics.js)
52
71
  */
53
- export function makeWithAuth({ authManager, logger, metrics = null }) {
72
+ export function makeWithAuth({ authManager, logger, metrics = null, mcpServer = null }) {
54
73
  return function withAuth(toolName, handler) {
55
74
  const invoke = async (params) => {
56
75
  const startTime = Date.now();
@@ -62,6 +81,13 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
62
81
  const internal = isInternalRequest();
63
82
  const billingExempt = creatorMode || internal;
64
83
  const creditCost = billingExempt ? 0 : authManager.getToolCost(toolName, params);
84
+ // redact_pii's model pass runs at this seam, AFTER the handler, so the
85
+ // handler's own setActualCost report cannot know about it. Split the
86
+ // projection: the tool prices its own work, and the model surcharge is
87
+ // added back only when a model actually completed (G4).
88
+ const redactionCost = billingExempt ? 0 : Math.min(redactionSurcharge(toolName, params), creditCost);
89
+ const toolCost = creditCost - redactionCost;
90
+ let redactionModelRan = false;
65
91
  let outcome = 'pending';
66
92
  let thrown = null;
67
93
  // Only bill the error-path half-charge once the handler has actually run.
@@ -73,7 +99,8 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
73
99
  // lower the charge, never raise it.
74
100
  const billable = () => {
75
101
  const reported = reportedActualCost();
76
- return reported == null ? creditCost : Math.min(reported, creditCost);
102
+ const spent = reported == null ? toolCost : Math.min(reported, toolCost);
103
+ return spent + (redactionModelRan ? redactionCost : 0);
77
104
  };
78
105
 
79
106
  try {
@@ -116,6 +143,19 @@ export function makeWithAuth({ authManager, logger, metrics = null }) {
116
143
  // result did real work and bills for it.
117
144
  const refused = isErrorResult && preflightRefusal() !== null;
118
145
 
146
+ // Phase 5 (5.3): redact_pii scrubs the page text this call returns.
147
+ // It runs BEFORE the inline-threshold stage below, so what gets
148
+ // stored — and what read_result later serves — is already redacted.
149
+ // Internal (website REST proxy) requests are redacted too: the text
150
+ // is the same text, and the website has no copy of it to scrub.
151
+ if (!isErrorResult && toolName in REDACTION_TOOLS) {
152
+ try {
153
+ redactionModelRan = await redactResult(toolName, result, params, mcpServer);
154
+ } catch {
155
+ /* redaction must never break the request path */
156
+ }
157
+ }
158
+
119
159
  // Phase 2: a result over max_inline_chars is stored and returned as
120
160
  // a preview plus a result_handle for read_result. Internal (website
121
161
  // REST proxy) requests are left whole — the website applies its own
@@ -0,0 +1,50 @@
1
+ /**
2
+ * batchSearch.js — the one declaration of `search_web`'s `queries` param
3
+ * (Phase 5, 5.1).
4
+ *
5
+ * `search_web` takes one query. A caller comparing five phrasings, or
6
+ * checking a term across five sites, pays five round-trips for what is one
7
+ * unit of work. `queries` accepts 1-10 of them in a single call; each still
8
+ * runs through the SAME `execute()` pipeline — expansion, cache, dedupe,
9
+ * rank — so a batch entry and a single call are the same result.
10
+ *
11
+ * The batch wrapper sits ABOVE the provider short-circuit in `execute()`, so
12
+ * the SearXNG branch (which is not part of the adapter factory) inherits
13
+ * `queries` without a second implementation.
14
+ *
15
+ * The field and the price rule live here so `server.js`, the tool module and
16
+ * `AuthManager.getToolCost` read one declaration (G5) — the same arrangement
17
+ * `escalation.js` has for `scrape`.
18
+ */
19
+
20
+ import { z } from 'zod';
21
+
22
+ /** How many queries one call may carry. */
23
+ export const MAX_SEARCH_QUERIES = 10;
24
+
25
+ /** search_web's price for ONE query, as AuthManager's table spells it. */
26
+ export const SEARCH_WEB_CREDITS = 5;
27
+
28
+ /** The public field, with the text the client sees. */
29
+ export const SEARCH_QUERIES_PARAM = {
30
+ queries: z.array(z.string().min(1)).min(1).max(MAX_SEARCH_QUERIES).optional().describe(`Run 1-${MAX_SEARCH_QUERIES} searches in one call instead of ${MAX_SEARCH_QUERIES} round-trips; every other parameter applies to each. Results come back in results_by_query, one entry per query, in order. Costs 5 per query. Use this OR query, not both`)
31
+ };
32
+
33
+ /** Rejected when a call carries both `query` and `queries`, or neither. */
34
+ export const EXACTLY_ONE_QUERY_MESSAGE =
35
+ `search_web needs exactly one of "query" (a single search) or "queries" (1-${MAX_SEARCH_QUERIES} searches in one call), not both and not neither`;
36
+
37
+ /**
38
+ * How many searches this call will run, for pricing. Reads the RAW param —
39
+ * getToolCost runs before validation — so anything that is not a non-empty
40
+ * array prices as the single-query form, and a caller who sends more than the
41
+ * schema allows never produces a silly projection (the schema rejects that
42
+ * call before it is billed at all).
43
+ *
44
+ * @param {unknown} queries the raw `queries` param
45
+ * @returns {number} 1..MAX_SEARCH_QUERIES
46
+ */
47
+ export function searchQueryCount(queries) {
48
+ if (!Array.isArray(queries) || queries.length === 0) return 1;
49
+ return Math.min(queries.length, MAX_SEARCH_QUERIES);
50
+ }
@@ -8,6 +8,10 @@ import { SearchResultCache } from './ranking/SearchResultCache.js';
8
8
  import LocalizationManager from '../../core/LocalizationManager.js';
9
9
  import { isCreatorModeVerified } from '../../core/creatorMode.js';
10
10
  import { searchViaSearxng } from './providers/searxng.js';
11
+ import { MAX_SEARCH_QUERIES, SEARCH_WEB_CREDITS, EXACTLY_ONE_QUERY_MESSAGE } from './batchSearch.js';
12
+ import { setActualCost } from '../../server/requestContext.js';
13
+
14
+ const BatchQueriesSchema = z.array(z.string().min(1)).min(1).max(MAX_SEARCH_QUERIES);
11
15
 
12
16
  const SearchWebSchema = z.object({
13
17
  query: z.string().min(1),
@@ -126,7 +130,47 @@ export class SearchWebTool {
126
130
  });
127
131
  }
128
132
 
133
+ /**
134
+ * Run 1-10 queries through the single-query pipeline and key the results by
135
+ * query (5.1). Sequential: each entry is a separate billed backend search,
136
+ * and firing ten at once at the provider buys nothing but rate limits.
137
+ *
138
+ * @param {object} params the call's params, `queries` included
139
+ * @returns {Promise<{queries: string[], count: number, results_by_query: object[]}>}
140
+ */
141
+ async _executeBatch(params) {
142
+ const { queries, query: _ignored, ...rest } = params;
143
+ const parsed = BatchQueriesSchema.parse(queries);
144
+
145
+ const results_by_query = [];
146
+ let ran = 0;
147
+ for (const query of parsed) {
148
+ try {
149
+ results_by_query.push({ query, ...await this.execute({ ...rest, query }) });
150
+ ran++;
151
+ } catch (error) {
152
+ // One failed query does not sink the other nine.
153
+ results_by_query.push({ query, error: error.message });
154
+ }
155
+ }
156
+
157
+ // The projection is 5 per query; a query that failed before it reached a
158
+ // backend did no work and must not be billed for one (G4). The projection
159
+ // is the ceiling, so this only ever lowers the charge.
160
+ setActualCost(SEARCH_WEB_CREDITS * ran);
161
+
162
+ return { queries: parsed, count: parsed.length, results_by_query };
163
+ }
164
+
129
165
  async execute(params) {
166
+ // Batch form (5.1). Handled above the provider short-circuit below, so
167
+ // the SearXNG branch runs each query through the same path the default
168
+ // provider does.
169
+ const hasQuery = params?.query !== undefined;
170
+ const hasQueries = params?.queries !== undefined;
171
+ if (hasQuery === hasQueries) throw new Error(EXACTLY_ONE_QUERY_MESSAGE);
172
+ if (hasQueries) return await this._executeBatch(params);
173
+
130
174
  try {
131
175
  const validated = SearchWebSchema.parse(params);
132
176
 
@@ -9,6 +9,8 @@
9
9
  * logger.error('fetch failed', maskSecrets({ apiKey, url, error }));
10
10
  */
11
11
 
12
+ import { redactPii } from 'crawlforge-extractors';
13
+
12
14
  const SECRET_KEYS_RE = /api[_-]?key|apikey|x-api-key|password|passwd|secret|token|authorization|auth|credential|login|formautofill|private[_-]?key|access[_-]?key|proxy_url|proxyurl|cookie/i;
13
15
 
14
16
  const MASK = '[REDACTED]';
@@ -97,16 +99,18 @@ export function maskError(error) {
97
99
 
98
100
  /**
99
101
  * Heuristic: redact strings that look like API keys / tokens embedded in text.
102
+ *
103
+ * The patterns moved to crawlforge-extractors' `redactPii` (Phase 5, 5.3), so
104
+ * the two surfaces run one implementation and page text gets the same
105
+ * treatment log lines already got. The SECRET class there is this heuristic,
106
+ * ported verbatim and asserted byte-for-byte against it upstream: the label
107
+ * survives and only the value is replaced, which is what keeps an error
108
+ * message saying WHICH credential the request carried.
109
+ *
100
110
  * @param {string} str
101
111
  * @returns {string}
102
112
  */
103
113
  function redactSecretsFromString(str) {
104
114
  if (typeof str !== 'string') return str;
105
- return str
106
- .replace(/(Bearer\s+)\S+/gi, `$1${MASK}`)
107
- .replace(/(api[_-]?key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
108
- .replace(/(x-api-key\s*[:=]\s*)\S+/gi, `$1${MASK}`)
109
- .replace(/(password\s*[:=]\s*)\S+/gi, `$1${MASK}`)
110
- .replace(/(secret\s*[:=]\s*)\S+/gi, `$1${MASK}`)
111
- .replace(/(token\s*[:=]\s*)\S+/gi, `$1${MASK}`);
115
+ return redactPii(str, { entities: ['SECRET'], replaceStyle: 'mask' }).text;
112
116
  }