crawlforge-mcp-server 5.9.0 → 6.0.0

package/src/core/ActionExecutor.js

@@ -166,7 +166,7 @@ const ActionChainSchema = z.object({
   continueOnError: z.boolean().default(false),
   timeout: z.number().min(1000).max(300000).default(30000),
   retryChain: z.number().min(0).max(3).default(0),
-  metadata: z.record(z.any()).default({})
+  metadata: z.record(z.any()).prefault({})
 });
 
 export class ActionExecutor extends EventEmitter {

package/src/core/AuthManager.js

@@ -14,6 +14,8 @@ import { logger } from '../utils/Logger.js';
 import { maskSecrets } from '../utils/secretMask.js';
 import { scrapeFormatSurcharge } from '../tools/scrape/formats.js';
 import { scrapeEscalationSurcharge } from '../tools/scrape/escalation.js';
+import { searchQueryCount } from '../tools/search/batchSearch.js';
+import { redactionSurcharge, REDACT_PII_MODEL_CREDITS } from '../server/redaction.js';
 // Stamped on every usage report so support can tell which client version
 // made a call; it read '3.0.3' from 3.0.3 to 5.6.10 (the website stores it
 // per record from its Phase 1.3 on).
@@ -626,6 +628,11 @@ class AuthManager {
       return costs.search_web;
     }
 
+    // redact_pii's regex pass is free; its opt-in model pass adds 3 once per
+    // call, on every tool that offers the param. The rule lives with the
+    // schema that declares it (src/server/redaction.js).
+    const redaction = redactionSurcharge(tool, params);
+
     // The query-scoped formats (highlights, question) add 1 once per call,
     // and mode:"model" adds 3 once; escalate:true adds the stealth browser's
     // own 5, since that is the ceiling the call may reach. The base price in
@@ -633,10 +640,17 @@ class AuthManager {
     // that declare the params.
     if (tool === 'scrape') {
       const { query, model } = scrapeFormatSurcharge(params?.formats);
-      return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate);
+      return costs.scrape + query + model + scrapeEscalationSurcharge(params?.escalate) + redaction;
+    }
+
+    // search_web's batch form (5.1) runs one backend search per query, so it
+    // is priced as that many search_web calls. The count rule lives with the
+    // schema that declares `queries` (src/tools/search/batchSearch.js).
+    if (tool === 'search_web') {
+      return costs.search_web * searchQueryCount(params?.queries) + redaction;
     }
 
-    return costs[tool] ?? 1;
+    return (costs[tool] ?? 1) + redaction;
   }
 
   /**
@@ -677,6 +691,13 @@ class AuthManager {
         note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
         break;
       }
+      case 'search_web': {
+        const queryCount = searchQueryCount(params?.queries);
+        note = queryCount > 1
+          ? `5 per query, ${queryCount} queries — exactly what ${queryCount} separate calls cost. A query that fails before it reaches a backend is not billed.`
+          : 'Fixed cost per invocation.';
+        break;
+      }
       case 'extract_with_llm':
         note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
         break;
@@ -708,6 +729,12 @@ class AuthManager {
         note = 'Fixed cost per invocation.';
     }
 
+    // redact_pii applies to nine different tools, so its note is appended
+    // rather than owned by any one case.
+    if (redactionSurcharge(toolName, params) > 0) {
+      note += ` redact_pii mode:"model" adds ${REDACT_PII_MODEL_CREDITS} once for the PERSON/LOCATION pass; the charge drops back when no LLM route answered.`;
+    }
+
     return { projected, note };
   }
 

package/src/core/ChangeTracker.js

@@ -37,7 +37,7 @@ const ChangeTrackingSchema = z.object({
       moderate: z.number().min(0).max(1).default(0.3),
       major: z.number().min(0).max(1).default(0.7)
     }).optional()
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const ChangeComparisonSchema = z.object({

package/src/core/ElicitationHelper.js

@@ -5,8 +5,58 @@
  * expensive or ambiguous operations. Falls back gracefully when the
  * MCP client does not support elicitation.
  *
- * MCP Spec 2025-11-25: client/elicit request with requestedSchema
+ * The request goes out as a 2025-era server→client `elicitation/create`,
+ * awaited inline in the middle of a tool's work. Two things the SDK does NOT
+ * do for us, and which `supported` therefore decides before we send:
+ *
+ *   - `Server.elicitInput()` refuses a client that declared a bare
+ *     `elicitation: {}` (it demands `elicitation.form`), even though the SDK's
+ *     own capability rule counts a bare declaration as form-capable — that is
+ *     the pre-mode 2025 meaning, and it is what the SDK's legacy shim and its
+ *     2026 seam both apply. We send through `Server.request()`, which applies
+ *     that same lenient rule, so those clients get their prompt.
+ *
+ *   - Both `elicitInput()` and `request()` throw on a 2026-07-28-era
+ *     connection: that revision has no server→client request channel at all.
+ *     There is no inline substitute — the replacement is an `input_required`
+ *     result RETURNED by a tools/call handler, which this helper cannot do
+ *     from the middle of a tool's execution. So `supported` reports false and
+ *     the operation proceeds unasked, exactly as it does for a client with no
+ *     elicitation capability. A confirmation prompt is a nicety; failing the
+ *     call is not an acceptable substitute. See docs/mcp-spec-adoption.md.
+ */
+
+/**
+ * First revision of the modern protocol era. Revisions are ISO dates, so
+ * lexicographic comparison orders them chronologically (the SDK's own rule).
  */
+const FIRST_MODERN_PROTOCOL_VERSION = '2026-07-28';
+
+/** The one-boolean schema a confirmation asks with. */
+const CONFIRM_SCHEMA = {
+  type: 'object',
+  properties: {
+    confirmed: {
+      type: 'boolean',
+      title: 'Proceed?',
+      description: 'Confirm to proceed with the operation',
+    },
+  },
+  required: ['confirmed'],
+};
+
+/**
+ * Whether the client's declared capabilities cover FORM elicitation, by the
+ * SDK's own rule: `elicitation.form` counts, and so does a bare `elicitation`
+ * declaration naming neither mode (the pre-mode 2025 meaning). A client that
+ * declared only `elicitation.url` has not declared form support.
+ */
+function formElicitationDeclared(caps) {
+  const elicitation = caps?.elicitation;
+  if (!elicitation) return false;
+  if (elicitation.form !== undefined) return true;
+  return elicitation.url === undefined;
+}
 
 export class ElicitationHelper {
   /**
@@ -20,22 +70,35 @@ export class ElicitationHelper {
   }
 
   /**
-   * Whether the connected MCP client supports elicitation.
+   * Whether an inline elicitation round trip will actually reach the user.
    * @returns {boolean}
    */
   get supported() {
     const server = this._mcpServer?.server;
-    // The MCP SDK exposes elicitation via Server.elicitInput(); it is only
-    // usable when the connected CLIENT advertised the `elicitation` capability.
-    if (typeof server?.elicitInput !== 'function') return false;
+    if (typeof server?.request !== 'function') return false;
     try {
-      const caps = server.getClientCapabilities?.();
-      return !!caps?.elicitation;
+      // No server→client request channel exists on the 2026-07-28 era.
+      const negotiated = server.getNegotiatedProtocolVersion?.();
+      if (typeof negotiated === 'string' && negotiated >= FIRST_MODERN_PROTOCOL_VERSION) return false;
+      return formElicitationDeclared(server.getClientCapabilities?.());
     } catch {
       return false;
     }
   }
 
+  /**
+   * Send one form-mode `elicitation/create` and return the ElicitResult.
+   * This is byte-for-byte the message `Server.elicitInput()` sends; it just
+   * does not impose that method's stricter `elicitation.form` gate.
+   * @private
+   */
+  async _elicit(message, requestedSchema) {
+    return this._mcpServer.server.request({
+      method: 'elicitation/create',
+      params: { message, requestedSchema, mode: 'form' },
+    });
+  }
+
   /**
    * Ask for user confirmation before proceeding with an expensive operation.
    * Returns true if confirmed (or if elicitation is unsupported — fail-open
@@ -57,20 +120,7 @@ export class ElicitationHelper {
         .join('\n');
       const fullMessage = detailLines ? `${message}\n\n${detailLines}` : message;
 
-      const result = await this._mcpServer.server.elicitInput({
-        message: fullMessage,
-        requestedSchema: {
-          type: 'object',
-          properties: {
-            confirmed: {
-              type: 'boolean',
-              title: 'Proceed?',
-              description: 'Confirm to proceed with the operation',
-            },
-          },
-          required: ['confirmed'],
-        },
-      });
+      const result = await this._elicit(fullMessage, CONFIRM_SCHEMA);
 
       // Only an explicit accept + confirmed=true proceeds; decline/cancel = stop.
       return result?.action === 'accept' && result?.content?.confirmed === true;
@@ -97,23 +147,22 @@ export class ElicitationHelper {
     }
 
     try {
-      const result = await this._mcpServer.server.elicitInput({
-        message,
-        requestedSchema: {
-          type: 'object',
-          properties: {
-            [fieldName]: {
-              type: 'string',
-              title: fieldName,
-              description: fieldDescription,
-              ...(defaultValue ? { default: defaultValue } : {}),
-            },
+      const result = await this._elicit(message, {
+        type: 'object',
+        properties: {
+          [fieldName]: {
+            type: 'string',
+            title: fieldName,
+            description: fieldDescription,
+            ...(defaultValue ? { default: defaultValue } : {}),
           },
-          required: [fieldName],
         },
+        required: [fieldName],
       });
 
-      if (result?.action === 'accept' && result?.content?.[fieldName] != null) {
+      // The answer is client-supplied and no longer schema-checked by the SDK
+      // on this path, so hold it to the type we asked for.
+      if (result?.action === 'accept' && typeof result?.content?.[fieldName] === 'string') {
         return result.content[fieldName];
       }
       return defaultValue || null;

package/src/core/SamplingClient.js

@@ -7,7 +7,9 @@
  * Fallback chain (applied in resolveCompletion):
  *   1. Ollama (local, no API key needed)
  *   2. Server-side API key (OPENAI_API_KEY / ANTHROPIC_API_KEY)
- *   3. MCP sampling request to client
+ *   3. MCP sampling request to client — DEPRECATED in MCP revision 2026-07-28
+ *      (SEP-2577); removal on or after 2027-07-28. Emits a one-line stderr
+ *      deprecation notice when it serves a completion.
  *   4. Error
  */
 
@@ -157,7 +159,11 @@ export class SamplingClient {
           includeContext: 'none',
         });
         const text = samplingResult?.content?.text || '';
-        if (text) return { text, provider: 'sampling' };
+        if (text) {
+          // stderr, never stdout — stdout is the JSON-RPC stream on stdio.
+          console.error('[deprecation] MCP sampling served this completion. Sampling was deprecated in MCP revision 2026-07-28 (SEP-2577); CrawlForge removes this fallback on or after 2027-07-28. Run Ollama or set OPENAI_API_KEY / ANTHROPIC_API_KEY.');
+          return { text, provider: 'sampling' };
+        }
       } catch (_samplingErr) {
         // Sampling not supported or failed
       }

package/src/core/analysis/ContentAnalyzer.js

@@ -28,7 +28,7 @@ const ContentAnalyzerSchema = z.object({
     maxKeywords: z.number().min(1).max(50).default(15),
     includeReadabilityMetrics: z.boolean().default(true),
     includeSentiment: z.boolean().default(true)
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const AnalysisResult = z.object({

package/src/core/processing/BrowserProcessor.js

@@ -74,7 +74,7 @@ const BrowserProcessorSchema = z.object({
       enableTimezoneSpoof: z.boolean().default(true),
       enableGeoLocationSpoof: z.boolean().default(true)
     }).optional()
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const BrowserResult = z.object({

package/src/core/processing/ContentProcessor.js

@@ -20,7 +20,7 @@ const ContentProcessorSchema = z.object({
     removeBoilerplate: z.boolean().default(true),
     preserveImageInfo: z.boolean().default(true),
     extractMetadata: z.boolean().default(true)
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const ReadabilityResult = z.object({

package/src/core/processing/PDFProcessor.js

@@ -31,8 +31,8 @@ const PDFProcessorSchema = z.object({
     parseOptions: z.object({
       normalizeWhitespace: z.boolean().default(true),
       disableCombineTextItems: z.boolean().default(false)
-    }).optional().default({})
-  }).optional().default({})
+    }).optional().prefault({})
+  }).optional().prefault({})
 });
 
 const PDFResult = z.object({

package/src/schemas/toolOutputSchemas.js

@@ -42,6 +42,16 @@ const resultHandleShape = {
   expires_at: z.string().optional().describe('When the stored result is dropped (ISO 8601)')
 };
 
+// ── redact_pii (Phase 5, 5.3) ───────────────────────────────────────────────
+
+// What the redaction stage reports back on every tool that offers redact_pii.
+const redactionShape = z.object({
+  entities: z.record(z.number()).optional().describe('How many spans were replaced, by entity class; a class with no hits is omitted'),
+  count: z.number().optional().describe('Total spans replaced'),
+  mode: z.enum(['fast', 'model']).optional().describe('"fast" is the free regex pass; "model" added an Ollama NER pass for PERSON and LOCATION'),
+  model_ran: z.boolean().optional().describe('mode "model" only: whether a model actually answered. False means no LLM route existed and the model surcharge was not charged')
+}).passthrough().optional().describe('Present when redact_pii was set: what was redacted from the text of this result');
+
 // ── scrape ──────────────────────────────────────────────────────────────────
 
 const scrapeLinkShape = z.object({
@@ -131,6 +141,7 @@ const scrapeShape = {
   }).passthrough().optional().describe('Present when escalated is true: the stealth engine that ran, and the bot-defence vendor the plain fetch hit (null when the block named none)'),
   content: z.object(scrapeFormatShapes).passthrough().optional().describe('One key per requested format'),
   warnings: z.array(z.string()).optional().describe('Per-format warnings; partial success never fails the whole call'),
+  redaction: redactionShape,
   ...resultHandleShape,
   _cost: costShape
 };
@@ -272,8 +283,20 @@ const searchWebResultShape = z.object({
   metadata: z.record(z.unknown()).optional()
 }).passthrough();
 
+// The batch form (5.1): one entry per query, in the order they were given.
+// Each carries the same fields a single-query call returns, plus the query
+// itself — or `error` when that one query failed and the others did not.
+const searchWebBatchEntryShape = z.object({
+  query: z.string().optional(),
+  error: z.string().optional().describe('Present when this query failed; the other queries in the batch are unaffected'),
+  results: z.array(searchWebResultShape).optional()
+}).passthrough();
+
 const searchWebShape = {
   query: z.string().optional(),
+  queries: z.array(z.string()).optional().describe('Batch form: the queries that ran, in order'),
+  count: z.number().optional().describe('Batch form: how many queries ran'),
+  results_by_query: z.array(searchWebBatchEntryShape).optional().describe('Batch form: one entry per query, in order'),
   effective_query: z.string().optional().describe('Present when query expansion changed the query actually used'),
   expanded_queries: z.array(z.string()).optional(),
   results: z.array(searchWebResultShape).optional(),
@@ -302,6 +325,7 @@ const searchWebShape = {
     query_expansion: z.record(z.unknown()).nullable().optional(),
     localization_applied: z.boolean().optional()
   }).passthrough().optional(),
+  redaction: redactionShape,
   _cost: costShape
 };
 
@@ -380,6 +404,7 @@ const crawlDeepShape = {
   }).passthrough().optional(),
   crawled_at: z.string().optional().describe('When the pages were actually fetched (ISO 8601)'),
   cached: z.boolean().optional().describe('True when this response was replayed from an earlier crawl rather than crawled now; crawled_at gives its age'),
+  redaction: redactionShape,
   ...resultHandleShape,
   _cost: costShape
 };

package/src/server/inlineThreshold.js

@@ -135,6 +135,11 @@ export function applyInlineThreshold(toolName, resultObject, params, { store, en
       shaped[key] = value;
     }
   }
+  // redact_pii's report is an object, so the scalar filter above drops it —
+  // and the caller was charged for it. Carry it through like warnings.
+  if (resultObject.redaction && typeof resultObject.redaction === 'object') {
+    shaped.redaction = resultObject.redaction;
+  }
   Object.assign(shaped, {
     preview,
     result_handle: handle,