crawlforge-mcp-server 5.10.0 → 6.1.0

package/src/tools/advanced/batchScrape/index.js

@@ -83,41 +83,51 @@ export class BatchScrapeTool extends EventEmitter {
     this._elicitation = new ElicitationHelper({ mcpServer });
   }
 
-  async execute(params) {
+  async execute(params, ctx) {
     try {
       const validated = BatchScrapeSchema.parse(params);
-      this.stats.totalBatches++;
       const batchId = this._generateBatchId();
-      const startTime = Date.now();
-
-      this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
-
-      const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
 
-      let webhookConfig = null;
-      if (validated.webhook && this.enableWebhookNotifications) {
-        webhookConfig = this._registerWebhook(validated.webhook, batchId);
-      }
-
-      // D1.4: Elicitation — warn when batch is large in sync mode
-      if (validated.mode === 'sync' && urlConfigs.length > 25) {
-        const proceed = await this._elicitation.confirm(
-          `batch_scrape (sync mode) will fetch ${urlConfigs.length} URLs synchronously. This may take a while and consume significant credits.`,
+      // D1.4: Elicitation — warn when batch is large in sync mode. An
+      // unanswered gate returns an input-required result the SDK answers by
+      // re-entering this handler from the top, so it runs before anything that
+      // leaves a trace: below it are the batch counter and the webhook
+      // registration, which a second entry would repeat (a second webhook
+      // registration for one batch). The count comes from validated.urls,
+      // which _normalizeUrlConfigs maps one-for-one.
+      if (validated.mode === 'sync' && validated.urls.length > 25) {
+        const gate = this._elicitation.confirm(
+          ctx,
+          'batch_scrape:large_sync',
+          `batch_scrape (sync mode) will fetch ${validated.urls.length} URLs synchronously. This may take a while and consume significant credits.`,
           {
-            url_count: urlConfigs.length,
+            url_count: validated.urls.length,
             mode: 'sync',
             suggestion: 'Consider using mode:"async" for large batches.',
           }
         );
-        if (!proceed) {
+        if (gate.status === 'ask') return gate.result;
+        if (gate.status === 'cancelled') {
           return {
             batchId, mode: 'sync', success: false,
             error: 'Batch scrape cancelled by user (elicitation declined).',
-            totalUrls: urlConfigs.length,
+            totalUrls: validated.urls.length,
           };
         }
       }
 
+      this.stats.totalBatches++;
+      const startTime = Date.now();
+
+      this._log('info', `Starting batch scrape ${batchId} with ${validated.urls.length} URLs in ${validated.mode} mode`);
+
+      const urlConfigs = this._normalizeUrlConfigs(validated.urls, validated);
+
+      let webhookConfig = null;
+      if (validated.webhook && this.enableWebhookNotifications) {
+        webhookConfig = this._registerWebhook(validated.webhook, batchId);
+      }
+
       if (validated.mode === 'sync') {
         return await this._processBatchSync(batchId, urlConfigs, validated, webhookConfig, startTime);
       } else {

package/src/tools/agent/agent.js

@@ -35,16 +35,21 @@ export class AgentTool {
     this._elicitation = new ElicitationHelper({ mcpServer });
   }
 
-  async execute(params) {
+  async execute(params, ctx) {
     const validated = AgentInputSchema.parse(params);
 
-    // Request confirmation before a pro run (expensive)
+    // Request confirmation before a pro run (expensive). An unanswered gate
+    // returns an input-required result the SDK answers by re-entering this
+    // handler from the top, so nothing above it may fetch or leave a trace.
     if (validated.model === 'pro') {
-      const proceed = await this._elicitation.confirm(
+      const gate = this._elicitation.confirm(
+        ctx,
+        'agent:pro_model',
         'agent tool: pro model uses ResearchOrchestrator and may incur significant costs.',
         { model: 'pro', maxUrls: validated.maxUrls, note: 'External LLM API costs billed separately if keys are set.' }
       );
-      if (!proceed) {
+      if (gate.status === 'ask') return gate.result;
+      if (gate.status === 'cancelled') {
         return {
           success: false,
           cancelled: true,

package/src/tools/crawl/crawlDeep.js

@@ -22,7 +22,7 @@ const CrawlDeepSchema = z.object({
     dampingFactor: z.number().min(0).max(1).optional().default(0.85),
     maxIterations: z.number().min(1).max(1000).optional().default(100),
     enableCaching: z.boolean().optional().default(true)
-  }).optional().default({}),
+  }).optional().prefault({}),
   // New domain filtering options
   domain_filter: z.object({
     whitelist: z.array(z.union([
@@ -59,7 +59,7 @@ const CrawlDeepSchema = z.object({
       timeout: z.number().optional(),
       maxPages: z.number().optional(),
       concurrency: z.number().optional()
-    })).optional().default({})
+    })).optional().prefault({})
   }).optional(),
   import_filter_config: z.string().optional(), // JSON string of exported config
   // Session reuse: when enabled, all page fetches share a cookie jar and
@@ -67,11 +67,11 @@ const CrawlDeepSchema = z.object({
   session: z.object({
     enabled: z.boolean(),
     persistCookies: z.boolean().optional().default(true),
-    headers: z.record(z.string()).optional().default({}),
+    headers: z.record(z.string()).optional().prefault({}),
     initialRequest: z.object({
       url: z.string().url(),
       method: z.string().optional().default('GET'),
-      headers: z.record(z.string()).optional().default({}),
+      headers: z.record(z.string()).optional().prefault({}),
       body: z.string().optional()
     }).optional()
   }).optional()
@@ -117,7 +117,7 @@ export class CrawlDeepTool {
     this._elicitation = new ElicitationHelper({ mcpServer });
   }
 
-  async execute(params) {
+  async execute(params, ctx) {
     try {
       const validated = CrawlDeepSchema.parse(params);
 
@@ -147,9 +147,14 @@ export class CrawlDeepTool {
         if (cached) return { ...cached, cached: true };
       }
 
-      // D1.4: Elicitation — warn when max_pages is very high
+      // D1.4: Elicitation — warn when max_pages is very high. An unanswered
+      // gate returns an input-required result the SDK answers by re-entering
+      // this handler from the top; everything above is the clamp arithmetic and
+      // a cache read, so a second entry fetches nothing and leaves no trace.
       if (effectiveMaxPages > 500) {
-        const proceed = await this._elicitation.confirm(
+        const gate = this._elicitation.confirm(
+          ctx,
+          'crawl_deep:max_pages',
           `crawl_deep will crawl up to ${effectiveMaxPages} pages from ${validated.url}. Large crawls consume many credits.`,
           {
             url: validated.url,
@@ -157,7 +162,8 @@ export class CrawlDeepTool {
             max_depth: effectiveMaxDepth,
           }
         );
-        if (!proceed) {
+        if (gate.status === 'ask') return gate.result;
+        if (gate.status === 'cancelled') {
           return {
             success: false,
             error: 'Crawl cancelled by user (elicitation declined).',

package/src/tools/extract/analyzeContent.js

@@ -26,7 +26,7 @@ const AnalyzeContentSchema = z.object({
     includeAdvancedMetrics: z.boolean().default(false),
     groupEntitiesByType: z.boolean().default(true),
     rankByRelevance: z.boolean().default(true)
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const AnalyzeContentResult = z.object({

package/src/tools/extract/extractContent.js

@@ -45,7 +45,7 @@ const ExtractContentSchema = z.object({
     includeRawHTML: z.boolean().default(false),
     includeCleanedHTML: z.boolean().default(false),
     outputFormat: z.enum(['text', 'markdown', 'structured']).default('structured')
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const ExtractContentResult = z.object({

package/src/tools/extract/extractStructured.js

@@ -174,36 +174,78 @@ export class ExtractStructuredTool {
    * @param {Object} params - Extraction parameters
    * @returns {Promise<Object>} Extraction result
    */
-  async execute(params) {
+  async execute(params, ctx) {
     const startTime = Date.now();
 
     try {
       const validated = ExtractStructuredSchema.parse(params);
       const { url, schema, prompt, llmConfig, fallbackToSelectors, selectorHints, respect_robots, user_agent, verify_numbers } = validated;
 
-      // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
-      const { html, $, textContent, warnings } = await fetchAndParse(url, {
-        userAgent: user_agent || this.userAgent,
-        respectRobots: respect_robots,
-        tool: 'extract_structured'
-      });
-
-      // What the model reads — see shownText().
-      const shown = shownText($, html, url, textContent);
-
-      // Step 3: Try LLM extraction first
       let extractionResult = null;
       let extractionMethod = 'llm';
       let llmErrorMessage = null;
       let llmAvailable = false;
+      let llm = null;
 
+      // Step 0: LLM readiness, resolved before the fetch so the D1.4 gate below
+      // can ask before any network work — an unanswered gate returns an
+      // input-required result the SDK answers by re-entering this handler from
+      // the top, and everything above the gate runs a second time.
       try {
-        const llm = this._ensureLLMManager(llmConfig || {});
+        llm = this._ensureLLMManager(llmConfig || {});
         // ready() probes Ollama, which has no API key to gate on. isAvailable()
         // alone reported false on any machine without a cloud key, so a running
         // local Ollama was never used.
         llmAvailable = await llm.ready();
-        if (llmAvailable) {
+      } catch (llmError) {
+        // No usable LLM — this falls through to the CSS fallback. Keep the
+        // message so callers can tell "LLM broken" apart from "no LLM
+        // configured".
+        llmErrorMessage = llmError.message;
+      }
+
+      // D1.4: no LLM configured and the schema demands more than 3 required
+      // fields — confirm before running the lower-fidelity CSS fallback. With
+      // no LLM, step 3 extracts nothing, so landing on that fallback is already
+      // settled here, before the page is fetched.
+      const requiredCount = (schema.required || []).length;
+      if (fallbackToSelectors !== false && !llmAvailable && requiredCount > 3) {
+        const gate = this._elicitation.confirm(
+          ctx,
+          'extract_structured:no_llm_required_fields',
+          `No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
+          `extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
+          { url, required_fields: requiredCount }
+        );
+        if (gate.status === 'ask') return gate.result;
+        if (gate.status === 'cancelled') {
+          return {
+            success: false,
+            url,
+            data: {},
+            extraction_method: 'none',
+            confidence: 0,
+            schema_used: schema,
+            processingTime: Date.now() - startTime,
+            error: 'Extraction cancelled by user (elicitation declined).',
+            validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
+          };
+        }
+      }
+
+      // Step 1: Fetch and parse — shared helper strips scripts/styles/iframes/svgs
+      const { html, $, textContent, warnings } = await fetchAndParse(url, {
+        userAgent: user_agent || this.userAgent,
+        respectRobots: respect_robots,
+        tool: 'extract_structured'
+      });
+
+      // What the model reads — see shownText().
+      const shown = shownText($, html, url, textContent);
+
+      // Step 3: Try LLM extraction first (readiness resolved in step 0)
+      if (llmAvailable) {
+        try {
           const result = await llm.extractStructured(shown, schema, {
             prompt: prompt || '',
             maxContentLength: SHOWN_TEXT_BUDGET
@@ -218,12 +260,11 @@ export class ExtractStructuredTool {
           } else {
             llmErrorMessage = result?.error || 'LLM did not return usable JSON';
           }
+        } catch (llmError) {
+          // LLM failed — will fall through to CSS fallback.
+          extractionResult = null;
+          llmErrorMessage = llmError.message;
         }
-      } catch (llmError) {
-        // LLM failed — will fall through to CSS fallback. Keep the message so
-        // callers can tell "LLM broken" apart from "no LLM configured".
-        extractionResult = null;
-        llmErrorMessage = llmError.message;
       }
 
       // Step 3b (3.4): numeric provenance. Only the LLM path invents numbers —
@@ -318,31 +359,9 @@ export class ExtractStructuredTool {
         if (guarded.checked.skipped) provenance.skipped = guarded.checked.skipped;
       }
 
-      // Step 4: CSS selector fallback if LLM unavailable or failed
+      // Step 4: CSS selector fallback if LLM unavailable or failed (the D1.4
+      // confirmation for this path is gated above, before the fetch)
       if (!extractionResult && fallbackToSelectors !== false) {
-        // D1.4: no LLM configured and the schema demands more than 3 required
-        // fields — confirm before running the lower-fidelity CSS fallback.
-        const requiredCount = (schema.required || []).length;
-        if (!llmAvailable && requiredCount > 3) {
-          const proceed = await this._elicitation.confirm(
-            `No LLM provider is configured and the requested schema has ${requiredCount} required fields. ` +
-            `extract_structured will fall back to lower-fidelity CSS selector extraction, which may miss required fields.`,
-            { url, required_fields: requiredCount }
-          );
-          if (!proceed) {
-            return {
-              success: false,
-              url,
-              data: {},
-              extraction_method: 'none',
-              confidence: 0,
-              schema_used: schema,
-              processingTime: Date.now() - startTime,
-              error: 'Extraction cancelled by user (elicitation declined).',
-              validation: { valid: false, errors: ['Extraction cancelled by user (elicitation declined).'] }
-            };
-          }
-        }
         extractionResult = this._cssExtraction($, schema, selectorHints || {});
         extractionMethod = 'css_fallback';
       }

package/src/tools/extract/processDocument.js

@@ -49,7 +49,7 @@ const ProcessDocumentSchema = z.object({
     // Content filtering
     minContentLength: z.number().min(0).default(50),
     removeBoilerplate: z.boolean().default(true)
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const ProcessDocumentResult = z.object({

package/src/tools/extract/summarizeContent.js

@@ -28,7 +28,7 @@ const SummarizeContentSchema = z.object({
     maxKeywords: z.number().min(1).max(20).default(10),
     preserveStructure: z.boolean().default(false),
     language: z.string().optional()
-  }).optional().default({})
+  }).optional().prefault({})
 });
 
 const SummarizeContentResult = z.object({

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -21,7 +21,7 @@ const GenerateLLMsTxtSchema = z.object({
     checkSecurity: z.boolean().optional().default(false).describe('Whether to probe security-sensitive paths (opt-in; sends requests to /admin, /login, etc.)'),
     probeRateLimit: z.boolean().optional().default(false).describe('Whether to send repeated probe requests to estimate rate limits (opt-in; fires ~5 requests)'),
     respectRobots: z.boolean().optional().default(true).describe('Whether to respect robots.txt')
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   outputOptions: z.object({
     includeDetailed: z.boolean().optional().default(true).describe('Generate detailed LLMs-full.txt'),
@@ -31,7 +31,7 @@ const GenerateLLMsTxtSchema = z.object({
     customGuidelines: z.array(z.string()).optional().describe('Additional custom guidelines'),
     customRestrictions: z.array(z.string()).optional().describe('Additional restrictions'),
     robotsStyle: z.boolean().optional().default(false).describe('Emit legacy robots.txt-style directives instead of spec-compliant llmstxt.org markdown')
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   complianceLevel: z.enum(['basic', 'standard', 'strict']).optional().default('standard').describe('Compliance level for generated guidelines'),
   

package/src/tools/research/deepResearch.js

@@ -115,7 +115,7 @@ export class DeepResearchTool {
     this._elicitation = new ElicitationHelper({ mcpServer });
   }
 
-  async execute(params) {
+  async execute(params, ctx) {
     try {
       const validated = DeepResearchSchema.parse(params);
       const sessionId = this.generateSessionId();
@@ -137,10 +137,15 @@ export class DeepResearchTool {
       }
 
       // D1.4: Elicitation — warn user if projected cost exceeds 50 credits
-      // deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm
+      // deep_research costs approximately 1 credit per URL; maxUrls > 50 → confirm.
+      // An unanswered gate returns an input-required result the SDK answers by
+      // re-entering this handler from the top, so it sits above the session
+      // registration below — a second entry would otherwise leak a session.
       if (validated.maxUrls > 50) {
         const projectedCredits = validated.maxUrls;
-        const proceed = await this._elicitation.confirm(
+        const gate = this._elicitation.confirm(
+          ctx,
+          'deep_research:max_urls',
           `deep_research will scan up to ${validated.maxUrls} URLs, projecting ~${projectedCredits} credits.`,
           {
             topic: validated.topic,
@@ -148,7 +153,8 @@ export class DeepResearchTool {
             max_urls: validated.maxUrls,
           }
         );
-        if (!proceed) {
+        if (gate.status === 'ask') return gate.result;
+        if (gate.status === 'cancelled') {
           return {
             success: false,
             error: 'Research cancelled by user before starting (elicitation declined).',
@@ -238,7 +244,7 @@ export class DeepResearchTool {
         return {
           success: false,
           error: 'Invalid parameters',
-          details: validationError.errors.map(err => ({
+          details: validationError.issues.map(err => ({
             field: err.path.join('.'),
             message: err.message,
             received: err.received

package/src/tools/tracking/trackChanges/schema.js

@@ -48,7 +48,7 @@ export const TrackChangesSchema = z.object({
       moderate: z.number().min(0).max(1).default(0.3),
       major: z.number().min(0).max(1).default(0.7)
     }).optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   monitoringOptions: z.object({
     enabled: z.boolean().default(false),
@@ -59,7 +59,7 @@ export const TrackChangesSchema = z.object({
     enableWebhook: z.boolean().default(false),
     webhookUrl: z.string().url().optional(),
     webhookSecret: z.string().optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   storageOptions: z.object({
     enableSnapshots: z.boolean().default(true),
@@ -67,7 +67,7 @@ export const TrackChangesSchema = z.object({
     maxHistoryEntries: z.number().min(1).max(1000).default(100),
     compressionEnabled: z.boolean().default(true),
     deltaStorageEnabled: z.boolean().default(true)
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   queryOptions: z.object({
     limit: z.number().min(1).max(500).default(50),
@@ -76,7 +76,7 @@ export const TrackChangesSchema = z.object({
     endTime: z.number().optional(),
     includeContent: z.boolean().default(false),
     significanceFilter: z.enum(['all', 'minor', 'moderate', 'major', 'critical']).optional()
-  }).optional().default({}),
+  }).optional().prefault({}),
 
   notificationOptions: z.object({
     email: z.object({

package/src/utils/HumanBehaviorSimulator.js

@@ -12,7 +12,7 @@ const BehaviorConfigSchema = z.object({
     accuracy: z.number().min(0.1).max(1.0).default(0.8), // 0.1 = very inaccurate, 1.0 = perfect
     naturalCurves: z.boolean().default(true),
     randomMicroMovements: z.boolean().default(true)
-  }).default({}),
+  }).prefault({}),
   
   typing: z.object({
     enabled: z.boolean().default(true),
@@ -22,30 +22,30 @@ const BehaviorConfigSchema = z.object({
       enabled: z.boolean().default(true),
       frequency: z.number().min(0).max(0.1).default(0.02), // 2% mistake rate
       correctionDelay: z.number().min(100).max(2000).default(500)
-    }).default({})
-  }).default({}),
+    }).prefault({})
+  }).prefault({}),
   
   scrolling: z.object({
     enabled: z.boolean().default(true),
     naturalAcceleration: z.boolean().default(true),
     randomPauses: z.boolean().default(true),
     scrollBackProbability: z.number().min(0).max(1).default(0.1)
-  }).default({}),
+  }).prefault({}),
   
   interactions: z.object({
     hoverBeforeClick: z.boolean().default(true),
     clickDelay: z.object({
       min: z.number().default(100),
       max: z.number().default(300)
-    }).default({}),
+    }).prefault({}),
     focusBlurSimulation: z.boolean().default(true),
     idlePeriods: z.object({
       enabled: z.boolean().default(true),
       frequency: z.number().min(0).max(1).default(0.1), // 10% chance
       minDuration: z.number().default(1000),
       maxDuration: z.number().default(5000)
-    }).default({})
-  }).default({})
+    }).prefault({})
+  }).prefault({})
 });
 
 export class HumanBehaviorSimulator {