crawlforge-mcp-server 3.4.0 → 4.2.1

package/src/cli/lib/runTool.js

@@ -0,0 +1,40 @@
+/**
+ * runTool.js — Thin wrapper that invokes a tool's execute() method directly
+ * and formats the output according to global CLI flags.
+ *
+ * This intentionally does NOT replicate withAuth credit logic — CLI invocations
+ * go through the same AuthManager path as MCP calls when a real API key is set.
+ * In creator mode (CRAWLFORGE_CREATOR_SECRET set) credits are skipped automatically.
+ */
+
+import { formatResult, formatError } from '../formatter.js';
+
+/**
+ * Run a tool and print formatted output.
+ * @param {object} tool         — tool instance with execute(params) method
+ * @param {object} params       — tool parameters
+ * @param {object} cliFlags     — { json, pretty, quiet }
+ * @param {object} [options]
+ * @param {boolean} [options.exitOnError=true]
+ */
+export async function runTool(tool, params, cliFlags, options = {}) {
+  const { exitOnError = true } = options;
+
+  try {
+    const result = await tool.execute(params);
+
+    // Check for MCP-style error response
+    if (result && result.isError) {
+      const errText = result.content?.[0]?.text ?? 'Tool returned an error';
+      process.stderr.write(formatError(errText, cliFlags) + '\n');
+      if (exitOnError) process.exit(1);
+      return;
+    }
+
+    const output = formatResult(result, cliFlags);
+    if (output) process.stdout.write(output + '\n');
+  } catch (error) {
+    process.stderr.write(formatError(error, cliFlags) + '\n');
+    if (exitOnError) process.exit(1);
+  }
+}

package/src/core/ActionExecutor.js

@@ -202,13 +202,15 @@ export class ActionExecutor extends EventEmitter {
       this.activeChains.set(chainId, executionContext);
       this.emit('chainStarted', executionContext);
 
-      // Initialize browser and navigate to page
-      const page = await this.initializePage(url, browserOptions);
-      executionContext.page = page;
-
+      // D2.4: initialize page INSIDE try/finally so it is always closed even on
+      // errors thrown between acquisition and the inner try block.
+      let page = null;
       let chainResult;
       
       try {
+        page = await this.initializePage(url, browserOptions);
+        executionContext.page = page;
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
         
@@ -235,9 +237,9 @@ export class ActionExecutor extends EventEmitter {
 
         throw error;
       } finally {
-        // Clean up page
+        // D2.4: always close page to prevent leaks
         if (page) {
-          await page.close();
+          try { await page.close(); } catch (_) { /* ignore close errors */ }
         }
         
         // Update execution time

package/src/core/AuthManager.js

@@ -10,6 +10,15 @@ import { randomUUID } from 'crypto';
 import { isCreatorModeVerified } from './creatorMode.js';
 import { resolveApiEndpoint } from './endpointGuard.js';
 import { logger } from '../utils/Logger.js';
+// D1.4: Elicitation for low-credit warnings (lazy import to avoid circular dep)
+let _ElicitationHelper = null;
+function getElicitationHelper() {
+  if (!_ElicitationHelper) {
+    // Dynamic import to avoid circular dependency at module load time
+    return null; // Will be set via setElicitation() from server.js
+  }
+  return _ElicitationHelper;
+}
 
 class AuthManager {
   constructor() {
@@ -22,9 +31,22 @@ class AuthManager {
     this.lastSuccessfulCreditCheck = new Map();
     this.CREDIT_CHECK_INTERVAL = 15000;
     this.initialized = false;
+    // D2.1: simple async mutex to prevent concurrent reportUsage calls from
+    // double-decrementing the credit cache before the backend ack arrives.
+    this._usageQueue = Promise.resolve();
+    // D1.4: Elicitation helper for low-credit warnings
+    this._elicitation = null;
     // NOTE: Don't read creator mode in constructor - it's set dynamically in server.js
   }
 
+  /**
+   * D1.4: Set elicitation helper for low-credit warnings.
+   * @param {object} elicitation - ElicitationHelper instance
+   */
+  setElicitation(elicitation) {
+    this._elicitation = elicitation;
+  }
+
   /**
    * Check if running in creator mode (unlimited access, no API required)
    * Uses module-scoped verified flag from server.js - cannot be bypassed via env vars
@@ -243,6 +265,24 @@ class AuthManager {
         this.creditCache.set(this.config.userId, data.creditsRemaining);
         this.lastCreditCheck = now;
         this.lastSuccessfulCreditCheck.set(this.config.userId, now);
+
+        // D1.4: If credits are close to running out, elicit confirmation instead of hard-failing
+        if (data.creditsRemaining < estimatedCredits) {
+          if (this._elicitation) {
+            const proceed = await this._elicitation.confirm(
+              `Low credits: ${data.creditsRemaining} remaining, this tool needs ~${estimatedCredits}. Proceed anyway?`,
+              {
+                credits_remaining: data.creditsRemaining,
+                credits_needed: estimatedCredits,
+                note: 'Top up at https://www.crawlforge.dev/dashboard',
+              }
+            );
+            if (!proceed) return false;
+            return true; // user confirmed — let tool attempt it
+          }
+          return false; // no elicitation — standard hard-fail behavior
+        }
+
         return data.creditsRemaining >= estimatedCredits;
       }
     } catch (error) {
@@ -269,9 +309,18 @@ class AuthManager {
       return; // Silently skip if not configured
     }
 
+    // D2.1: serialize via promise queue so concurrent tool calls do not race
+    // on creditCache and double-decrement before the backend ack arrives.
+    this._usageQueue = this._usageQueue.then(() =>
+      this._reportUsageOnce(tool, creditsUsed, requestData, responseStatus, processingTime)
+    );
+    return this._usageQueue;
+  }
+
+  async _reportUsageOnce(tool, creditsUsed, requestData = {}, responseStatus = 200, processingTime = 0) {
     const userId = this.config.userId;
 
-    // Pre-decrement cache before fetch so network failures still deplete credits
+    // Decrement only inside the serialized task -- no concurrent races
     const cached = this.creditCache.get(userId);
     if (cached !== undefined) {
       this.creditCache.set(userId, Math.max(0, cached - creditsUsed));
@@ -484,13 +533,64 @@ class AuthManager {
       // Phase 1: LLM-Powered Structured Extraction
       extract_structured: 4,
 
-      // Phase C5: Natural-language LLM extraction (external paid API call per invocation)
-      extract_with_llm: 5
+      // D3.3: Pre-built site templates (1 credit — same as fetch_url)
+      extract_with_llm: 5,
+
+      // D3.3: Pre-built site templates (1 credit per template scrape)
+      scrape_template: 1
     };
 
     return costs[tool] || 1;
   }
 
+  /**
+   * D3.5: Project the cost of calling a tool with given params.
+   *
+   * Returns a lower-bound estimate.  Dynamic tools (deep_research, crawl_deep)
+   * have variable costs that depend on runtime behaviour (e.g. how many URLs
+   * get fetched).  The projection is a MINIMUM — actual cost may be higher.
+   * Accuracy caveats are documented in each tool description.
+   *
+   * @param {string} toolName
+   * @param {object} params
+   * @returns {{ projected: number, note: string }}
+   */
+  projectCost(toolName, params) {
+    const base = this.getToolCost(toolName);
+
+    // Override for tools whose cost scales with params
+    let projected = base;
+    let note = 'Fixed cost per invocation.';
+
+    switch (toolName) {
+      case 'batch_scrape': {
+        const urlCount = Array.isArray(params?.urls) ? params.urls.length : 1;
+        projected = Math.max(base, Math.ceil(urlCount / 10));
+        note = `Estimated from ${urlCount} URLs. Actual may be higher for slow/large pages.`;
+        break;
+      }
+      case 'deep_research': {
+        const maxUrls = params?.maxUrls || params?.options?.maxUrls || 20;
+        projected = Math.max(base, Math.ceil(maxUrls / 5) + base);
+        note = `Lower-bound estimate. deep_research cost grows with source count (${maxUrls} max URLs).`;
+        break;
+      }
+      case 'crawl_deep': {
+        const maxPages = params?.maxPages || params?.options?.maxPages || 10;
+        projected = Math.max(base, Math.ceil(maxPages / 20) * base);
+        note = `Lower-bound estimate. crawl_deep cost grows with page count (${maxPages} max).`;
+        break;
+      }
+      case 'extract_with_llm':
+        note = 'Includes external LLM API call cost (not billed in credits, billed by your LLM provider).';
+        break;
+      default:
+        note = 'Fixed cost per invocation.';
+    }
+
+    return { projected, note };
+  }
+
   /**
    * Check if authenticated
    */

package/src/core/ChangeTracker.js

@@ -6,6 +6,7 @@ import crypto from "crypto";
  */
 
 import { createHash } from 'crypto';
+import { Worker } from 'worker_threads';
 import { z } from 'zod';
 import { EventEmitter } from 'events';
 import { load } from 'cheerio';
@@ -828,6 +829,39 @@ export class ChangeTracker extends EventEmitter {
       .update(content || '')
       .digest('hex');
   }
+
+  /**
+   * D2.8: Hash large content (>256KB) off the main thread to avoid event-loop blocking.
+   * Falls back to synchronous hashContent for smaller payloads.
+   * @param {string} content
+   * @returns {Promise<string>}
+   */
+  async hashContentAsync(content) {
+    const THRESHOLD = 256 * 1024; // 256 KB
+    const str = content || '';
+    if (str.length <= THRESHOLD) {
+      return this.hashContent(str);
+    }
+
+    const algorithm = this.options.hashAlgorithm || 'sha256';
+    return new Promise((resolve, reject) => {
+      const workerCode = `
+        const { createHash } = require('crypto');
+        const { workerData, parentPort } = require('worker_threads');
+        const hash = createHash(workerData.algorithm).update(workerData.content).digest('hex');
+        parentPort.postMessage(hash);
+      `;
+      const worker = new Worker(workerCode, {
+        eval: true,
+        workerData: { content: str, algorithm }
+      });
+      worker.once('message', resolve);
+      worker.once('error', (err) => {
+        // Fallback to sync on worker error
+        try { resolve(this.hashContent(str)); } catch (e) { reject(e); }
+      });
+    });
+  }
   
   calculateSimilarity(hash1, hash2) {
     if (hash1 === hash2) return 1;

package/src/core/ElicitationHelper.js

@@ -0,0 +1,112 @@
+/**
+ * ElicitationHelper — MCP Elicitation for CrawlForge
+ *
+ * Allows tools to request user confirmation or input mid-execution for
+ * expensive or ambiguous operations. Falls back gracefully when the
+ * MCP client does not support elicitation.
+ *
+ * MCP Spec 2025-11-25: client/elicit request with requestedSchema
+ */
+
+export class ElicitationHelper {
+  /**
+   * @param {object} options
+   * @param {object|null} options.mcpServer - McpServer instance
+   * @param {object|null} options.logger
+   */
+  constructor({ mcpServer, logger } = {}) {
+    this._mcpServer = mcpServer || null;
+    this._logger = logger || { warn: () => {}, info: () => {} };
+  }
+
+  /**
+   * Whether the connected MCP client supports elicitation.
+   * @returns {boolean}
+   */
+  get supported() {
+    return !!(this._mcpServer?.server?.elicit);
+  }
+
+  /**
+   * Ask for user confirmation before proceeding with an expensive operation.
+   * Returns true if confirmed (or if elicitation is unsupported — fail-open
+   * so tools continue working in non-elicitation clients).
+   *
+   * @param {string} message - Human-readable explanation of what requires confirmation
+   * @param {object} [details] - Additional context (projected cost, URL count, etc.)
+   * @returns {Promise<boolean>} - true = proceed, false = cancel
+   */
+  async confirm(message, details = {}) {
+    if (!this.supported) {
+      this._logger.warn('Elicitation not supported by client — proceeding without confirmation', { message });
+      return true;
+    }
+
+    try {
+      const detailLines = Object.entries(details)
+        .map(([k, v]) => `  ${k}: ${v}`)
+        .join('\n');
+      const fullMessage = detailLines ? `${message}\n\n${detailLines}` : message;
+
+      const result = await this._mcpServer.server.elicit({
+        message: fullMessage,
+        requestedSchema: {
+          type: 'object',
+          properties: {
+            confirmed: {
+              type: 'boolean',
+              title: 'Proceed?',
+              description: 'Confirm to proceed with the operation',
+            },
+          },
+          required: ['confirmed'],
+        },
+      });
+
+      return result?.content?.confirmed === true;
+    } catch (err) {
+      this._logger.warn('Elicitation request failed — proceeding without confirmation', { error: err.message });
+      return true; // fail-open
+    }
+  }
+
+  /**
+   * Ask the user to provide a string value (e.g. missing schema field).
+   *
+   * @param {string} message
+   * @param {object} [options]
+   * @param {string} [options.fieldName]
+   * @param {string} [options.fieldDescription]
+   * @param {string} [options.defaultValue]
+   * @returns {Promise<string|null>} - The user-provided value or null if cancelled/unsupported
+   */
+  async requestString(message, { fieldName = 'value', fieldDescription = '', defaultValue } = {}) {
+    if (!this.supported) {
+      this._logger.warn('Elicitation not supported — using default value', { fieldName, defaultValue });
+      return defaultValue || null;
+    }
+
+    try {
+      const result = await this._mcpServer.server.elicit({
+        message,
+        requestedSchema: {
+          type: 'object',
+          properties: {
+            [fieldName]: {
+              type: 'string',
+              title: fieldName,
+              description: fieldDescription,
+              ...(defaultValue ? { default: defaultValue } : {}),
+            },
+          },
+          required: [fieldName],
+        },
+      });
+
+      return result?.content?.[fieldName] || defaultValue || null;
+    } catch (err) {
+      this._logger.warn('Elicitation request failed', { error: err.message });
+      return defaultValue || null;
+    }
+  }
+}

package/src/core/JobManager.js

@@ -139,6 +139,28 @@ export class JobManager extends EventEmitter {
       logs: []
     };
 
+    // D2.6: LRU eviction -- remove oldest completed/failed/cancelled job when at capacity
+    if (this.jobs.size >= this.maxJobs) {
+      let evicted = false;
+      for (const [eid, ejob] of this.jobs) {
+        if ([this.JOB_STATES.COMPLETED, this.JOB_STATES.FAILED, this.JOB_STATES.CANCELLED].includes(ejob.status)) {
+          this.jobs.delete(eid);
+          this.jobsByStatus.get(ejob.status).delete(eid);
+          evicted = true;
+          break;
+        }
+      }
+      if (!evicted) {
+        // All jobs are active -- evict the oldest regardless of state
+        const oldestId = this.jobs.keys().next().value;
+        const oldest = this.jobs.get(oldestId);
+        if (oldest) {
+          this.jobs.delete(oldestId);
+          this.jobsByStatus.get(oldest.status).delete(oldestId);
+        }
+      }
+    }
+
     // Store job
     this.jobs.set(jobId, job);
     this.jobsByStatus.get(this.JOB_STATES.PENDING).add(jobId);
@@ -345,7 +367,17 @@ export class JobManager extends EventEmitter {
 
     await this.updateJobStatus(jobId, this.JOB_STATES.CANCELLED);
     this.emit('jobCancelled', job);
-    
+
+    // D2.6: cascade-cancel all jobs that depend on this one
+    for (const [depId, depJob] of this.jobs) {
+      if (depJob.dependencies && depJob.dependencies.includes(jobId)) {
+        if (![this.JOB_STATES.COMPLETED, this.JOB_STATES.FAILED, this.JOB_STATES.CANCELLED].includes(depJob.status)) {
+          await this.updateJobStatus(depId, this.JOB_STATES.CANCELLED);
+          this.emit('jobCancelled', depJob);
+        }
+      }
+    }
+
     return job;
   }
 
@@ -456,8 +488,10 @@ export class JobManager extends EventEmitter {
     const now = Date.now();
     const expiredJobs = [];
 
+    // D2.6: expire ALL jobs past their TTL regardless of state (was previously only checking expiresAt)
     for (const [jobId, job] of this.jobs) {
-      if (job.expiresAt && now > job.expiresAt) {
+      const expiry = job.expiresAt || (job.createdAt + (job.ttl || this.defaultTtl));
+      if (now > expiry) {
         expiredJobs.push(jobId);
       }
     }

package/src/core/LocalizationManager.js

@@ -157,11 +157,25 @@ export class LocalizationManager extends EventEmitter {
     };
     
     this.currentSettings = { ...this.defaultSettings, ...options };
-    this.localeCache = new Map();
-    this.geoLocationCache = new Map();
-    this.timezoneCache = new Map();
-    this.proxyCache = new Map();
-    this.translationCache = new Map();
+    // D2.8: cap all caches to prevent unbounded growth under long-lived sessions.
+    const MAX_CACHE = parseInt(process.env.LOCALIZATION_CACHE_MAX || '500', 10);
+    const makeLRUMap = (max) => {
+      const m = new Map();
+      m._max = max;
+      const origSet = m.set.bind(m);
+      m.set = (k, v) => {
+        if (m.size >= m._max) {
+          m.delete(m.keys().next().value); // evict oldest
+        }
+        return origSet(k, v);
+      };
+      return m;
+    };
+    this.localeCache = makeLRUMap(MAX_CACHE);
+    this.geoLocationCache = makeLRUMap(MAX_CACHE);
+    this.timezoneCache = makeLRUMap(MAX_CACHE);
+    this.proxyCache = makeLRUMap(MAX_CACHE);
+    this.translationCache = makeLRUMap(MAX_CACHE);
     
     // Proxy management
     this.proxyManager = {

package/src/core/PerformanceManager.js

@@ -206,19 +206,40 @@ export class PerformanceManager extends EventEmitter {
       return this.taskRouting[taskType];
     }
 
-    // Auto-select based on task characteristics
-    const dataSize = this.getDataSize(data);
-    const isLargeDataset = dataSize > 10 * 1024 * 1024; // 10MB
-    const isCpuIntensive = this.isCpuIntensive(data);
-    const isNetworkOperation = this.isNetworkOperation(taskType);
-
-    if (isLargeDataset && !isCpuIntensive) {
-      return 'stream';
-    } else if (isCpuIntensive) {
-      return 'worker';
-    } else if (isNetworkOperation) {
-      return 'connection';
-    } else {
+    // D2.7: route by live queue depth + wait time, not just static heuristics
+    try {
+      const workerStats = this.workerPool.getStats ? this.workerPool.getStats() : {};
+      const connStats = this.connectionPool.getStats ? this.connectionPool.getStats() : {};
+      const workerDepth = workerStats.pendingCount || workerStats.queueDepth || 0;
+      const connDepth = connStats.pendingCount || connStats.activeConnections || 0;
+      const workerAvgWait = workerStats.averageWaitMs || workerStats.avgWaitTime || 0;
+      const connAvgWait = connStats.averageWaitMs || connStats.avgWaitTime || 0;
+
+      const dataSize = this.getDataSize(data);
+      const isLargeDataset = dataSize > 10 * 1024 * 1024; // 10MB
+      const isCpuIntensive = this.isCpuIntensive(data);
+      const isNetworkOperation = this.isNetworkOperation(taskType);
+
+      if (isNetworkOperation) {
+        if (connDepth < 50 && connAvgWait < 2000) return 'connection';
+        if (workerDepth < connDepth) return 'worker';
+        return 'queue';
+      }
+      if (isCpuIntensive) {
+        if (workerDepth < 20 && workerAvgWait < 5000) return 'worker';
+        return 'queue';
+      }
+      if (isLargeDataset) return 'stream';
+      return 'queue';
+    } catch (_statsErr) {
+      // Stats API unavailable -- fall back to static heuristics
+      const dataSize = this.getDataSize(data);
+      const isLargeDataset = dataSize > 10 * 1024 * 1024;
+      const isCpuIntensive = this.isCpuIntensive(data);
+      const isNetworkOperation = this.isNetworkOperation(taskType);
+      if (isLargeDataset && !isCpuIntensive) return 'stream';
+      if (isCpuIntensive) return 'worker';
+      if (isNetworkOperation) return 'connection';
       return 'queue';
     }
   }
@@ -809,16 +830,31 @@ export class PerformanceManager extends EventEmitter {
   async shutdown() {
     this.emit('shutdown');
 
+    // D2.7: signal all in-flight tasks to abort via AbortController
+    if (this._shutdownController) {
+      this._shutdownController.abort();
+    }
+    this._shutdownController = new AbortController();
+
     // Stop metrics collection
     if (this.metricsTimer) {
       clearInterval(this.metricsTimer);
     }
 
-    // Shutdown all components
+    // Shutdown all components with a 5-second timeout each
+    const shutdownWithTimeout = (component, name) => {
+      const timeout = new Promise((_, reject) =>
+        setTimeout(() => reject(new Error(`${name} shutdown timed out`)), 5000)
+      );
+      return Promise.race([component.shutdown(), timeout]).catch(err => {
+        console.error(`PerformanceManager: ${err.message}`);
+      });
+    };
+
     await Promise.all([
-      this.workerPool.shutdown(),
-      this.connectionPool.shutdown(),
-      this.streamProcessor.shutdown()
+      shutdownWithTimeout(this.workerPool, 'WorkerPool'),
+      shutdownWithTimeout(this.connectionPool, 'ConnectionPool'),
+      shutdownWithTimeout(this.streamProcessor, 'StreamProcessor')
     ]);
 
     this.emit('shutdownComplete');

package/src/core/ResearchOrchestrator.js

@@ -177,6 +177,8 @@ export class ResearchOrchestrator extends EventEmitter {
    * Initialize research session state
    */
   initializeResearchSession(sessionId, topic, startTime) {
+    // D2.3: per-session token budget (approx 4 chars = 1 token, 1M char cap = ~250K tokens)
+    const TOKEN_BUDGET_CHARS = parseInt(process.env.RESEARCH_TOKEN_BUDGET_CHARS || String(1_000_000), 10);
     this.researchState = {
       sessionId,
       topic,
@@ -188,7 +190,11 @@ export class ResearchOrchestrator extends EventEmitter {
       researchFindings: [],
       credibilityScores: new Map(),
       conflictMap: new Map(),
-      activityLog: []
+      activityLog: [],
+      // D2.3 token budget tracking
+      tokenBudgetChars: TOKEN_BUDGET_CHARS,
+      tokenBudgetUsed: 0,
+      tokenBudgetExceeded: false
     };
 
     // Reset metrics
@@ -461,7 +467,9 @@ export class ResearchOrchestrator extends EventEmitter {
         const batchPromises = batch.map(async (source) => {
           try {
             if (this.researchState.visitedUrls.has(source.link)) {
-              return null;
+              // D2.10: return already-extracted content rather than null,
+              // so overlapping query batches can reuse it.
+              return this.researchState.extractedContent.get(source.link) || null;
             }
             
             this.researchState.visitedUrls.add(source.link);
@@ -530,6 +538,22 @@ export class ResearchOrchestrator extends EventEmitter {
                 readabilityScore: this.calculateReadabilityScore(contentText)
               };
 
+              // D2.3: charge content length to token budget (rough 4 chars/token heuristic)
+              if (this.researchState.tokenBudgetUsed !== undefined) {
+                this.researchState.tokenBudgetUsed += contentText.length;
+                if (this.researchState.tokenBudgetUsed > this.researchState.tokenBudgetChars) {
+                  if (!this.researchState.tokenBudgetExceeded) {
+                    this.researchState.tokenBudgetExceeded = true;
+                    this.logger.warn('Research token budget exceeded -- skipping remaining LLM calls', {
+                      sessionId: this.researchState.sessionId,
+                      budgetChars: this.researchState.tokenBudgetChars,
+                      usedChars: this.researchState.tokenBudgetUsed
+                    });
+                  }
+                  this.enableLLMFeatures = false; // disable for remainder of session
+                }
+              }
+
               // LLM-powered relevance analysis
               if (this.enableLLMFeatures && topic) {
                 try {
@@ -1085,11 +1109,16 @@ export class ResearchOrchestrator extends EventEmitter {
   }
 
   deduplicateSources(sources) {
-    const seen = new Set();
+    // D2.10: use per-session visitedUrls so URLs are deduped across all query batches,
+    // not just within a single gatherInitialSources call.
+    const sessionSeen = this.researchState && this.researchState.visitedUrls
+      ? this.researchState.visitedUrls
+      : new Set();
+    const localSeen = new Set();
     return sources.filter(source => {
       const key = source.link;
-      if (seen.has(key)) return false;
-      seen.add(key);
+      if (sessionSeen.has(key) || localSeen.has(key)) return false;
+      localSeen.add(key);
       return true;
     });
   }
@@ -1226,6 +1255,12 @@ export class ResearchOrchestrator extends EventEmitter {
           timeLimit: this.timeLimit,
           completedWithinLimit: this.metrics.totalProcessingTime < this.timeLimit
         },
+        // D2.3: cost transparency
+        _cost: {
+          tokenBudgetChars: this.researchState.tokenBudgetChars,
+          tokenBudgetUsed: this.researchState.tokenBudgetUsed,
+          tokenBudgetExceeded: this.researchState.tokenBudgetExceeded
+        },
         metadata: {
           generatedAt: new Date().toISOString(),
           researchDepth: this.researchState.currentDepth,