crawlforge-mcp-server 4.2.3 → 4.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "4.2.3",
+  "version": "4.2.4",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 23 web scraping, crawling, and content processing tools. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. v4.0 adds Markdown-first output, pre-built site templates, Camoufox stealth engine, and cost transparency.",
   "main": "server.js",
   "bin": {

package/server.js

@@ -57,7 +57,8 @@ if (!AuthManager.isAuthenticated() && !AuthManager.isCreatorMode()) {
   const apiKey = process.env.CRAWLFORGE_API_KEY;
   if (apiKey) {
     // Auto-setup if API key is provided via environment
-    console.log('🔧 Auto-configuring CrawlForge with provided API key...');
+    // Status → stderr; stdout is reserved for the MCP JSON-RPC stream.
+    console.error('🔧 Auto-configuring CrawlForge with provided API key...');
     const success = await AuthManager.runSetup(apiKey);
     if (!success) {
       console.error('❌ Failed to authenticate with provided API key');

package/src/cli/commands/analyze.js

@@ -1,6 +1,9 @@
 /**
  * analyze command — analyze content of a URL.
+ * Fetches and cleans the page content first (extract_content), then runs
+ * NLP analysis (analyze_content) on the extracted text.
  */
+import { ExtractContentTool } from '../../tools/extract/extractContent.js';
 import { AnalyzeContentTool } from '../../tools/extract/analyzeContent.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
@@ -13,7 +16,29 @@ export function register(program) {
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+
+      // analyze_content operates on text, so fetch & clean the page first.
+      const extractor = new ExtractContentTool(getToolConfig('extract_content'));
+      let text;
+      try {
+        const extracted = await extractor.execute({ url });
+        text = extracted?.content?.text;
+      } catch (e) {
+        process.stderr.write(`Error fetching content from ${url}: ${e.message}\n`);
+        process.exit(1);
+      }
+
+      if (!text || text.trim().length < 10) {
+        process.stderr.write(`Error: could not extract enough text from ${url} to analyze\n`);
+        process.exit(1);
+      }
+
       const tool = new AnalyzeContentTool(getToolConfig('analyze_content'));
-      await runTool(tool, { url, analysis_depth: opts.depth }, cliFlags);
+      // All analyses (language, topics, entities, sentiment, readability) default to true;
+      // --depth full additionally enables advanced metrics.
+      await runTool(tool, {
+        text,
+        options: { includeAdvancedMetrics: opts.depth === 'full' }
+      }, cliFlags);
     });
 }

package/src/cli/commands/localize.js

@@ -1,29 +1,64 @@
 /**
- * localize command — fetch content with locale/geo awareness.
+ * localize command — fetch a URL with locale/geo-aware request headers.
+ * Builds a localization config (Accept-Language, User-Agent) for the target
+ * country via LocalizationManager, then fetches the URL with those headers.
  */
 import { LocalizationManager } from '../../core/LocalizationManager.js';
+import { fetchUrlHandler } from '../../tools/basic/fetchUrl.js';
 import { getToolConfig } from '../../constants/config.js';
 import { runTool } from '../lib/runTool.js';
 
+// Derive a 2-letter country code from a --country flag or an en-US style locale.
+function resolveCountry(country, locale) {
+  if (country) return country.toUpperCase();
+  if (locale && locale.includes('-')) return locale.split('-')[1].toUpperCase();
+  return 'US';
+}
+
 export function register(program) {
   program
     .command('localize <url>')
-    .description('Fetch URL with locale/geo-aware settings')
+    .description('Fetch URL with locale/geo-aware request headers')
     .option('--locale <locale>', 'Locale code (e.g. en-US, fr-FR)', 'en-US')
     .option('--country <code>', 'Country code for geo-targeting (e.g. US, FR)')
     .option('--currency <code>', 'Currency code (e.g. USD, EUR)')
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
-      const mgr = new LocalizationManager(getToolConfig('localization'));
+
+      const countryCode = resolveCountry(opts.country, opts.locale);
+      const language = opts.locale ? opts.locale.split('-')[0] : undefined;
+
       const wrapperTool = {
-        execute: (p) => mgr.fetchWithLocalization(p)
+        execute: async () => {
+          const mgr = new LocalizationManager(getToolConfig('localization'));
+          await mgr.initialize();
+          const config = await mgr.configureCountry(countryCode, {
+            language,
+            currency: opts.currency
+          });
+
+          const headers = {
+            'Accept-Language': config.acceptLanguage,
+            'User-Agent': mgr.generateUserAgent(countryCode),
+            ...(config.customHeaders || {})
+          };
+
+          const fetched = await fetchUrlHandler({ url, headers });
+          return {
+            localization: {
+              countryCode: config.countryCode,
+              language: config.language,
+              timezone: config.timezone,
+              currency: config.currency,
+              acceptLanguage: config.acceptLanguage
+            },
+            request_headers: headers,
+            response: fetched
+          };
+        }
       };
-      await runTool(wrapperTool, {
-        url,
-        locale: opts.locale,
-        country: opts.country,
-        currency: opts.currency
-      }, cliFlags);
+
+      await runTool(wrapperTool, {}, cliFlags);
     });
 }

package/src/cli/commands/monitor.js

@@ -17,6 +17,7 @@ export function register(program) {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
       const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      // monitor runs continuously — do not auto-exit after the first result.
       await runTool(tool, {
         url,
         scheduled: true,
@@ -24,6 +25,6 @@ export function register(program) {
         selector: opts.selector,
         webhook_url: opts.webhook,
         change_threshold: parseFloat(opts.threshold)
-      }, cliFlags);
+      }, cliFlags, { exitOnSuccess: false });
     });
 }

package/src/cli/commands/template.js

@@ -7,7 +7,7 @@ import { runTool } from '../lib/runTool.js';
 
 export function register(program) {
   program
-    .command('template <id> <target>')
+    .command('template [id] [target]')
     .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo)')
     .option('--list', 'List all available templates')
     .action(async (id, target, opts, cmd) => {
@@ -16,11 +16,16 @@ export function register(program) {
       const tool = new ScrapeTemplateTool(getToolConfig('scrape_template'));
 
       if (opts.list) {
-        const wrapperTool = { execute: () => tool.listTemplates() };
+        const wrapperTool = { execute: () => tool.execute({ template: 'list' }) };
         await runTool(wrapperTool, {}, cliFlags);
         return;
       }
 
-      await runTool(tool, { template_id: id, url: target }, cliFlags);
+      if (!id || !target) {
+        process.stderr.write('Error: template requires <id> and <target>, or use --list\n');
+        process.exit(1);
+      }
+
+      await runTool(tool, { template: id, url: target }, cliFlags);
     });
 }

package/src/cli/index.js

@@ -58,11 +58,28 @@ program
   .option('--api-key <key>', 'CrawlForge API key (overrides CRAWLFORGE_API_KEY env var)')
   .option('--timeout <ms>', 'Global request timeout in milliseconds', '30000');
 
+// Resolve the API key from (in priority order): --api-key flag, CRAWLFORGE_API_KEY env,
+// then the stored ~/.crawlforge/config.json written by `crawlforge-setup`.
+function loadStoredApiKey() {
+  try {
+    const home = process.env.HOME || process.env.USERPROFILE;
+    if (!home) return undefined;
+    const cfgPath = join(home, '.crawlforge', 'config.json');
+    const cfg = JSON.parse(readFileSync(cfgPath, 'utf8'));
+    return cfg.apiKey || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 // Apply --api-key globally before commands run
 program.hook('preAction', (thisCommand) => {
   const opts = program.opts();
   if (opts.apiKey) {
     process.env.CRAWLFORGE_API_KEY = opts.apiKey;
+  } else if (!process.env.CRAWLFORGE_API_KEY) {
+    const stored = loadStoredApiKey();
+    if (stored) process.env.CRAWLFORGE_API_KEY = stored;
   }
   if (opts.timeout) {
     process.env.CRAWLFORGE_CLI_TIMEOUT = opts.timeout;

package/src/cli/lib/runTool.js

@@ -16,9 +16,13 @@ import { formatResult, formatError } from '../formatter.js';
  * @param {object} cliFlags     — { json, pretty, quiet }
  * @param {object} [options]
  * @param {boolean} [options.exitOnError=true]
+ * @param {boolean} [options.exitOnSuccess=true]  Exit the process after writing
+ *        output. One-shot CLI commands need this because background timers
+ *        (metrics, cache/connection cleanup, etc.) otherwise keep the event loop
+ *        alive. Long-running commands (e.g. `monitor`) pass false.
  */
 export async function runTool(tool, params, cliFlags, options = {}) {
-  const { exitOnError = true } = options;
+  const { exitOnError = true, exitOnSuccess = true } = options;
 
   try {
     const result = await tool.execute(params);
@@ -32,7 +36,12 @@ export async function runTool(tool, params, cliFlags, options = {}) {
     }
 
     const output = formatResult(result, cliFlags);
-    if (output) process.stdout.write(output + '\n');
+    if (output) {
+      // Wait for stdout to flush (pipes/files buffer) before exiting.
+      process.stdout.write(output + '\n', () => { if (exitOnSuccess) process.exit(0); });
+    } else if (exitOnSuccess) {
+      process.exit(0);
+    }
   } catch (error) {
     process.stderr.write(formatError(error, cliFlags) + '\n');
     if (exitOnError) process.exit(1);

package/src/core/ActionExecutor.js

@@ -926,7 +926,8 @@ export class ActionExecutor extends EventEmitter {
    */
   log(level, message) {
     if (this.enableLogging) {
-      console.log('[ActionExecutor:' + level.toUpperCase() + '] ' + message);
+      // → stderr so stdout stays clean for MCP JSON-RPC / CLI --json output.
+      console.error('[ActionExecutor:' + level.toUpperCase() + '] ' + message);
     }
   }
 

package/src/core/AuthManager.js

@@ -69,7 +69,8 @@ class AuthManager {
 
     // Skip config loading in creator mode
     if (this.isCreatorMode()) {
-      console.log('🚀 Creator Mode Active - Unlimited Access Enabled');
+      // Status → stderr; stdout is reserved for MCP JSON-RPC / CLI --json output.
+      console.error('🚀 Creator Mode Active - Unlimited Access Enabled');
       this.initialized = true;
       return;
     }
@@ -78,7 +79,7 @@ class AuthManager {
       await this.loadConfig();
       this.initialized = true;
     } catch (error) {
-      console.log('No existing CrawlForge configuration found. Run setup to configure.');
+      console.error('No existing CrawlForge configuration found. Run setup to configure.');
       this.initialized = true;
     }
 

package/src/core/PerformanceManager.js

@@ -771,6 +771,9 @@ export class PerformanceManager extends EventEmitter {
     this.metricsTimer = setInterval(() => {
       this.collectMetrics();
     }, this.metricsInterval);
+    // Don't let the metrics interval keep a short-lived process (e.g. a one-shot
+    // CLI command) alive. The long-running server stays up via its stdio transport.
+    if (typeof this.metricsTimer.unref === 'function') this.metricsTimer.unref();
   }
 
   /**

package/src/core/creatorMode.js

@@ -29,7 +29,8 @@ if (process.env.CRAWLFORGE_CREATOR_SECRET) {
 
   if (crypto.timingSafeEqual(Buffer.from(providedHash, 'hex'), Buffer.from(CREATOR_SECRET_HASH, 'hex'))) {
     _creatorModeVerified = true;
-    console.log('Creator Mode Enabled - Unlimited Access');
+    // Status message → stderr so stdout stays clean (MCP JSON-RPC / CLI --json output).
+    console.error('Creator Mode Enabled - Unlimited Access');
   } else {
     console.warn('Invalid creator secret provided');
   }

package/src/tools/advanced/batchScrape/index.js

@@ -301,7 +301,8 @@ export class BatchScrapeTool extends EventEmitter {
   }
 
   _log(level, message) {
-    if (this.enableLogging) console.log(`[BatchScrapeTool:${level.toUpperCase()}] ${message}`);
+    // → stderr so stdout stays clean for MCP JSON-RPC / CLI --json output.
+    if (this.enableLogging) console.error(`[BatchScrapeTool:${level.toUpperCase()}] ${message}`);
   }
 
   _initializeJobExecutors() {

package/src/tools/search/adapters/searchProviderFactory.js

@@ -36,7 +36,8 @@ export class SearchProviderFactory {
         );
       }
 
-      console.log('🔍 Creator Mode: Using Google Search API directly');
+      // Status message → stderr so stdout stays clean (MCP JSON-RPC / CLI --json).
+      console.error('🔍 Creator Mode: Using Google Search API directly');
       return new GoogleSearchAdapter(googleApiKey, googleSearchEngineId);
     }
 

package/src/utils/Logger.js

@@ -116,6 +116,9 @@ export class Logger {
 
     if (enableConsole) {
       transports.push(new winston.transports.Console({
+        // Route ALL log levels to stderr so stdout stays reserved for structured
+        // output (MCP JSON-RPC protocol and CLI --json results).
+        stderrLevels: ['error', 'warn', 'info', 'http', 'verbose', 'debug', 'silly'],
         format: winston.format.combine(
           winston.format.colorize(),
           winston.format.simple()