crawlforge-mcp-server 3.5.1 → 4.2.2

package/src/cli/commands/llmstxt.js

@@ -0,0 +1,24 @@
+/**
+ * llmstxt command — generate llms.txt for a website.
+ */
+import { GenerateLLMsTxtTool } from '../../tools/llmstxt/generateLLMsTxt.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('llmstxt <url>')
+    .description('Generate llms.txt for a website (AI compliance file)')
+    .option('--include-full', 'Also generate llms-full.txt')
+    .option('--max-pages <n>', 'Maximum pages to analyze', '50')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new GenerateLLMsTxtTool(getToolConfig('generate_llms_txt'));
+      await runTool(tool, {
+        url,
+        include_full_txt: !!opts.includeFull,
+        max_pages: parseInt(opts.maxPages, 10)
+      }, cliFlags);
+    });
+}

package/src/cli/commands/localize.js

@@ -0,0 +1,29 @@
+/**
+ * localize command — fetch content with locale/geo awareness.
+ */
+import { LocalizationManager } from '../../core/LocalizationManager.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('localize <url>')
+    .description('Fetch URL with locale/geo-aware settings')
+    .option('--locale <locale>', 'Locale code (e.g. en-US, fr-FR)', 'en-US')
+    .option('--country <code>', 'Country code for geo-targeting (e.g. US, FR)')
+    .option('--currency <code>', 'Currency code (e.g. USD, EUR)')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const mgr = new LocalizationManager(getToolConfig('localization'));
+      const wrapperTool = {
+        execute: (p) => mgr.fetchWithLocalization(p)
+      };
+      await runTool(wrapperTool, {
+        url,
+        locale: opts.locale,
+        country: opts.country,
+        currency: opts.currency
+      }, cliFlags);
+    });
+}

package/src/cli/commands/map.js

@@ -0,0 +1,26 @@
+/**
+ * map command — generate a sitemap using map_site tool.
+ */
+import { MapSiteTool } from '../../tools/crawl/mapSite.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('map <url>')
+    .description('Generate a sitemap for a website')
+    .option('--depth <n>', 'Maximum crawl depth', '3')
+    .option('--max-pages <n>', 'Maximum pages to include', '500')
+    .option('--format <fmt>', 'Output format: json or xml', 'json')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new MapSiteTool(getToolConfig('map_site'));
+      await runTool(tool, {
+        url,
+        max_depth: parseInt(opts.depth, 10),
+        max_pages: parseInt(opts.maxPages, 10),
+        output_format: opts.format
+      }, cliFlags);
+    });
+}

package/src/cli/commands/monitor.js

@@ -0,0 +1,29 @@
+/**
+ * monitor command — continuously monitor a URL for changes (scheduled mode).
+ */
+import { TrackChangesTool } from '../../tools/tracking/trackChanges/index.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('monitor <url>')
+    .description('Continuously monitor a URL for content changes')
+    .option('--interval <seconds>', 'Check interval in seconds', '300')
+    .option('--selector <css>', 'CSS selector to scope monitoring')
+    .option('--webhook <url>', 'Webhook URL to notify on changes')
+    .option('--threshold <pct>', 'Change threshold percentage (0-100)', '5')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      await runTool(tool, {
+        url,
+        scheduled: true,
+        interval_seconds: parseInt(opts.interval, 10),
+        selector: opts.selector,
+        webhook_url: opts.webhook,
+        change_threshold: parseFloat(opts.threshold)
+      }, cliFlags);
+    });
+}

package/src/cli/commands/research.js

@@ -0,0 +1,26 @@
+/**
+ * research command — deep research on a topic.
+ */
+import { DeepResearchTool } from '../../tools/research/deepResearch.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('research <topic>')
+    .description('Conduct deep research on a topic')
+    .option('--depth <level>', 'Research depth: basic, standard, or deep', 'standard')
+    .option('--max-urls <n>', 'Maximum URLs to analyze', '20')
+    .option('--output-format <fmt>', 'Output format: summary or detailed', 'summary')
+    .action(async (topic, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new DeepResearchTool(getToolConfig('deep_research'));
+      await runTool(tool, {
+        query: topic,
+        depth: opts.depth,
+        max_urls: parseInt(opts.maxUrls, 10),
+        output_format: opts.outputFormat
+      }, cliFlags);
+    });
+}

package/src/cli/commands/scrape.js

@@ -0,0 +1,37 @@
+/**
+ * scrape command — fetches a URL and returns its content.
+ * Without --extract: uses fetch_url (raw HTML + headers).
+ * With --extract: uses extract_content (cleaned text/markdown).
+ */
+import { fetchUrlHandler } from '../../tools/basic/fetchUrl.js';
+import { ExtractContentTool } from '../../tools/extract/extractContent.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('scrape <url>')
+    .description('Fetch a URL and return its content')
+    .option('--extract', 'Use extract_content for cleaned text/markdown output')
+    .option('--format <format>', 'Output format: text, markdown, html (default: text)', 'text')
+    .option('--timeout <ms>', 'Request timeout in milliseconds', '10000')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const apiKey = globals.apiKey || process.env.CRAWLFORGE_API_KEY;
+      const timeout = parseInt(opts.timeout, 10);
+
+      if (opts.extract) {
+        const tool = new ExtractContentTool(getToolConfig('extract_content'));
+        const wrapperTool = {
+          execute: (p) => tool.execute(p)
+        };
+        await runTool(wrapperTool, { url, output_format: opts.format, timeout }, cliFlags);
+      } else {
+        const wrapperTool = {
+          execute: (p) => fetchUrlHandler(p)
+        };
+        await runTool(wrapperTool, { url, timeout }, cliFlags);
+      }
+    });
+}

package/src/cli/commands/search.js

@@ -0,0 +1,28 @@
+/**
+ * search command — searches the web using search_web tool.
+ */
+import { SearchWebTool } from '../../tools/search/searchWeb.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('search <query>')
+    .description('Search the web')
+    .option('--limit <n>', 'Number of results', '10')
+    .option('--lang <lang>', 'Language code (e.g. en, fr)', 'en')
+    .option('--provider <p>', 'Search provider: crawlforge or searxng', 'crawlforge')
+    .option('--no-safe-search', 'Disable safe search')
+    .action(async (query, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new SearchWebTool(getToolConfig('search_web'));
+      await runTool(tool, {
+        query,
+        limit: parseInt(opts.limit, 10),
+        lang: opts.lang,
+        provider: opts.provider,
+        safe_search: opts.safeSearch !== false
+      }, cliFlags);
+    });
+}

package/src/cli/commands/stealth.js

@@ -0,0 +1,29 @@
+/**
+ * stealth command — scrape a URL using stealth mode.
+ */
+import { StealthBrowserManager } from '../../core/StealthBrowserManager.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('stealth <url>')
+    .description('Scrape a URL using stealth/anti-bot browser mode')
+    .option('--engine <engine>', 'Browser engine: playwright or camoufox', 'playwright')
+    .option('--wait <ms>', 'Wait time after page load in milliseconds', '2000')
+    .option('--screenshot', 'Capture a screenshot')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const mgr = new StealthBrowserManager(getToolConfig('stealth_mode'));
+      const wrapperTool = {
+        execute: (p) => mgr.scrapeWithStealth(p)
+      };
+      await runTool(wrapperTool, {
+        url,
+        engine: opts.engine,
+        wait_for: parseInt(opts.wait, 10),
+        screenshot: !!opts.screenshot
+      }, cliFlags);
+    });
+}

package/src/cli/commands/template.js

@@ -0,0 +1,26 @@
+/**
+ * template command — scrape a target URL using a pre-built site template.
+ */
+import { ScrapeTemplateTool } from '../../tools/templates/ScrapeTemplateTool.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('template <id> <target>')
+    .description('Scrape using a pre-built site template (e.g. amazon-product, github-repo)')
+    .option('--list', 'List all available templates')
+    .action(async (id, target, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new ScrapeTemplateTool(getToolConfig('scrape_template'));
+
+      if (opts.list) {
+        const wrapperTool = { execute: () => tool.listTemplates() };
+        await runTool(wrapperTool, {}, cliFlags);
+        return;
+      }
+
+      await runTool(tool, { template_id: id, url: target }, cliFlags);
+    });
+}

package/src/cli/commands/track.js

@@ -0,0 +1,24 @@
+/**
+ * track command — track content changes on a URL.
+ */
+import { TrackChangesTool } from '../../tools/tracking/trackChanges/index.js';
+import { getToolConfig } from '../../constants/config.js';
+import { runTool } from '../lib/runTool.js';
+
+export function register(program) {
+  program
+    .command('track <url>')
+    .description('Track content changes on a URL')
+    .option('--selector <css>', 'CSS selector to scope tracking')
+    .option('--threshold <pct>', 'Change threshold percentage (0-100)', '5')
+    .action(async (url, opts, cmd) => {
+      const globals = cmd.parent.opts();
+      const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      const tool = new TrackChangesTool(getToolConfig('track_changes'));
+      await runTool(tool, {
+        url,
+        selector: opts.selector,
+        change_threshold: parseFloat(opts.threshold)
+      }, cliFlags);
+    });
+}

package/src/cli/commands/uninstall-skills.js

@@ -0,0 +1,35 @@
+/**
+ * uninstall-skills command -- remove CrawlForge skill files.
+ */
+import { uninstall } from '../../skills/installer.js';
+
+export function register(program) {
+  program
+    .command('uninstall-skills')
+    .description('Remove CrawlForge skill files from Claude Code, Cursor, or VS Code')
+    .option('--target <target>', 'Target: claude-code, cursor, vscode, or all', 'all')
+    .action(async (opts) => {
+      try {
+        const results = await uninstall({
+          target: opts.target,
+          cwd: process.cwd()
+        });
+
+        if (results.removed.length > 0) {
+          process.stdout.write('Removed:\n');
+          results.removed.forEach(p => process.stdout.write('  ' + p + '\n'));
+        }
+        if (results.notFound.length > 0) {
+          process.stdout.write('Not found (already removed):\n');
+          results.notFound.forEach(p => process.stdout.write('  ' + p + '\n'));
+        }
+        if (results.removed.length === 0) {
+          process.stdout.write('No skill files found to remove.\n');
+        }
+        process.exit(0);
+      } catch (err) {
+        process.stderr.write('Error: ' + err.message + '\n');
+        process.exit(1);
+      }
+    });
+}

package/src/cli/formatter.js

@@ -0,0 +1,57 @@
+/**
+ * formatter.js — CLI output formatter shared across all CLI commands.
+ * Respects global flags: --json, --pretty, --quiet.
+ * No logic duplication with MCP tools — formats the same tool execute() output.
+ */
+
+/**
+ * Format a tool result for CLI output.
+ * @param {object} result  — raw object from tool.execute() or MCP handler
+ * @param {{ json?: boolean, pretty?: boolean, quiet?: boolean }} flags
+ * @returns {string}
+ */
+export function formatResult(result, flags = {}) {
+  const { json = false, pretty = false, quiet = false } = flags;
+
+  if (quiet) return '';
+
+  // If result has MCP content array, extract the text
+  if (result && Array.isArray(result.content)) {
+    const texts = result.content
+      .filter(c => c.type === 'text')
+      .map(c => c.text);
+
+    if (json || pretty) {
+      // Try to parse each text as JSON and re-serialize
+      const parsed = texts.map(t => {
+        try { return JSON.parse(t); } catch { return t; }
+      });
+      const output = parsed.length === 1 ? parsed[0] : parsed;
+      return pretty
+        ? JSON.stringify(output, null, 2)
+        : JSON.stringify(output);
+    }
+
+    // Plain text: return text blocks joined
+    return texts.join('\n');
+  }
+
+  // Plain object
+  if (json) return JSON.stringify(result);
+  if (pretty) return JSON.stringify(result, null, 2);
+  return typeof result === 'string' ? result : JSON.stringify(result, null, 2);
+}
+
+/**
+ * Format an error for CLI output.
+ * @param {Error|string} error
+ * @param {{ json?: boolean }} flags
+ * @returns {string}
+ */
+export function formatError(error, flags = {}) {
+  const message = error instanceof Error ? error.message : String(error);
+  if (flags.json) {
+    return JSON.stringify({ error: message });
+  }
+  return `Error: ${message}`;
+}

package/src/cli/index.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+/**
+ * CrawlForge CLI — src/cli/index.js
+ * Entry point for the `crawlforge` command.
+ *
+ * Global flags:
+ *   --json        Output raw JSON (compact)
+ *   --pretty      Output pretty-printed JSON
+ *   --quiet       Suppress all output (exit code only)
+ *   --api-key     CrawlForge API key (overrides CRAWLFORGE_API_KEY env)
+ *   --timeout     Global request timeout in ms (default: 30000)
+ */
+
+import { Command } from 'commander';
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import { dirname, join } from 'node:path';
+import { readFileSync } from 'node:fs';
+
+// Load package.json for version
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const pkgPath = join(__dirname, '../../package.json');
+let version = '4.1.0';
+try {
+  const pkg = JSON.parse(readFileSync(pkgPath, 'utf8'));
+  version = pkg.version;
+} catch { /* use fallback */ }
+
+// Import all command registrars
+import { register as registerScrape } from './commands/scrape.js';
+import { register as registerSearch } from './commands/search.js';
+import { register as registerCrawl } from './commands/crawl.js';
+import { register as registerMap } from './commands/map.js';
+import { register as registerExtract } from './commands/extract.js';
+import { register as registerTrack } from './commands/track.js';
+import { register as registerAnalyze } from './commands/analyze.js';
+import { register as registerResearch } from './commands/research.js';
+import { register as registerStealth } from './commands/stealth.js';
+import { register as registerBatch } from './commands/batch.js';
+import { register as registerActions } from './commands/actions.js';
+import { register as registerLocalize } from './commands/localize.js';
+import { register as registerLlmstxt } from './commands/llmstxt.js';
+import { register as registerTemplate } from './commands/template.js';
+import { register as registerMonitor } from './commands/monitor.js';
+import { register as registerInstallSkills } from './commands/install-skills.js';
+import { register as registerUninstallSkills } from './commands/uninstall-skills.js';
+
+const program = new Command();
+
+program
+  .name('crawlforge')
+  .description('CrawlForge CLI — web scraping, crawling, and content processing')
+  .version(version)
+  .option('--json', 'Output compact JSON')
+  .option('--pretty', 'Output pretty-printed JSON')
+  .option('--quiet', 'Suppress all stdout output (exit code only)')
+  .option('--api-key <key>', 'CrawlForge API key (overrides CRAWLFORGE_API_KEY env var)')
+  .option('--timeout <ms>', 'Global request timeout in milliseconds', '30000');
+
+// Apply --api-key globally before commands run
+program.hook('preAction', (thisCommand) => {
+  const opts = program.opts();
+  if (opts.apiKey) {
+    process.env.CRAWLFORGE_API_KEY = opts.apiKey;
+  }
+  if (opts.timeout) {
+    process.env.CRAWLFORGE_CLI_TIMEOUT = opts.timeout;
+  }
+});
+
+// Register all 15 tool commands + 2 skills commands
+registerScrape(program);
+registerSearch(program);
+registerCrawl(program);
+registerMap(program);
+registerExtract(program);
+registerTrack(program);
+registerAnalyze(program);
+registerResearch(program);
+registerStealth(program);
+registerBatch(program);
+registerActions(program);
+registerLocalize(program);
+registerLlmstxt(program);
+registerTemplate(program);
+registerMonitor(program);
+registerInstallSkills(program);
+registerUninstallSkills(program);
+
+program.parseAsync(process.argv).catch((err) => {
+  process.stderr.write(`Fatal error: ${err.message}\n`);
+  process.exit(1);
+});

package/src/cli/lib/runTool.js

@@ -0,0 +1,40 @@
+/**
+ * runTool.js — Thin wrapper that invokes a tool's execute() method directly
+ * and formats the output according to global CLI flags.
+ *
+ * This intentionally does NOT replicate withAuth credit logic — CLI invocations
+ * go through the same AuthManager path as MCP calls when a real API key is set.
+ * In creator mode (CRAWLFORGE_CREATOR_SECRET set) credits are skipped automatically.
+ */
+
+import { formatResult, formatError } from '../formatter.js';
+
+/**
+ * Run a tool and print formatted output.
+ * @param {object} tool         — tool instance with execute(params) method
+ * @param {object} params       — tool parameters
+ * @param {object} cliFlags     — { json, pretty, quiet }
+ * @param {object} [options]
+ * @param {boolean} [options.exitOnError=true]
+ */
+export async function runTool(tool, params, cliFlags, options = {}) {
+  const { exitOnError = true } = options;
+
+  try {
+    const result = await tool.execute(params);
+
+    // Check for MCP-style error response
+    if (result && result.isError) {
+      const errText = result.content?.[0]?.text ?? 'Tool returned an error';
+      process.stderr.write(formatError(errText, cliFlags) + '\n');
+      if (exitOnError) process.exit(1);
+      return;
+    }
+
+    const output = formatResult(result, cliFlags);
+    if (output) process.stdout.write(output + '\n');
+  } catch (error) {
+    process.stderr.write(formatError(error, cliFlags) + '\n');
+    if (exitOnError) process.exit(1);
+  }
+}

package/src/core/ActionExecutor.js

@@ -202,13 +202,15 @@ export class ActionExecutor extends EventEmitter {
       this.activeChains.set(chainId, executionContext);
       this.emit('chainStarted', executionContext);
 
-      // Initialize browser and navigate to page
-      const page = await this.initializePage(url, browserOptions);
-      executionContext.page = page;
-
+      // D2.4: initialize page INSIDE try/finally so it is always closed even on
+      // errors thrown between acquisition and the inner try block.
+      let page = null;
       let chainResult;
       
       try {
+        page = await this.initializePage(url, browserOptions);
+        executionContext.page = page;
+
         // Execute chain with potential retries
         chainResult = await this.executeChainWithRetries(executionContext);
         
@@ -235,9 +237,9 @@ export class ActionExecutor extends EventEmitter {
 
         throw error;
       } finally {
-        // Clean up page
+        // D2.4: always close page to prevent leaks
         if (page) {
-          await page.close();
+          try { await page.close(); } catch (_) { /* ignore close errors */ }
         }
         
         // Update execution time