crawlforge-mcp-server 4.7.1 → 4.8.0

package/src/tools/tracking/trackChanges/schema.js

@@ -7,7 +7,7 @@
 import { z } from 'zod';
 
 export const TrackChangesSchema = z.object({
-  url: z.string().url(),
+  url: z.string().url().optional(),
   operation: z.enum([
     'create_baseline',
     'compare',
@@ -16,6 +16,7 @@ export const TrackChangesSchema = z.object({
     'get_stats',
     'create_scheduled_monitor',
     'stop_scheduled_monitor',
+    'list_scheduled_monitors',
     'get_dashboard',
     'export_history',
     'create_alert_rule',
@@ -100,7 +101,11 @@ export const TrackChangesSchema = z.object({
   scheduledMonitorOptions: z.object({
     schedule: z.string().optional(),
     templateId: z.string().optional(),
-    enabled: z.boolean().default(true)
+    enabled: z.boolean().default(true),
+    interval: z.number().min(60000).optional(),
+    goal: z.string().optional(),
+    monitorId: z.string().optional(),
+    notificationThreshold: z.enum(['minor', 'moderate', 'major', 'critical']).optional()
   }).optional(),
 
   alertRuleOptions: z.object({

package/src/utils/hostRateLimiter.js

@@ -0,0 +1,46 @@
+/**
+ * Shared per-host outbound rate limiter (politeness / abuse protection).
+ *
+ * Throttles outbound scraping requests per target host so a single tool call
+ * (batch_scrape, map_site, the basic fetch path, etc.) cannot hammer one origin.
+ * Mirrors the per-domain limiter BFSCrawler already uses, driven by the shared
+ * config so all paths agree on a default.
+ *
+ * Backwards-compatible: default 10 req/s + 100 req/min per host (the existing
+ * effective behaviour), enabled by RATE_LIMIT_PER_DOMAIN (default true). Setting
+ * RATE_LIMIT_PER_DOMAIN=false disables the throttle entirely — there is no global
+ * cross-host cap, so broad multi-host crawls are never slowed by this.
+ */
+import { RateLimiter } from './rateLimiter.js';
+import { config } from '../constants/config.js';
+
+let _limiter = null;
+function limiter() {
+  if (!_limiter) {
+    _limiter = new RateLimiter({
+      requestsPerSecond: config.rateLimit.requestsPerSecond,
+      requestsPerMinute: config.rateLimit.requestsPerMinute,
+      perDomain: true,
+    });
+  }
+  return _limiter;
+}
+
+/**
+ * Wait (if necessary) until another request to this URL's host is allowed.
+ * Never throws — a limiter failure must not block a legitimate fetch.
+ * @param {string} url
+ */
+export async function throttleHost(url) {
+  if (config.rateLimit.perDomain === false) return; // feature disabled
+  try {
+    await limiter().checkLimit(url);
+  } catch {
+    /* never block a fetch on a limiter error */
+  }
+}
+
+/** Test/diagnostic hook. */
+export function _resetHostRateLimiter() {
+  _limiter = null;
+}

package/src/utils/robotsChecker.js

@@ -1,4 +1,5 @@
 import robotsParser from 'robots-parser';
+import { safeFetch } from './ssrfGuard.js';
 
 export class RobotsChecker {
   constructor(userAgent = 'CrawlForge/1.0') {
@@ -32,7 +33,7 @@ export class RobotsChecker {
       const controller = new AbortController();
       const timeoutId = setTimeout(() => controller.abort(), 5000);
       
-      const response = await fetch(robotsUrl, {
+      const response = await safeFetch(robotsUrl, {
         signal: controller.signal,
         headers: {
           'User-Agent': this.userAgent

package/src/utils/sitemapParser.js

@@ -3,6 +3,7 @@ import zlib from 'zlib';
 import { promisify } from 'util';
 import { CacheManager } from '../core/cache/CacheManager.js';
 import { normalizeUrl } from './urlNormalizer.js';
+import { safeFetch } from './ssrfGuard.js';
 
 const gunzip = promisify(zlib.gunzip);
 
@@ -632,7 +633,7 @@ export class SitemapParser {
     const timeoutId = setTimeout(() => controller.abort(), this.timeout);
 
     try {
-      const response = await fetch(url, {
+      const response = await safeFetch(url, {
         signal: controller.signal,
         headers: {
           'User-Agent': this.userAgent,

package/src/utils/ssrfGuard.js

@@ -0,0 +1,161 @@
+/**
+ * SSRF guard for the live outbound fetch path.
+ *
+ * This wires the (previously unused) SSRF protections into the actual scraping
+ * fetch helpers. Enforcement happens at TCP connect time via a custom undici
+ * dispatcher `lookup`, so it covers the initial request, every redirect hop, and
+ * closes the DNS-rebinding window (the validated IP is the one connected to —
+ * there is no second, unchecked resolution).
+ *
+ * Two levels:
+ *  - Stage 1 (default): blocks connections to loopback, link-local /
+ *    cloud-metadata (169.254.0.0/16, incl. 169.254.169.254), and 0.0.0.0.
+ *    These are never legitimate public-scrape targets, so impact is ~zero.
+ *  - Stage 2 (SSRF_STRICT=true): full private-range enforcement (RFC1918, ULA,
+ *    multicast, CGNAT, etc.) via the existing SSRFProtection range logic.
+ *
+ * Controls (backwards-compatible defaults):
+ *  - SSRF_PROTECTION_ENABLED=false  -> disable the guard entirely (kill switch).
+ *  - ALLOWED_DOMAINS=a.com,b.com    -> bypass the guard for trusted hosts
+ *    (e.g. a local dev server at localhost). Matches host or any subdomain.
+ *  - SSRF_STRICT=true               -> Stage 2 full enforcement.
+ */
+import dns from 'node:dns';
+import { Agent } from 'undici';
+import { config } from '../constants/config.js';
+import { SSRFProtection } from './ssrfProtection.js';
+
+// Reused only for its (well-tested) CIDR range math — no network state.
+const _ssrf = new SSRFProtection();
+
+// Narrow Stage-1 ranges: things no legitimate public scrape ever targets.
+const STAGE1_RANGES = ['127.0.0.0/8', '169.254.0.0/16', '0.0.0.0/8', '::1/128', 'fe80::/10'];
+
+// Literal cloud-metadata / service-discovery hostnames blocked even before DNS.
+const METADATA_HOSTS = new Set(['metadata.google.internal', 'metadata.azure.com', 'metadata']);
+
+function strictMode() {
+  return process.env.SSRF_STRICT === 'true';
+}
+
+/**
+ * Whether a resolved IP must be blocked for the current mode.
+ * @param {string} ip
+ * @returns {boolean}
+ */
+export function ipBlocked(ip) {
+  if (strictMode()) {
+    // Full enforcement: anything not explicitly allowed by SSRFProtection.
+    return !_ssrf.isIPAllowed(ip);
+  }
+  if (ip === '127.0.0.1' || ip === '::1' || ip === '0.0.0.0') return true;
+  return STAGE1_RANGES.some((range) => _ssrf.isIPInRange(ip, range));
+}
+
+/**
+ * undici connect-time lookup: resolves the host, rejects if ANY resolved address
+ * is blocked, otherwise hands undici the validated address(es) — so the socket
+ * connects to exactly what we checked (rebinding-safe).
+ */
+function ssrfLookup(hostname, opts, callback) {
+  dns.lookup(hostname, { all: true, verbatim: true }, (err, addresses) => {
+    if (err) return callback(err);
+    for (const { address } of addresses) {
+      if (ipBlocked(address)) {
+        return callback(
+          Object.assign(
+            new Error(`SSRF Protection: ${hostname} resolves to blocked address ${address}`),
+            { code: 'SSRF_BLOCKED' }
+          )
+        );
+      }
+    }
+    if (opts && opts.all) return callback(null, addresses);
+    const first = addresses[0];
+    callback(null, first.address, first.family);
+  });
+}
+
+let _agent = null;
+function guardedDispatcher() {
+  if (!_agent) {
+    _agent = new Agent({ connect: { lookup: ssrfLookup } });
+  }
+  return _agent;
+}
+
+function isAllowlisted(host, allowed) {
+  return (allowed || []).some((d) => {
+    const dd = String(d).trim().toLowerCase();
+    return dd && (host === dd || host.endsWith('.' + dd));
+  });
+}
+
+/**
+ * Pre-flight check + dispatcher selection for an outbound scrape target.
+ * Returns `{ dispatcher }` to spread into fetch options. `dispatcher` is
+ * undefined when the guard is disabled or the host is explicitly allowlisted.
+ * Throws (code SSRF_BLOCKED) for protocol / metadata-host pre-flight violations.
+ *
+ * @param {string} url
+ * @returns {{ dispatcher?: import('undici').Agent }}
+ */
+export function ssrfGuard(url) {
+  const sec = config.security?.ssrfProtection;
+  if (!sec || sec.enabled === false) return {}; // kill switch -> default fetch behavior
+
+  let u;
+  try {
+    u = new URL(url);
+  } catch {
+    return {}; // let fetch surface its own invalid-URL error
+  }
+
+  if (!['http:', 'https:'].includes(u.protocol)) {
+    throw Object.assign(new Error(`SSRF Protection: protocol '${u.protocol}' is not allowed`), {
+      code: 'SSRF_BLOCKED',
+    });
+  }
+
+  const host = u.hostname.toLowerCase();
+  if (isAllowlisted(host, sec.allowedDomains)) return {}; // explicit escape hatch
+
+  if (METADATA_HOSTS.has(host)) {
+    throw Object.assign(new Error(`SSRF Protection: blocked metadata host '${host}'`), {
+      code: 'SSRF_BLOCKED',
+    });
+  }
+
+  return { dispatcher: guardedDispatcher() };
+}
+
+/** True if an error (or its fetch `cause`) came from the SSRF guard. */
+export function isSsrfError(err) {
+  return err?.code === 'SSRF_BLOCKED' || err?.cause?.code === 'SSRF_BLOCKED';
+}
+
+/**
+ * Drop-in replacement for global `fetch` that applies the SSRF guard.
+ * Behaviour-preserving for allowed URLs: all options pass through unchanged and
+ * the native Response is returned; only a guarded dispatcher is injected. For
+ * blocked targets it throws a clear `SSRF Protection: ...` error (pre-flight) or
+ * the fetch rejects at connect time with an SSRF_BLOCKED cause.
+ *
+ * @param {string} url
+ * @param {RequestInit} [options]
+ * @returns {Promise<Response>}
+ */
+export async function safeFetch(url, options = {}) {
+  const guard = ssrfGuard(url); // throws on protocol / metadata-host violations
+  try {
+    return await fetch(url, { ...options, ...guard });
+  } catch (err) {
+    if (isSsrfError(err)) {
+      throw new Error(err.cause?.message || err.message);
+    }
+    throw err;
+  }
+}
+
+// Exposed for unit tests.
+export const __ssrfInternals = { ssrfLookup, isAllowlisted, STAGE1_RANGES };

package/src/utils/ssrfProtection.js

@@ -414,14 +414,7 @@ export class SSRFProtection {
       return false;
     }
   }
-  /**
 
-  /**
-   * Check for path traversal patterns in raw URL before parsing
-   * @param {string} url - Raw URL to check
-   * @returns {Object} - Result with violations array
-   */
-   * Validate URL path for suspicious patterns
   /**
    * Check for path traversal patterns in raw URL before parsing
    * @param {string} url - Raw URL to check
@@ -454,9 +447,13 @@ export class SSRFProtection {
     
     return { violations };
   }
-   * @param {string} path 
+
+  /**
+   * Validate URL path for suspicious patterns
+   * @param {string} path
    * @returns {Object}
-   */  validatePath(path) {
+   */
+  validatePath(path) {
     const suspiciousPatterns = [
       /\.\.\//, // Directory traversal
       /\/etc\//, // System files

package/src/skills/crawlforge-cli.md

@@ -1,157 +0,0 @@
-# CrawlForge CLI Usage Guide
-
-The `crawlforge` CLI exposes all 23 MCP tools as command-line subcommands.
-
-## Installation
-
-```bash
-npm install -g crawlforge-mcp-server
-# or run without installing:
-npx crawlforge-mcp-server <command>
-```
-
-## Global Flags
-
-All commands support these flags:
-- `--json` — output compact JSON
-- `--pretty` — output pretty-printed JSON
-- `--quiet` — suppress output (exit code only)
-- `--api-key <key>` — override CRAWLFORGE_API_KEY env var
-- `--timeout <ms>` — global request timeout (default: 30000)
-
-## Commands
-
-### scrape — Fetch a URL
-```bash
-crawlforge scrape https://example.com
-crawlforge scrape https://example.com --extract --format markdown
-crawlforge scrape https://example.com --pretty
-```
-
-### search — Search the web
-```bash
-crawlforge search "MCP server tutorial" --limit 10
-crawlforge search "nodejs scraping" --provider searxng --json
-```
-
-### crawl — Deep website crawl
-```bash
-crawlforge crawl https://docs.example.com --depth 3 --max-pages 200
-crawlforge crawl https://example.com --no-robots --concurrency 20
-```
-
-### map — Generate sitemap
-```bash
-crawlforge map https://example.com --pretty
-crawlforge map https://example.com --format xml > sitemap.xml
-```
-
-### extract — Structured data extraction
-```bash
-# Schema-based extraction
-crawlforge extract https://example.com/product --schema product-schema.json
-
-# LLM-guided extraction
-crawlforge extract https://example.com/article --prompt "extract title, author, date, summary"
-```
-
-### track — Track content changes
-```bash
-crawlforge track https://example.com --threshold 10
-crawlforge track https://example.com --selector ".main-content"
-```
-
-### analyze — Content analysis
-```bash
-crawlforge analyze https://example.com --depth full --pretty
-```
-
-### research — Deep research
-```bash
-crawlforge research "state of AI in 2025" --depth deep --max-urls 30
-crawlforge research "competitor pricing" --output-format detailed --json
-```
-
-### stealth — Anti-bot scraping
-```bash
-crawlforge stealth https://protected-site.com
-crawlforge stealth https://protected-site.com --engine camoufox --screenshot
-```
-
-### batch — Batch scrape from file
-```bash
-# Create a URLs file:
-cat > urls.txt << EOF
-https://example.com/page1
-https://example.com/page2
-https://example.com/page3
-EOF
-
-crawlforge batch urls.txt --format markdown --concurrency 10
-```
-
-### actions — Browser automation
-```bash
-# Create an actions script:
-cat > login.json << EOF
-[
-  { "type": "click", "selector": "#login-btn" },
-  { "type": "type", "selector": "#email", "text": "user@example.com" },
-  { "type": "wait", "duration": 1000 }
-]
-EOF
-
-crawlforge actions https://example.com --script login.json --screenshot
-```
-
-### localize — Geo-targeted fetch
-```bash
-crawlforge localize https://example.com --locale fr-FR --country FR
-crawlforge localize https://shop.example.com --locale en-GB --currency GBP
-```
-
-### llmstxt — Generate llms.txt
-```bash
-crawlforge llmstxt https://example.com
-crawlforge llmstxt https://example.com --include-full > llms.txt
-```
-
-### template — Pre-built site scrapers
-```bash
-crawlforge template github-repo https://github.com/owner/repo
-crawlforge template amazon-product https://amazon.com/dp/B0XXXXX
-crawlforge template npm-package https://npmjs.com/package/commander
-crawlforge template --list  # list all available templates
-```
-
-### monitor — Continuous change monitoring
-```bash
-crawlforge monitor https://example.com --interval 60 --webhook https://my-site.com/hook
-crawlforge monitor https://example.com --selector ".price" --threshold 1
-```
-
-### install-skills — Install AI assistant skills
-```bash
-crawlforge install-skills --target claude-code
-crawlforge install-skills --target cursor --force
-crawlforge install-skills --target all --dry-run
-```
-
-### uninstall-skills — Remove AI assistant skills
-```bash
-crawlforge uninstall-skills --target claude-code
-crawlforge uninstall-skills --target all
-```
-
-## Output Piping Examples
-
-```bash
-# Extract markdown and save to file
-crawlforge scrape https://example.com --extract --format markdown > page.md
-
-# Search and parse with jq
-crawlforge search "nodejs MCP" --json | jq '.results[].url'
-
-# Batch scrape and process results
-crawlforge batch urls.txt --json | jq '.results | length'
-```

package/src/skills/crawlforge-mcp.md

@@ -1,80 +0,0 @@
-# CrawlForge MCP Tools — When and How to Use
-
-CrawlForge is a professional MCP server with 23 tools for web scraping, crawling, content extraction, research, and AI compliance.
-
-## When to Use MCP Tools vs CLI
-
-Use MCP tools when you need results inline within an AI assistant session.
-Use the CLI (`crawlforge <command>`) for scripts, CI, and automation pipelines.
-
-## All 23 Tools
-
-### Basic Fetching (5 tools)
-- **fetch_url** — Raw HTTP fetch; returns headers + body. Use for quick single-URL fetches.
-- **extract_text** — Clean readable text from a page (strips HTML). Use for reading articles.
-- **extract_links** — All links from a page with anchor text. Use for link analysis.
-- **extract_metadata** — Title, description, OG tags, schema.org from a page.
-- **scrape_structured** — CSS-selector based data extraction from a page.
-
-### Search (1 tool)
-- **search_web** — Search via CrawlForge API or SearXNG. Supports query expansion, ranking, dedup.
-
-### Crawling (2 tools)
-- **crawl_deep** — BFS crawl up to 1000 pages with configurable depth, content extraction, link analysis.
-- **map_site** — Fast sitemap generation via sitemap.xml or crawl. Returns URL list with metadata.
-
-### Content Extraction (7 tools)
-- **extract_content** — Main content extraction with Readability, markdown output, image handling.
-- **process_document** — PDF, DOCX, TXT processing with chunking and metadata.
-- **summarize_content** — Abstractive summarization (via Ollama/API/sampling).
-- **analyze_content** — Sentiment, entities, readability, keyword density, topic detection.
-- **extract_structured** — JSON schema-driven extraction with LLM or CSS selectors.
-- **extract_with_llm** — Natural language prompt-based extraction. Fallback: Ollama → API keys → sampling.
-- **list_ollama_models** — List locally available Ollama models.
-
-### Advanced (2 tools)
-- **batch_scrape** — Scrape multiple URLs concurrently. Default output: markdown (RAG-ready).
-- **scrape_with_actions** — Browser automation (click, type, scroll, wait) before scraping.
-
-### Research (1 tool)
-- **deep_research** — Multi-stage research: query expansion → parallel fetch → dedup → synthesis.
-
-### Tracking (1 tool)
-- **track_changes** — Snapshot URL and diff against baseline. Returns change percentage + diff.
-
-### LLMs.txt (1 tool)
-- **generate_llms_txt** — Generate llms.txt and llms-full.txt for AI compliance.
-
-### Stealth (1 tool)
-- **stealth_mode** — Anti-bot browser scraping. Engines: playwright (default) or camoufox.
-
-### Localization (1 tool)
-- **localization** — Fetch with locale/geo targeting, proxy routing, currency awareness.
-
-### Templates (1 tool)
-- **scrape_template** — Pre-built extractors for: amazon-product, linkedin-profile, github-repo, youtube-video, tweet, reddit-thread, hacker-news-front-page, producthunt-launch, stackoverflow-question, npm-package.
-
-## Cost Reference (Credits)
-- fetch_url, extract_text, extract_links, extract_metadata: 1 credit
-- search_web, map_site: 2 credits
-- extract_content, scrape_structured, analyze_content, summarize_content: 3 credits
-- crawl_deep, batch_scrape, track_changes, generate_llms_txt: 5 credits
-- extract_structured, extract_with_llm, stealth_mode, localization, scrape_with_actions: 5 credits
-- deep_research: 10–50 credits (dynamic, triggers elicitation when >50)
-
-## Example Tool Calls
-
-Fetch a page:
-```json
-{ "tool": "fetch_url", "params": { "url": "https://example.com" } }
-```
-
-Search the web:
-```json
-{ "tool": "search_web", "params": { "query": "MCP server Node.js", "limit": 5 } }
-```
-
-Extract markdown from an article:
-```json
-{ "tool": "extract_content", "params": { "url": "https://example.com/article", "output_format": "markdown" } }
-```

package/src/skills/crawlforge-research.md

@@ -1,104 +0,0 @@
-# CrawlForge Deep Research Workflow
-
-## When to Use deep_research
-
-Use `deep_research` for comprehensive topic research that requires multiple sources:
-- Competitive analysis (compare multiple competitors)
-- Technology landscape research
-- Fact-gathering with citations
-- Market research with multiple data points
-- Any topic requiring 5+ web sources synthesized
-
-Do NOT use for:
-- Single URL content extraction → use `extract_content`
-- Simple web searches → use `search_web`
-- Known URLs you want to read → use `fetch_url` or `batch_scrape`
-
-## How deep_research Works
-
-1. **Query Expansion** — Generates 3–5 related queries from your topic
-2. **Parallel Fetching** — Fetches up to `max_urls` sources simultaneously
-3. **URL Deduplication** — Skips already-visited URLs within the session
-4. **Content Extraction** — Extracts clean text from each source
-5. **Synthesis** — If Ollama/API key available: returns synthesized report; otherwise returns raw evidence for the calling LLM to synthesize
-
-## LLM Fallback Chain
-
-```
-Ollama (local, default) → OpenAI API key → Anthropic API key → MCP Sampling → Raw evidence
-```
-
-With no LLM configured, `deep_research` returns structured raw evidence that Claude or another LLM can synthesize.
-
-## MCP Tool Usage
-
-```json
-// Standard research
-{
-  "tool": "deep_research",
-  "params": {
-    "query": "React vs Vue vs Angular in 2025",
-    "depth": "standard",
-    "max_urls": 20
-  }
-}
-
-// Deep research with all sources
-{
-  "tool": "deep_research",
-  "params": {
-    "query": "competitor pricing analysis for B2B SaaS",
-    "depth": "deep",
-    "max_urls": 50,
-    "output_format": "detailed"
-  }
-}
-```
-
-Note: When `max_urls > 50`, the tool triggers an elicitation asking for confirmation before proceeding (cost guard).
-
-## CLI Usage
-
-```bash
-# Standard research
-crawlforge research "React vs Vue in 2025" --depth standard
-
-# Deep research with JSON output
-crawlforge research "B2B SaaS pricing trends" --depth deep --max-urls 30 --json
-
-# Save research report to file
-crawlforge research "competitor analysis" --pretty > research-report.json
-```
-
-## Depth Levels
-
-| Depth | URLs Analyzed | Use Case | Approx. Credits |
-|-------|--------------|----------|-----------------|
-| basic | 5–10 | Quick overview | 10–15 |
-| standard | 15–25 | General research | 15–30 |
-| deep | 30–75 | Comprehensive analysis | 30–75+ |
-
-## Cost Management
-
-- `deep_research` costs 10 base credits + 1 per URL analyzed
-- Elicitation fires when projected cost > 50 credits
-- Use `max_urls` to cap costs: `max_urls: 10` ≈ 20 credits max
-- Token budget auto-limits LLM synthesis costs (default: 200,000 chars)
-
-## Accessing Research Results as Resources
-
-Completed research sessions are available as MCP Resources:
-```
-crawlforge://research/{sessionId}
-```
-
-List via `resources/list` — no need to re-run the research.
-
-## Combining with Other Tools
-
-For targeted competitive research:
-```
-1. search_web "competitor X pricing"         → get URLs
-2. batch_scrape [competitor URLs]             → get content in parallel
-3. deep_research "competitor X vs us"         → synthesized analysis
-```