crawlforge-mcp-server 4.7.2 → 4.8.0

package/src/core/MonitorScheduler.js

@@ -0,0 +1,281 @@
+/**
+ * MonitorScheduler — recurring change-monitoring engine.
+ *
+ * Lifecycle:
+ *   start()        load persisted monitors -> rehydrate baseline -> schedule
+ *                  setInterval timers -> catch-up fire anything already due.
+ *   _fire(def)     fetch -> (first run) create baseline, else compareWithBaseline
+ *                  -> significance threshold gate -> optional plain-English goal
+ *                  judge (SamplingClient, degrades gracefully) -> notify -> persist.
+ *   runDueOnce()   fire everything currently due exactly once (the external-cron
+ *                  one-shot path; guaranteed firing regardless of process uptime).
+ *   stopAll()      clear every timer (graceful shutdown — no leaked handles).
+ *
+ * Honest firing model: a stdio MCP server is not a daemon. In-process timers only
+ * fire while the process is alive; missed runs are caught up on next start(). For
+ * guaranteed firing, run `crawlforge monitor:run-due` from system cron.
+ *
+ * All dependencies are injected so the engine is unit-testable without network,
+ * timers, or an LLM.
+ */
+import { fetchContent, meetsNotificationThreshold } from '../tools/tracking/trackChanges/differ.js';
+import { sendNotifications } from '../tools/tracking/trackChanges/notifier.js';
+
+export const FIRING_GUARANTEE_NOTE =
+  'In-process scheduling fires only while the MCP server process is alive; missed runs are ' +
+  'caught up on restart. For guaranteed firing, run `crawlforge monitor:run-due` from system cron.';
+
+const MIN_INTERVAL = 60_000;
+
+export class MonitorScheduler {
+  constructor({ tool, store, samplingClient = null, logger = null, now = () => Date.now() } = {}) {
+    this.tool = tool;
+    this.store = store;
+    this.samplingClient = samplingClient;
+    this.logger = logger || { warn() {}, info() {}, error() {} };
+    this.now = now;
+    this.timers = new Map();
+    this._started = false;
+  }
+
+  async start() {
+    if (this._started) return;
+    this._started = true;
+    await this.store.load();
+    for (const def of this.store.list()) {
+      if (def.enabled === false) continue;
+      try {
+        await this._ensureBaseline(def);
+      } catch (err) {
+        this.logger.warn('monitor baseline rehydrate failed', { id: def.id, error: err.message });
+      }
+      this._schedule(def);
+      if (!def.nextDueAt || def.nextDueAt <= this.now()) {
+        this._fire(def).catch(() => {});
+      }
+    }
+  }
+
+  _schedule(def) {
+    this._clearTimer(def.id);
+    if (def.enabled === false) return;
+    const interval = Math.max(MIN_INTERVAL, def.interval || 3_600_000);
+    const timer = setInterval(() => {
+      this._fire(def).catch(() => {});
+    }, interval);
+    if (typeof timer.unref === 'function') timer.unref(); // never keep the process alive
+    this.timers.set(def.id, timer);
+  }
+
+  _clearTimer(id) {
+    const t = this.timers.get(id);
+    if (t) {
+      clearInterval(t);
+      this.timers.delete(id);
+    }
+  }
+
+  stopAll() {
+    for (const t of this.timers.values()) clearInterval(t);
+    this.timers.clear();
+  }
+
+  async createMonitor(input) {
+    if (!input || !input.url) throw new Error('createMonitor requires a url');
+    const interval = Math.max(MIN_INTERVAL, input.interval || 3_600_000);
+    const t = this.now();
+    const def = {
+      id: this.store.newId(),
+      url: input.url,
+      interval,
+      schedule: input.schedule || null,
+      goal: input.goal || null,
+      notificationThreshold: input.notificationThreshold || 'moderate',
+      trackingOptions: input.trackingOptions || {},
+      notificationOptions: input.notificationOptions || null,
+      enabled: true,
+      createdAt: t,
+      nextDueAt: t + interval,
+      lastCheckAt: null,
+      lastChangeAt: null,
+      stats: { checks: 0, changesDetected: 0, notificationsSent: 0, errors: 0 },
+    };
+    await this.store.save(def);
+    try {
+      await this._ensureBaseline(def);
+    } catch {
+      /* baseline will be created on first fire */
+    }
+    this._schedule(def);
+    return { ...def, firingGuarantee: FIRING_GUARANTEE_NOTE };
+  }
+
+  async stopMonitor(id) {
+    this._clearTimer(id);
+    const existed = !!this.store.get(id);
+    await this.store.remove(id);
+    return { stopped: existed, id };
+  }
+
+  async stopByUrl(url) {
+    let count = 0;
+    for (const def of this.store.list()) {
+      if (def.url === url) {
+        await this.stopMonitor(def.id);
+        count++;
+      }
+    }
+    return { stopped: count, url };
+  }
+
+  list() {
+    return this.store.list().map((d) => ({
+      id: d.id,
+      url: d.url,
+      interval: d.interval,
+      goal: d.goal,
+      enabled: d.enabled,
+      nextDueAt: d.nextDueAt,
+      lastCheckAt: d.lastCheckAt,
+      lastChangeAt: d.lastChangeAt,
+      stats: d.stats,
+      scheduled: this.timers.has(d.id),
+    }));
+  }
+
+  async runDueOnce() {
+    if (!this.store._loaded) await this.store.load();
+    const t = this.now();
+    const due = this.store.list().filter((d) => d.enabled !== false && (!d.nextDueAt || d.nextDueAt <= t));
+    const results = [];
+    for (const def of due) {
+      try {
+        results.push(await this._fire(def));
+      } catch (err) {
+        results.push({ id: def.id, url: def.url, error: err.message });
+      }
+    }
+    return { fired: results.length, results };
+  }
+
+  async _ensureBaseline(def) {
+    const ct = this.tool.changeTracker;
+    if (ct?.snapshots?.has(def.url)) return;
+    try {
+      const q = await this.tool.snapshotManager.querySnapshots({ url: def.url, limit: 1, includeContent: true });
+      const content = q?.snapshots?.[0]?.content;
+      if (content && typeof content === 'string') {
+        await ct.createBaseline(def.url, content, def.trackingOptions);
+      }
+    } catch {
+      /* no usable snapshot — first fire will create the baseline */
+    }
+  }
+
+  async _fire(def) {
+    const ct = this.tool.changeTracker;
+    const finish = async (extra) => {
+      def.lastCheckAt = this.now();
+      def.nextDueAt = this.now() + Math.max(MIN_INTERVAL, def.interval || 3_600_000);
+      await this.store.save(def);
+      return { id: def.id, url: def.url, ...extra };
+    };
+
+    let fetched;
+    try {
+      fetched = await fetchContent(def.url);
+    } catch (err) {
+      def.stats.errors++;
+      return finish({ error: err.message });
+    }
+    const content = fetched.content;
+
+    // First fire (or unrehydrated) — establish a baseline.
+    if (!ct?.snapshots?.has(def.url)) {
+      await ct.createBaseline(def.url, content, def.trackingOptions);
+      try {
+        await this.tool.snapshotManager.storeSnapshot(def.url, content, { baseline: true, scheduledMonitor: def.id });
+      } catch {
+        /* snapshot persistence best-effort */
+      }
+      def.stats.checks++;
+      return finish({ baselineCreated: true });
+    }
+
+    let cmp;
+    try {
+      cmp = await ct.compareWithBaseline(def.url, content, def.trackingOptions);
+    } catch (err) {
+      def.stats.errors++;
+      return finish({ error: err.message });
+    }
+
+    def.stats.checks++;
+    const meets = cmp.hasChanges && meetsNotificationThreshold(cmp.significance, def.notificationThreshold);
+    let judge = null;
+    let notified = false;
+
+    if (meets) {
+      def.stats.changesDetected++;
+      def.lastChangeAt = this.now();
+      judge = await this._judgeGoal(def, cmp);
+      try {
+        await this.tool.snapshotManager.storeSnapshot(def.url, content, {
+          changes: cmp.summary,
+          significance: cmp.significance,
+          scheduledMonitor: def.id,
+        });
+      } catch {
+        /* best-effort */
+      }
+      if (judge.meaningful && def.notificationOptions) {
+        try {
+          await sendNotifications(def.url, { ...cmp, goalJudgment: judge }, def.notificationOptions, this.tool);
+          def.stats.notificationsSent++;
+          notified = true;
+        } catch (err) {
+          def.stats.errors++;
+          this.logger.warn('monitor notification failed', { id: def.id, error: err.message });
+        }
+      }
+    }
+
+    return finish({
+      hasChanges: cmp.hasChanges,
+      significance: cmp.significance,
+      notified,
+      mode: judge?.mode || 'threshold',
+      reason: judge?.reason,
+    });
+  }
+
+  /**
+   * Decide whether a detected change is "meaningful" given the monitor's
+   * plain-English goal. Degrades gracefully: no goal -> threshold; no LLM ->
+   * notify and tag the mode; never hard-requires an LLM key.
+   */
+  async _judgeGoal(def, cmp) {
+    if (!def.goal) return { meaningful: true, mode: 'threshold' };
+    if (!this.samplingClient) return { meaningful: true, mode: 'degraded-no-llm' };
+    try {
+      const summary = JSON.stringify(cmp.summary || {}).slice(0, 1500);
+      const prompt =
+        `A monitored web page changed. The user's alert goal is:\n"${def.goal}"\n\n` +
+        `Change summary (JSON):\n${summary}\n\n` +
+        `Does this change match the goal and warrant an alert? ` +
+        `Reply ONLY with JSON: {"meaningful": true|false, "reason": "short reason"}.`;
+      const res = await this.samplingClient.complete(prompt, { maxTokens: 200 });
+      const text = typeof res === 'string' ? res : res?.text || '';
+      const m = text.match(/\{[\s\S]*\}/);
+      if (m) {
+        const parsed = JSON.parse(m[0]);
+        return { meaningful: parsed.meaningful !== false, reason: parsed.reason || '', mode: 'llm' };
+      }
+      return { meaningful: true, mode: 'degraded-llm-error', reason: 'unparseable judge output' };
+    } catch (err) {
+      return { meaningful: true, mode: 'degraded-llm-error', reason: err.message };
+    }
+  }
+}
+
+export default MonitorScheduler;

package/src/core/MonitorStore.js

@@ -0,0 +1,79 @@
+/**
+ * MonitorStore — disk persistence for scheduled change-monitors.
+ *
+ * One JSON file per monitor under ./monitors/<id>.json. Mirrors JobManager's
+ * persistence *pattern* (mkdir-recursive, per-file JSON, randomUUID, load-on-
+ * start) but deliberately omits TTL/eviction — scheduled monitors are long-lived
+ * and must never be auto-expired.
+ */
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+
+export class MonitorStore {
+  constructor({ storageDir = './monitors' } = {}) {
+    this.storageDir = storageDir;
+    this.monitors = new Map();
+    this._loaded = false;
+  }
+
+  async load() {
+    try {
+      await fs.mkdir(this.storageDir, { recursive: true });
+      const files = await fs.readdir(this.storageDir);
+      for (const f of files) {
+        if (!f.endsWith('.json')) continue;
+        try {
+          const def = JSON.parse(await fs.readFile(path.join(this.storageDir, f), 'utf8'));
+          if (def && def.id) this.monitors.set(def.id, def);
+        } catch {
+          /* skip corrupt file */
+        }
+      }
+    } catch {
+      /* dir unavailable — start empty */
+    }
+    this._loaded = true;
+    return this.monitors;
+  }
+
+  newId() {
+    return randomUUID();
+  }
+
+  async save(def) {
+    this.monitors.set(def.id, def);
+    try {
+      await fs.mkdir(this.storageDir, { recursive: true });
+      await fs.writeFile(
+        path.join(this.storageDir, `${def.id}.json`),
+        JSON.stringify(def, null, 2),
+        'utf8'
+      );
+    } catch (err) {
+      /* keep the in-memory copy even if the write fails */
+      return { def, persisted: false, error: err.message };
+    }
+    return { def, persisted: true };
+  }
+
+  async remove(id) {
+    this.monitors.delete(id);
+    try {
+      await fs.unlink(path.join(this.storageDir, `${id}.json`));
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  get(id) {
+    return this.monitors.get(id);
+  }
+
+  list() {
+    return [...this.monitors.values()];
+  }
+}
+
+export default MonitorStore;

package/src/core/ResearchOrchestrator.js

@@ -8,6 +8,7 @@ import { ResultRanker } from '../tools/search/ranking/ResultRanker.js';
 import { CacheManager } from './cache/CacheManager.js';
 import { Logger } from '../utils/Logger.js';
 import { LLMManager } from './llm/LLMManager.js';
+import { safeFetch } from '../utils/ssrfGuard.js';
 
 /**
  * ResearchOrchestrator - Multi-stage research orchestration engine with LLM integration
@@ -552,7 +553,7 @@ export class ResearchOrchestrator extends EventEmitter {
               });
               // Fallback: use fetch + basic text extraction
               try {
-                const fetchResponse = await fetch(source.link, {
+                const fetchResponse = await safeFetch(source.link, {
                   headers: { 'User-Agent': 'CrawlForge-Research/1.0' },
                   signal: AbortSignal.timeout(10000)
                 });

package/src/core/crawlers/BFSCrawler.js

@@ -7,6 +7,7 @@ import { DomainFilter } from '../../utils/domainFilter.js';
 import { LinkAnalyzer } from '../analysis/LinkAnalyzer.js';
 import { normalizeUrl, extractLinks, isValidUrl } from '../../utils/urlNormalizer.js';
 import { Logger } from '../../utils/Logger.js';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const logger = new Logger('BFSCrawler');
 
@@ -284,7 +285,7 @@ export class BFSCrawler {
         setTimeout(() => controller.abort(), effectiveTimeout);
       }
 
-      const response = await fetch(url, {
+      const response = await safeFetch(url, {
         signal: controller.signal,
         headers
       });

package/src/skills/agent-skills/crawlforge-batch-automation/SKILL.md

@@ -0,0 +1,126 @@
+---
+name: crawlforge-batch-automation
+description: "Automates large scraping jobs and browser interactions with CrawlForge's batch_scrape, get_batch_results, scrape_with_actions, and generate_llms_txt tools. Use when the user wants to scrape many URLs at once, batch-scrape a list of pages, collect dozens of product, news, or competitor pages, run browser actions (click, type, scroll, wait) before scraping, log in or fill a form before extracting, or generate an llms.txt for a site. Use sync mode for up to about 25 URLs and async with a webhook for large batches; retrieve paginated output with get_batch_results."
+metadata:
+  version: 4.8.0
+  source: crawlforge-mcp-server
+---
+
+# CrawlForge Batch & Automation
+
+Scale up scraping and drive interactive pages. Use `batch_scrape` for many URLs,
+`get_batch_results` to page through async output, `scrape_with_actions` to
+interact before scraping, and `generate_llms_txt` to produce a site's AI policy
+file.
+
+## When to use
+
+- "Scrape these 30 URLs" / "batch-scrape this list" → `batch_scrape`
+- "Collect dozens of product / news / competitor pages" → `batch_scrape` (async)
+- "Get the rest of the results from that batch" → `get_batch_results`
+- "Click / type / scroll / wait before scraping" / "log in then extract" →
+  `scrape_with_actions`
+- "Generate an llms.txt for this site" → `generate_llms_txt`
+
+## batch_scrape — many URLs in parallel (cost: 5)
+
+Sync mode (results returned immediately), good for up to ~25 URLs:
+
+```json
+{
+  "tool": "batch_scrape",
+  "params": {
+    "urls": ["https://a.com", "https://b.com", "https://c.com"],
+    "formats": ["markdown"],
+    "mode": "sync",
+    "maxConcurrency": 5
+  }
+}
+```
+
+Async mode with a webhook for large batches (returns a `batchId` immediately):
+
+```json
+{
+  "tool": "batch_scrape",
+  "params": {
+    "urls": ["https://a.com", "https://b.com"],
+    "formats": ["json"],
+    "mode": "async",
+    "webhook": { "url": "https://my-site.com/hook", "events": ["batch_completed", "batch_failed"] }
+  }
+}
+```
+
+- `urls` accepts plain strings OR objects `{url, selectors, headers, timeout,
+  metadata}` for per-URL config. 1–50 URLs.
+- `formats`: `markdown`, `html`, `json`, `text`.
+- `extractionSchema` applies structured extraction to every URL.
+- `maxConcurrency` 1–20 (default 10); `delayBetweenRequests` throttles.
+- Sync batches over ~25 URLs trigger a confirmation prompt (elicitation) — use
+  async for large jobs.
+
+CLI: `crawlforge batch urls.txt --format markdown --concurrency 10`.
+
+## get_batch_results — page through output (cost: 1)
+
+```json
+{ "tool": "get_batch_results", "params": { "batchId": "batch_1234567890_abc", "page": 2, "pageSize": 25 } }
+```
+
+Use the `batchId` from `batch_scrape` to retrieve paginated results for a
+completed or in-progress job. Cheap (1 credit) because the batch was already
+paid for. Completed jobs are also exposed as `crawlforge://job/{jobId}`
+resources.
+
+## scrape_with_actions — interact, then scrape (cost: 5)
+
+For SPAs, login-gated content, or multi-step flows. Run an ordered list of
+browser actions before extraction.
+
+```json
+{
+  "tool": "scrape_with_actions",
+  "params": {
+    "url": "https://app.com/login",
+    "actions": [
+      { "type": "type", "selector": "#email", "text": "user@a.com" },
+      { "type": "type", "selector": "#password", "text": "secret" },
+      { "type": "click", "selector": "#login" },
+      { "type": "wait", "duration": 1500 }
+    ],
+    "formats": ["markdown"]
+  }
+}
+```
+
+Allowed action types: `wait`, `click`, `type`, `press`, `scroll`, `screenshot`,
+`executeJavaScript`. `executeJavaScript` is disabled unless the deploy sets
+`ALLOW_JAVASCRIPT_EXECUTION=true`. 1–20 actions per call. Screenshots are stored
+as `crawlforge://screenshot/{actionId}` resources. Full action schemas:
+[actions](references/actions.md). CLI:
+`crawlforge actions https://example.com --script login.json --screenshot`.
+
+## generate_llms_txt — AI policy file (cost: 5)
+
+```json
+{ "tool": "generate_llms_txt", "params": { "url": "https://example.com", "format": "both" } }
+```
+
+Generates `llms.txt` (and optionally `llms-full.txt`) describing how AI models
+should interact with a site. `format`: `both`, `llms-txt`, `llms-full-txt`.
+Tune `analysisOptions` (`maxDepth`, `maxPages`, `respectRobots`) and
+`outputOptions` (`contactEmail`, `organizationName`, custom guidelines). CLI:
+`crawlforge llmstxt https://example.com --include-full > llms.txt`.
+
+## Sync vs async decision
+
+- **≤ ~25 URLs, need results now** → `batch_scrape` `mode:"sync"`.
+- **Large list or long-running** → `mode:"async"` + `webhook`, then
+  `get_batch_results` (or read the `crawlforge://job/{jobId}` resource).
+
+## Cost note
+
+`batch_scrape`, `scrape_with_actions`, `generate_llms_txt` = 5 credits each;
+`get_batch_results` = 1 (retrieval of an already-paid batch). Async batches are
+billed once at submission — paging results afterward stays cheap.

package/src/skills/agent-skills/crawlforge-batch-automation/references/actions.md

@@ -0,0 +1,127 @@
+# scrape_with_actions — Action Types
+
+`scrape_with_actions` runs an ordered `actions[]` array (1–20 items) before
+scraping. Only these 7 action types are allowed (allow-listed in ActionExecutor).
+Each action object has a `type` plus type-specific fields. Common optional
+fields on every action: `timeout`, `description`, `continueOnError`, `retries`
+(0–5), `captureAfter`.
+
+## 1. wait
+
+Pause or wait for a condition.
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `duration` | number | Milliseconds to wait (0–30000). |
+| `selector` | string | Element to wait on. |
+| `condition` | enum | `visible`, `hidden`, `enabled`, `disabled`, `stable`. |
+
+```json
+{ "type": "wait", "duration": 1500 }
+{ "type": "wait", "selector": "#results", "condition": "visible" }
+```
+
+## 2. click
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `selector` | string | Element to click. |
+| `button` | enum | `left`, `right`, `middle`. |
+| `clickCount` | number | 1–3. |
+| `delay` | number | ms (0–1000). |
+| `force` | boolean | Bypass actionability checks. |
+| `position` | object | `{x, y}` relative offset. |
+
+```json
+{ "type": "click", "selector": "#login" }
+```
+
+## 3. type
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `selector` | string | Input to type into. |
+| `text` | string | Text to enter. |
+| `clear` | boolean | Clear the field first. |
+| `delay` | number | Per-keystroke delay (ms). |
+
+```json
+{ "type": "type", "selector": "#email", "text": "user@a.com", "clear": true }
+```
+
+## 4. press
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `key` | string | Key to press (e.g. `Enter`). |
+| `modifiers` | enum[] | `Alt`, `Control`, `Meta`, `Shift`. |
+
+```json
+{ "type": "press", "key": "Enter" }
+```
+
+## 5. scroll
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `direction` | enum | `up`, `down`, `left`, `right`. |
+| `distance` | number | Pixels. |
+| `smooth` | boolean | Smooth scrolling. |
+| `toElement` | string | Selector to scroll to. |
+
+```json
+{ "type": "scroll", "direction": "down", "distance": 800 }
+```
+
+## 6. screenshot
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `fullPage` | boolean | Capture full page. |
+| `format` | enum | `png`, `jpeg`. |
+| `quality` | number | 0–100 (jpeg). |
+
+Saved as a `crawlforge://screenshot/{actionId}` resource.
+
+```json
+{ "type": "screenshot", "fullPage": true, "format": "png" }
+```
+
+## 7. executeJavaScript (gated)
+
+Disabled unless the deployment sets `ALLOW_JAVASCRIPT_EXECUTION=true`.
+
+| Field | Type | Notes |
+|-------|------|-------|
+| `script` | string | JS to run in page context. |
+| `args` | any[] | Arguments passed to the script. |
+| `returnResult` | boolean | Return the script's result. |
+
+```json
+{ "type": "executeJavaScript", "script": "return document.title", "returnResult": true }
+```
+
+## Top-level options
+
+| Option | Default | Notes |
+|--------|---------|-------|
+| `formats` | `["json"]` | `markdown`, `html`, `json`, `text`, `screenshots`. |
+| `captureIntermediateStates` | `false` | Snapshot after each action. |
+| `captureScreenshots` | `true` | Screenshot during execution. |
+| `formAutoFill` | — | Declarative form fill: `fields[]` + `submitSelector`. |
+| `browserOptions` | — | `headless`, `userAgent`, `viewportWidth/Height`, `timeout`. |
+| `continueOnActionError` | `false` | Keep going if one action fails. |
+| `maxRetries` | `1` | 0–3 retries on failure. |
+| `screenshotOnError` | `true` | Capture a screenshot when an error occurs. |
+
+## CLI action-script format
+
+```json
+[
+  { "type": "click", "selector": "#button" },
+  { "type": "type", "selector": "#input", "text": "hello" },
+  { "type": "wait", "duration": 1000 }
+]
+```
+
+Run with `crawlforge actions <url> --script ./flow.json --screenshot`.