crawlforge-mcp-server 4.7.2 → 4.8.0

package/src/skills/installer.js

@@ -1,16 +1,32 @@
 /**
  * installer.js — Skills installer for CrawlForge.
- * Installs skill markdown files into Claude Code, Cursor, or VS Code.
+ *
+ * Installs CrawlForge's Claude Agent Skills into Claude Code, Cursor, or VS Code.
  *
  * Targets:
- *   claude-code  — ~/.claude/skills/crawlforge-*.md (one file per skill)
- *   cursor       — .cursor/rules/crawlforge.mdc (concatenated)
+ *   claude-code  — ~/.claude/skills/<skill-name>/SKILL.md  (real Agent Skill
+ *                  folders with YAML frontmatter, so they auto-activate)
+ *   cursor       — .cursor/rules/crawlforge.mdc            (concatenated bodies)
  *   vscode       — .github/instructions/crawlforge.instructions.md (concatenated)
  *
- * Idempotent: skips if already installed (use --force to overwrite).
+ * Source of truth: src/skills/agent-skills/<skill-name>/SKILL.md (+ references/).
+ *
+ * Idempotent: skips if already installed (use --force to overwrite). Installing /
+ * uninstalling also removes any leftover bare crawlforge-*.md files written by
+ * pre-4.8.0 versions (migration), without touching unrelated user skills.
  */
 
-import { readFileSync, writeFileSync, mkdirSync, existsSync, unlinkSync } from 'node:fs';
+import {
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  existsSync,
+  unlinkSync,
+  readdirSync,
+  statSync,
+  cpSync,
+  rmSync,
+} from 'node:fs';
 import { join, dirname } from 'node:path';
 import { homedir } from 'node:os';
 import { fileURLToPath } from 'node:url';
@@ -18,51 +34,123 @@ import { fileURLToPath } from 'node:url';
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 
-// Skill files shipped with the package
-const SKILL_FILES = [
+// New source of truth: one folder per skill, each containing a SKILL.md.
+const AGENT_SKILLS_DIR = join(__dirname, 'agent-skills');
+
+// Pre-4.8.0 bare files that may linger in ~/.claude/skills (migration cleanup).
+const LEGACY_SKILL_FILES = [
   'crawlforge-mcp.md',
   'crawlforge-cli.md',
   'crawlforge-stealth.md',
-  'crawlforge-research.md'
+  'crawlforge-research.md',
 ];
 
-const SKILL_DIR = __dirname; // src/skills/
+/**
+ * Discover the shipped Agent Skills.
+ * @returns {{ name: string, srcDir: string, skillMd: string }[]}
+ */
+export function listAgentSkills() {
+  if (!existsSync(AGENT_SKILLS_DIR)) {
+    throw new Error(`Agent skills directory not found: ${AGENT_SKILLS_DIR}`);
+  }
+  return readdirSync(AGENT_SKILLS_DIR)
+    .filter((name) => !name.startsWith('_') && !name.startsWith('.'))
+    .map((name) => ({ name, srcDir: join(AGENT_SKILLS_DIR, name) }))
+    .filter((s) => {
+      try {
+        return statSync(s.srcDir).isDirectory() && existsSync(join(s.srcDir, 'SKILL.md'));
+      } catch {
+        return false;
+      }
+    })
+    .map((s) => ({ ...s, skillMd: join(s.srcDir, 'SKILL.md') }));
+}
+
+/**
+ * Read a skill's SKILL.md and strip the leading YAML frontmatter block, leaving
+ * just the markdown body (used for the concatenated cursor/vscode outputs).
+ * @param {string} skillMdPath
+ * @returns {string}
+ */
+export function readSkillBody(skillMdPath) {
+  const raw = readFileSync(skillMdPath, 'utf8');
+  const m = raw.match(/^---\n[\s\S]*?\n---\n?/);
+  return (m ? raw.slice(m[0].length) : raw).trim();
+}
 
-function readSkillFile(name) {
-  const p = join(SKILL_DIR, name);
-  if (!existsSync(p)) throw new Error(`Skill file not found: ${p}`);
-  return readFileSync(p, 'utf8');
+/**
+ * Concatenate all skill bodies into a single document (root SKILL.md + the
+ * cursor/vscode single-file targets). Kept named for backwards compatibility.
+ * @returns {string}
+ */
+export function concatenateSkills() {
+  return listAgentSkills()
+    .map((s) => readSkillBody(s.skillMd))
+    .join('\n\n---\n\n');
 }
 
-function concatenateSkills() {
-  return SKILL_FILES.map(f => readSkillFile(f)).join('\n\n---\n\n');
+function copySkillFolder(srcDir, destDir) {
+  cpSync(srcDir, destDir, { recursive: true, force: true });
+}
+
+/**
+ * Remove leftover pre-4.8.0 bare crawlforge-*.md files from a skills dir.
+ * Strictly scoped to the four known filenames — never globs, never touches
+ * unrelated skills or folders.
+ * @param {string} skillsDir
+ * @returns {string[]} removed paths
+ */
+export function cleanupLegacyClaudeSkills(skillsDir) {
+  const removed = [];
+  for (const fname of LEGACY_SKILL_FILES) {
+    const p = join(skillsDir, fname);
+    try {
+      if (existsSync(p) && statSync(p).isFile()) {
+        unlinkSync(p);
+        removed.push(p);
+      }
+    } catch {
+      /* ignore */
+    }
+  }
+  return removed;
 }
 
 /**
  * Install skills into the given target.
- * @param {{ target: 'claude-code'|'cursor'|'vscode'|'all', force?: boolean, dryRun?: boolean, cwd?: string }} opts
+ * @param {{ target?: 'claude-code'|'cursor'|'vscode'|'all', force?: boolean, dryRun?: boolean, cwd?: string, homeDir?: string }} opts
  * @returns {{ installed: string[], skipped: string[], paths: string[] }}
  */
-export async function install({ target = 'all', force = false, dryRun = false, cwd = process.cwd() } = {}) {
+export async function install({
+  target = 'all',
+  force = false,
+  dryRun = false,
+  cwd = process.cwd(),
+  homeDir = homedir(),
+} = {}) {
   const targets = target === 'all' ? ['claude-code', 'cursor', 'vscode'] : [target];
   const results = { installed: [], skipped: [], paths: [] };
+  const skills = listAgentSkills();
 
   for (const t of targets) {
     if (t === 'claude-code') {
-      const skillsDir = join(homedir(), '.claude', 'skills');
-      for (const fname of SKILL_FILES) {
-        const dest = join(skillsDir, fname);
-        results.paths.push(dest);
+      const skillsDir = join(homeDir, '.claude', 'skills');
+      for (const skill of skills) {
+        const destDir = join(skillsDir, skill.name);
+        const destSkillMd = join(destDir, 'SKILL.md');
+        results.paths.push(destSkillMd);
         if (!dryRun) {
-          if (existsSync(dest) && !force) {
-            results.skipped.push(dest);
+          if (existsSync(destSkillMd) && !force) {
+            results.skipped.push(destSkillMd);
             continue;
           }
           mkdirSync(skillsDir, { recursive: true });
-          writeFileSync(dest, readSkillFile(fname), 'utf8');
+          copySkillFolder(skill.srcDir, destDir);
         }
-        results.installed.push(dest);
+        results.installed.push(destSkillMd);
       }
+      // Migration: remove stale bare files from older versions.
+      if (!dryRun) cleanupLegacyClaudeSkills(skillsDir);
     } else if (t === 'cursor') {
       const dir = join(cwd, '.cursor', 'rules');
       const dest = join(dir, 'crawlforge.mdc');
@@ -99,25 +187,27 @@ export async function install({ target = 'all', force = false, dryRun = false, c
 
 /**
  * Uninstall skills from the given target.
- * @param {{ target: 'claude-code'|'cursor'|'vscode'|'all', cwd?: string }} opts
+ * @param {{ target?: 'claude-code'|'cursor'|'vscode'|'all', cwd?: string, homeDir?: string }} opts
  * @returns {{ removed: string[], notFound: string[] }}
  */
-export async function uninstall({ target = 'all', cwd = process.cwd() } = {}) {
+export async function uninstall({ target = 'all', cwd = process.cwd(), homeDir = homedir() } = {}) {
   const targets = target === 'all' ? ['claude-code', 'cursor', 'vscode'] : [target];
   const results = { removed: [], notFound: [] };
+  const skills = listAgentSkills();
 
   for (const t of targets) {
     if (t === 'claude-code') {
-      const skillsDir = join(homedir(), '.claude', 'skills');
-      for (const fname of SKILL_FILES) {
-        const dest = join(skillsDir, fname);
-        if (existsSync(dest)) {
-          unlinkSync(dest);
-          results.removed.push(dest);
+      const skillsDir = join(homeDir, '.claude', 'skills');
+      for (const skill of skills) {
+        const destDir = join(skillsDir, skill.name);
+        if (existsSync(destDir)) {
+          rmSync(destDir, { recursive: true, force: true });
+          results.removed.push(destDir);
         } else {
-          results.notFound.push(dest);
+          results.notFound.push(destDir);
         }
       }
+      results.removed.push(...cleanupLegacyClaudeSkills(skillsDir));
     } else if (t === 'cursor') {
       const dest = join(cwd, '.cursor', 'rules', 'crawlforge.mdc');
       if (existsSync(dest)) {
@@ -139,3 +229,65 @@ export async function uninstall({ target = 'all', cwd = process.cwd() } = {}) {
 
   return results;
 }
+
+// --- Optional, opt-in forced-eval hook (boosts skill auto-activation) ---
+
+const HOOK_MARKER = 'CrawlForge skill';
+const HOOK_COMMAND =
+  "echo 'Consider whether a CrawlForge skill applies: web scraping, deep research, " +
+  "stealth browsing, structured extraction, change tracking, or batch automation.'";
+
+/**
+ * Add a UserPromptSubmit forced-eval reminder to ~/.claude/settings.json.
+ * Idempotent additive merge — preserves all existing settings and only adds the
+ * hook if an equivalent one is not already present. Opt-in (CLI --with-hook).
+ * @param {{ homeDir?: string }} opts
+ * @returns {{ added: boolean, path: string }}
+ */
+export function installHook({ homeDir = homedir() } = {}) {
+  const dir = join(homeDir, '.claude');
+  const path = join(dir, 'settings.json');
+  let settings = {};
+  if (existsSync(path)) {
+    try {
+      settings = JSON.parse(readFileSync(path, 'utf8'));
+    } catch {
+      settings = {};
+    }
+  }
+  settings.hooks = settings.hooks || {};
+  const list = Array.isArray(settings.hooks.UserPromptSubmit)
+    ? settings.hooks.UserPromptSubmit
+    : [];
+  const already = JSON.stringify(list).includes(HOOK_MARKER);
+  if (already) return { added: false, path };
+
+  list.push({ hooks: [{ type: 'command', command: HOOK_COMMAND }] });
+  settings.hooks.UserPromptSubmit = list;
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(path, JSON.stringify(settings, null, 2) + '\n', 'utf8');
+  return { added: true, path };
+}
+
+/**
+ * Remove the CrawlForge forced-eval hook from ~/.claude/settings.json.
+ * @param {{ homeDir?: string }} opts
+ * @returns {{ removed: boolean, path: string }}
+ */
+export function uninstallHook({ homeDir = homedir() } = {}) {
+  const path = join(homeDir, '.claude', 'settings.json');
+  if (!existsSync(path)) return { removed: false, path };
+  let settings;
+  try {
+    settings = JSON.parse(readFileSync(path, 'utf8'));
+  } catch {
+    return { removed: false, path };
+  }
+  const list = settings?.hooks?.UserPromptSubmit;
+  if (!Array.isArray(list)) return { removed: false, path };
+  const filtered = list.filter((entry) => !JSON.stringify(entry).includes(HOOK_MARKER));
+  if (filtered.length === list.length) return { removed: false, path };
+  settings.hooks.UserPromptSubmit = filtered;
+  writeFileSync(path, JSON.stringify(settings, null, 2) + '\n', 'utf8');
+  return { removed: true, path };
+}

package/src/tools/advanced/batchScrape/worker.js

@@ -5,25 +5,31 @@
  */
 
 import { load } from 'cheerio';
+import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
+import { throttleHost } from '../../../utils/hostRateLimiter.js';
 
 const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
 
 /**
- * Fetch a URL with AbortController timeout.
+ * Fetch a URL with AbortController timeout (SSRF-guarded + per-host throttled).
  */
 export async function fetchUrl(url, options = {}) {
   const { timeout = 15000, headers = {} } = options;
+  const guard = ssrfGuard(url); // SSRF pre-flight (throws before connecting)
+  await throttleHost(url);
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
   try {
     const response = await fetch(url, {
       signal: controller.signal,
-      headers: { 'User-Agent': USER_AGENT, ...headers }
+      headers: { 'User-Agent': USER_AGENT, ...headers },
+      ...guard
     });
     clearTimeout(timeoutId);
     return response;
   } catch (error) {
     clearTimeout(timeoutId);
+    if (isSsrfError(error)) throw new Error(error.cause?.message || error.message);
     if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
     throw error;
   }

package/src/tools/basic/_fetch.js

@@ -5,6 +5,8 @@
 
 import { config } from '../../constants/config.js';
 import { createRequire } from 'module';
+import { ssrfGuard, isSsrfError } from '../../utils/ssrfGuard.js';
+import { throttleHost } from '../../utils/hostRateLimiter.js';
 
 // Derive User-Agent from package version so it reflects the actual release.
 const _require = createRequire(import.meta.url);
@@ -27,6 +29,13 @@ export async function fetchWithTimeout(url, options = {}) {
   const { timeout = 10000, headers = {} } = options;
   const maxBodySize = config.fetch.maxBodySize;
 
+  // SSRF pre-flight (protocol / metadata host). Throws a clear error before any
+  // connection is attempted; `guard.dispatcher` enforces IP rules at connect time.
+  const guard = ssrfGuard(url);
+
+  // Per-host politeness throttle (before the timeout window starts).
+  await throttleHost(url);
+
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), timeout);
 
@@ -37,11 +46,15 @@ export async function fetchWithTimeout(url, options = {}) {
       headers: {
         'User-Agent': CRAWLFORGE_UA,
         ...headers
-      }
+      },
+      ...guard
     });
     clearTimeout(timeoutId);
   } catch (error) {
     clearTimeout(timeoutId);
+    if (isSsrfError(error)) {
+      throw new Error(error.cause?.message || error.message);
+    }
     if (error.name === 'AbortError') {
       throw new Error(`Request timeout after ${timeout}ms`);
     }

package/src/tools/crawl/_sessionContext.js

@@ -15,6 +15,8 @@
  *   - Keeping zero new runtime deps satisfies the project constraint.
  */
 
+import { safeFetch } from '../../utils/ssrfGuard.js';
+
 /**
  * Parse a single Set-Cookie header value into a cookie object.
  * Returns null if the header is empty or unparseable.
@@ -220,7 +222,7 @@ export class SessionContext {
       fetchOpts.body = body;
     }
 
-    const response = await fetch(url, fetchOpts);
+    const response = await safeFetch(url, fetchOpts);
     this.recordCookies(response, url);
 
     const text = await response.text().catch(() => '');

package/src/tools/extract/_fetchAndParse.js

@@ -11,6 +11,7 @@
  */
 
 import { load } from 'cheerio';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; CrawlForge-MCP/3.0)';
 const DEFAULT_TIMEOUT_MS = 15000;
@@ -32,7 +33,7 @@ export async function fetchAndParse(url, options = {}) {
     stripTags = ['script', 'style', 'noscript', 'iframe', 'svg']
   } = options;
 
-  const response = await fetch(url, {
+  const response = await safeFetch(url, {
     headers: {
       'User-Agent': userAgent,
       'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'

package/src/tools/extract/extractContent.js

@@ -8,6 +8,7 @@ import { ContentProcessor } from '../../core/processing/ContentProcessor.js';
 import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const ExtractContentSchema = z.object({
   url: z.string().url(),
@@ -169,7 +170,7 @@ export class ExtractContentTool {
         pageTitle = browserResult.title;
       } else {
         // Simple HTTP fetch
-        const response = await fetch(url, {
+        const response = await safeFetch(url, {
           headers: {
             'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/3.0; Enhanced-Content-Extractor)'
           },

package/src/tools/extract/processDocument.js

@@ -9,6 +9,7 @@ import { ContentProcessor } from '../../core/processing/ContentProcessor.js';
 import { BrowserProcessor } from '../../core/processing/BrowserProcessor.js';
 import { HTMLCleaner, ContentQualityAssessor } from '../../utils/contentUtils.js';
 import { htmlToMarkdown } from '../../utils/htmlToMarkdown.js'; // D3.1
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const ProcessDocumentSchema = z.object({
   source: z.string().min(1),
@@ -275,7 +276,7 @@ export class ProcessDocumentTool {
       pageTitle = browserResult.title;
     } else {
       // Simple HTTP fetch
-      const response = await fetch(source, {
+      const response = await safeFetch(source, {
         headers: {
           'User-Agent': 'Mozilla/5.0 (compatible; MCP-WebScraper/3.0; Document-Processor)'
         },