crawlforge-mcp-server 3.0.18 → 3.4.0

package/src/tools/advanced/batchScrape/worker.js

@@ -0,0 +1,179 @@
+/**
+ * batchScrape — worker module.
+ * URL fetching, content extraction, format generation.
+ * Used by queue.js (the Semaphore-based batch runner).
+ */
+
+import { load } from 'cheerio';
+
+const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
+
+/**
+ * Fetch a URL with AbortController timeout.
+ */
+export async function fetchUrl(url, options = {}) {
+  const { timeout = 15000, headers = {} } = options;
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: { 'User-Agent': USER_AGENT, ...headers }
+    });
+    clearTimeout(timeoutId);
+    return response;
+  } catch (error) {
+    clearTimeout(timeoutId);
+    if (error.name === 'AbortError') throw new Error(`Request timeout after ${timeout}ms`);
+    throw error;
+  }
+}
+
+/**
+ * Scrape a single URL and return a result object.
+ */
+export async function scrapeUrl(config, options, defaultTimeout) {
+  const startTime = Date.now();
+  try {
+    const response = await fetchUrl(config.url, {
+      headers: config.headers,
+      timeout: config.timeout || defaultTimeout
+    });
+
+    if (!response.ok) throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+
+    const html = await response.text();
+    const $ = load(html);
+
+    const result = {
+      success: true,
+      url: config.url,
+      timestamp: Date.now(),
+      executionTime: Date.now() - startTime,
+      metadata: {
+        status: response.status,
+        contentType: response.headers.get('content-type'),
+        contentLength: html.length,
+        ...(config.metadata || {})
+      }
+    };
+
+    if (options.extractionSchema || config.selectors) {
+      result.extracted = extractStructuredData($, { ...config.selectors, ...options.extractionSchema });
+    }
+
+    result.content = generateFormats($, html, options.formats);
+    return result;
+  } catch (error) {
+    return {
+      success: false,
+      url: config.url,
+      error: error.message,
+      timestamp: Date.now(),
+      executionTime: Date.now() - startTime,
+      metadata: config.metadata || {}
+    };
+  }
+}
+
+function extractStructuredData($, selectors) {
+  const extracted = {};
+  for (const [key, selector] of Object.entries(selectors)) {
+    try {
+      const elements = $(selector);
+      if (elements.length === 0) extracted[key] = null;
+      else if (elements.length === 1) extracted[key] = elements.text().trim();
+      else extracted[key] = elements.map((_, el) => $(el).text().trim()).get();
+    } catch {
+      extracted[key] = { error: `Invalid selector: ${selector}` };
+    }
+  }
+  return extracted;
+}
+
+function generateFormats($, html, formats) {
+  const content = {};
+  if (formats.includes('html')) content.html = html;
+  if (formats.includes('text')) content.text = $('body').text().replace(/\s+/g, ' ').trim();
+  if (formats.includes('markdown')) content.markdown = buildMarkdown($);
+  if (formats.includes('json')) {
+    content.json = {
+      title: $('title').text().trim(),
+      headings: extractHeadings($),
+      links: extractLinks($),
+      images: extractImages($),
+      metadata: extractMetadata($)
+    };
+  }
+  return content;
+}
+
+function buildMarkdown($) {
+  let md = '';
+  const title = $('title').text().trim();
+  if (title) md += `# ${title}\n\n`;
+
+  const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
+  let $body = null;
+  for (const sel of selectors) {
+    $body = $(sel);
+    if ($body.length > 0) break;
+  }
+  if (!$body || $body.length === 0) $body = $('body');
+
+  $body.find('h1').each((_, el) => { md += `# ${$(el).text().trim()}\n\n`; });
+  $body.find('h2').each((_, el) => { md += `## ${$(el).text().trim()}\n\n`; });
+  $body.find('h3').each((_, el) => { md += `### ${$(el).text().trim()}\n\n`; });
+  $body.find('p').each((_, el) => { const t = $(el).text().trim(); if (t) md += `${t}\n\n`; });
+  $body.find('ul li').each((_, el) => { md += `- ${$(el).text().trim()}\n`; });
+  $body.find('ol li').each((_, el) => { md += `1. ${$(el).text().trim()}\n`; });
+
+  return md.trim();
+}
+
+function extractHeadings($) {
+  const headings = [];
+  $('h1, h2, h3, h4, h5, h6').each((_, el) => {
+    headings.push({ level: parseInt(el.name.substring(1)), text: $(el).text().trim(), id: $(el).attr('id') || null });
+  });
+  return headings;
+}
+
+function extractLinks($) {
+  const links = [];
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    const text = $(el).text().trim();
+    if (href && text) links.push({ href, text, title: $(el).attr('title') || null });
+  });
+  return links;
+}
+
+function extractImages($) {
+  const images = [];
+  $('img[src]').each((_, el) => {
+    images.push({
+      src: $(el).attr('src'),
+      alt: $(el).attr('alt') || null,
+      title: $(el).attr('title') || null,
+      width: $(el).attr('width') || null,
+      height: $(el).attr('height') || null
+    });
+  });
+  return images;
+}
+
+function extractMetadata($) {
+  const m = {};
+  m.title = $('title').text().trim();
+  m.description = $('meta[name="description"]').attr('content') || '';
+  m.og = {};
+  $('meta[property^="og:"]').each((_, el) => {
+    m.og[$(el).attr('property').replace('og:', '')] = $(el).attr('content');
+  });
+  m.twitter = {};
+  $('meta[name^="twitter:"]').each((_, el) => {
+    m.twitter[$(el).attr('name').replace('twitter:', '')] = $(el).attr('content');
+  });
+  return m;
+}

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -0,0 +1,188 @@
+/**
+ * recorder.js — Recording and replay support for scrape_with_actions.
+ *
+ * Responsibilities:
+ *   - Validate recording names (path-traversal prevention)
+ *   - Persist recorded action sequences to disk (atomic write)
+ *   - Load saved recordings for replay
+ *   - List available recordings
+ *
+ * The recordings directory is resolved from:
+ *   1. process.env.CRAWLFORGE_HOME_OVERRIDE  (tests only)
+ *   2. os.homedir()
+ *
+ * File layout: <homeDir>/.crawlforge/recordings/<name>.json
+ */
+
+import os from 'os';
+import fs from 'fs/promises';
+import path from 'path';
+
+// Regex enforcing safe recording names — no path separators or special chars.
+const VALID_NAME_RE = /^[a-zA-Z0-9_-]{1,64}$/;
+
+/**
+ * Return the base home directory respecting the test override env var.
+ * @returns {string}
+ */
+function homeDir() {
+  return process.env.CRAWLFORGE_HOME_OVERRIDE || os.homedir();
+}
+
+/**
+ * Return the recordings directory path (not guaranteed to exist).
+ * @returns {string}
+ */
+function recordingsDir() {
+  return path.join(homeDir(), '.crawlforge', 'recordings');
+}
+
+/**
+ * Validate a recording name.
+ * @param {string} name
+ * @throws {Error} if the name is invalid
+ */
+export function validateRecordingName(name) {
+  if (typeof name !== 'string' || !VALID_NAME_RE.test(name)) {
+    throw new Error(
+      `Invalid recording name "${name}". ` +
+      'Names must be 1–64 characters and contain only letters, digits, underscores, or hyphens.'
+    );
+  }
+}
+
+/**
+ * Persist a recorded action sequence to disk atomically.
+ *
+ * @param {string} name - Recording name (validated before writing)
+ * @param {Array<Object>} recordedActions - Array of annotated action entries
+ * @param {Object} [meta] - Optional metadata (original url, timestamp, etc.)
+ * @returns {Promise<string>} Resolved file path
+ */
+export async function saveRecording(name, recordedActions, meta = {}) {
+  validateRecordingName(name);
+
+  const dir = recordingsDir();
+  await fs.mkdir(dir, { recursive: true });
+
+  const filePath = path.join(dir, `${name}.json`);
+  const tmpPath = `${filePath}.tmp`;
+
+  const payload = JSON.stringify(
+    {
+      name,
+      savedAt: new Date().toISOString(),
+      ...meta,
+      recordedActions
+    },
+    null,
+    2
+  );
+
+  // Atomic write: write to .tmp then rename
+  await fs.writeFile(tmpPath, payload, 'utf8');
+  await fs.rename(tmpPath, filePath);
+
+  return filePath;
+}
+
+/**
+ * Load a saved recording from disk.
+ *
+ * @param {string} name - Recording name
+ * @returns {Promise<Object>} Parsed recording object (includes `recordedActions`)
+ * @throws {Error} if the recording does not exist or cannot be parsed
+ */
+export async function loadRecording(name) {
+  validateRecordingName(name);
+
+  const filePath = path.join(recordingsDir(), `${name}.json`);
+
+  let raw;
+  try {
+    raw = await fs.readFile(filePath, 'utf8');
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      throw new Error(`Recording "${name}" not found. Use replayRecording: "__list__" to see available recordings.`);
+    }
+    throw new Error(`Failed to read recording "${name}": ${err.message}`);
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Recording "${name}" is corrupted (invalid JSON): ${err.message}`);
+  }
+
+  if (!Array.isArray(parsed.recordedActions)) {
+    throw new Error(`Recording "${name}" has an unexpected format (missing recordedActions array).`);
+  }
+
+  return parsed;
+}
+
+/**
+ * List all available recording names.
+ *
+ * @returns {Promise<string[]>} Sorted array of recording names (without .json extension)
+ */
+export async function listRecordings() {
+  const dir = recordingsDir();
+
+  let entries;
+  try {
+    entries = await fs.readdir(dir);
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      return [];
+    }
+    throw new Error(`Failed to list recordings: ${err.message}`);
+  }
+
+  return entries
+    .filter(f => f.endsWith('.json') && !f.endsWith('.tmp'))
+    .map(f => f.slice(0, -5)) // strip .json
+    .sort();
+}
+
+/**
+ * Build a recordedActions entry from an action definition and timing info.
+ *
+ * Only the fields meaningful for replay are kept.
+ *
+ * @param {Object} action - Original action object
+ * @param {number} timestampMsSinceStart - ms since recording session started
+ * @returns {Object}
+ */
+export function buildRecordedEntry(action, timestampMsSinceStart) {
+  const entry = {
+    type: action.type,
+    timestamp_ms_since_start: timestampMsSinceStart
+  };
+
+  // Preserve replay-relevant fields per action type
+  if (action.selector !== undefined) entry.selector = action.selector;
+  if (action.text !== undefined) entry.text = action.text;
+  if (action.key !== undefined) entry.key = action.key;
+  if (action.duration !== undefined) entry.duration = action.duration;
+  if (action.url !== undefined) entry.url = action.url;
+  if (action.value !== undefined) entry.value = action.value;
+  if (action.direction !== undefined) entry.direction = action.direction;
+  if (action.distance !== undefined) entry.distance = action.distance;
+  if (action.description !== undefined) entry.description = action.description;
+
+  return entry;
+}
+
+/**
+ * Convert a recorded entry back into an action object suitable for ActionExecutor.
+ *
+ * @param {Object} entry - Recorded entry
+ * @returns {Object} Action object
+ */
+export function recordedEntryToAction(entry) {
+  // Pass through all fields except the recording-specific timestamp
+  const { timestamp_ms_since_start: _ignored, ...action } = entry;
+  return action;
+}

package/src/tools/basic/_fetch.js

@@ -0,0 +1,35 @@
+/**
+ * Shared HTTP fetch helper for basic tools.
+ * Applies an AbortController timeout and a default User-Agent.
+ */
+
+/**
+ * Fetch a URL with a configurable timeout.
+ * @param {string} url
+ * @param {{ timeout?: number, headers?: Record<string,string> }} [options]
+ * @returns {Promise<Response>}
+ */
+export async function fetchWithTimeout(url, options = {}) {
+  const { timeout = 10000, headers = {} } = options;
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      headers: {
+        'User-Agent': 'CrawlForge/1.0.0',
+        ...headers
+      }
+    });
+    clearTimeout(timeoutId);
+    return response;
+  } catch (error) {
+    clearTimeout(timeoutId);
+    if (error.name === 'AbortError') {
+      throw new Error(`Request timeout after ${timeout}ms`);
+    }
+    throw error;
+  }
+}

package/src/tools/basic/extractLinks.js

@@ -0,0 +1,74 @@
+/**
+ * extract_links — Extract all links from a webpage with optional filtering.
+ * Extracted from server.js inline handler.
+ */
+
+import { load } from 'cheerio';
+import { fetchWithTimeout } from './_fetch.js';
+
+/**
+ * @param {{ url: string, filter_external?: boolean, base_url?: string }} params
+ */
+export async function extractLinksHandler({ url, filter_external, base_url }) {
+  try {
+    const response = await fetchWithTimeout(url);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    const $ = load(html);
+
+    const baseUrl = base_url || new URL(url).origin;
+    const pageUrl = new URL(url);
+    const links = [];
+
+    $('a[href]').each((_, element) => {
+      const href = $(element).attr('href');
+      const text = $(element).text().trim();
+
+      if (!href) return;
+
+      let absoluteUrl;
+      let isExternal = false;
+
+      try {
+        if (href.startsWith('http://') || href.startsWith('https://')) {
+          absoluteUrl = href;
+          isExternal = new URL(href).origin !== pageUrl.origin;
+        } else {
+          absoluteUrl = new URL(href, baseUrl).toString();
+          isExternal = false;
+        }
+
+        if (filter_external && isExternal) return;
+
+        links.push({ href: absoluteUrl, text, is_external: isExternal, original_href: href });
+      } catch {
+        // skip invalid URLs
+      }
+    });
+
+    const uniqueLinks = links.filter((link, index, arr) =>
+      arr.findIndex(l => l.href === link.href) === index
+    );
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          links: uniqueLinks,
+          total_count: uniqueLinks.length,
+          internal_count: uniqueLinks.filter(l => !l.is_external).length,
+          external_count: uniqueLinks.filter(l => l.is_external).length,
+          base_url: baseUrl
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract links: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/basic/extractMetadata.js

@@ -0,0 +1,74 @@
+/**
+ * extract_metadata — Extract page metadata (title, description, OG tags, etc.).
+ * Extracted from server.js inline handler.
+ */
+
+import { load } from 'cheerio';
+import { fetchWithTimeout } from './_fetch.js';
+
+/**
+ * @param {{ url: string }} params
+ */
+export async function extractMetadataHandler({ url }) {
+  try {
+    const response = await fetchWithTimeout(url);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    const $ = load(html);
+
+    const title = $('title').text().trim() || $('h1').first().text().trim();
+    const description =
+      $('meta[name="description"]').attr('content') ||
+      $('meta[property="og:description"]').attr('content') || '';
+    const keywords = $('meta[name="keywords"]').attr('content') || '';
+    const canonical = $('link[rel="canonical"]').attr('href') || '';
+
+    const ogTags = {};
+    $('meta[property^="og:"]').each((_, el) => {
+      const property = $(el).attr('property');
+      const content = $(el).attr('content');
+      if (property && content) ogTags[property.replace('og:', '')] = content;
+    });
+
+    const twitterTags = {};
+    $('meta[name^="twitter:"]').each((_, el) => {
+      const name = $(el).attr('name');
+      const content = $(el).attr('content');
+      if (name && content) twitterTags[name.replace('twitter:', '')] = content;
+    });
+
+    const author = $('meta[name="author"]').attr('content') || '';
+    const robots = $('meta[name="robots"]').attr('content') || '';
+    const viewport = $('meta[name="viewport"]').attr('content') || '';
+    const charset =
+      $('meta[charset]').attr('charset') ||
+      $('meta[http-equiv="Content-Type"]').attr('content') || '';
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          title,
+          description,
+          keywords: keywords.split(',').map(k => k.trim()).filter(Boolean),
+          canonical_url: canonical,
+          author,
+          robots,
+          viewport,
+          charset,
+          og_tags: ogTags,
+          twitter_tags: twitterTags,
+          url: response.url
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract metadata: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/basic/extractText.js

@@ -0,0 +1,46 @@
+/**
+ * extract_text — Extract clean text content from HTML.
+ * Extracted from server.js inline handler.
+ */
+
+import { load } from 'cheerio';
+import { fetchWithTimeout } from './_fetch.js';
+
+/**
+ * @param {{ url: string, remove_scripts?: boolean, remove_styles?: boolean }} params
+ */
+export async function extractTextHandler({ url, remove_scripts, remove_styles }) {
+  try {
+    const response = await fetchWithTimeout(url);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    const $ = load(html);
+
+    if (remove_scripts !== false) $('script').remove();
+    if (remove_styles !== false) $('style').remove();
+
+    $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
+
+    const text = $('body').text().replace(/\s+/g, ' ').trim();
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          text,
+          word_count: text.split(/\s+/).filter(w => w.length > 0).length,
+          char_count: text.length,
+          url: response.url
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to extract text: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/basic/fetchUrl.js

@@ -0,0 +1,44 @@
+/**
+ * fetch_url — Basic URL fetching with headers and response handling.
+ * Extracted from server.js inline handler.
+ */
+
+import { fetchWithTimeout } from './_fetch.js';
+
+/**
+ * @param {{ url: string, headers?: Record<string,string>, timeout?: number }} params
+ */
+export async function fetchUrlHandler({ url, headers, timeout }) {
+  try {
+    const response = await fetchWithTimeout(url, {
+      timeout: timeout || 10000,
+      headers: headers || {}
+    });
+
+    const body = await response.text();
+    const responseHeaders = {};
+    response.headers.forEach((value, key) => {
+      responseHeaders[key] = value;
+    });
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          status: response.status,
+          statusText: response.statusText,
+          headers: responseHeaders,
+          body,
+          contentType: response.headers.get('content-type') || 'unknown',
+          size: body.length,
+          url: response.url
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to fetch URL: ${error.message}` }],
+      isError: true
+    };
+  }
+}

package/src/tools/basic/scrapeStructured.js

@@ -0,0 +1,58 @@
+/**
+ * scrape_structured — Extract structured data using CSS selectors.
+ * Extracted from server.js inline handler.
+ */
+
+import { load } from 'cheerio';
+import { fetchWithTimeout } from './_fetch.js';
+
+/**
+ * @param {{ url: string, selectors: Record<string, string> }} params
+ */
+export async function scrapeStructuredHandler({ url, selectors }) {
+  try {
+    const response = await fetchWithTimeout(url);
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const html = await response.text();
+    const $ = load(html);
+    const results = {};
+
+    for (const [fieldName, selector] of Object.entries(selectors)) {
+      try {
+        const elements = $(selector);
+        if (elements.length === 0) {
+          results[fieldName] = null;
+        } else if (elements.length === 1) {
+          results[fieldName] = elements.text().trim();
+        } else {
+          results[fieldName] = elements.map((_, el) => $(el).text().trim()).get();
+        }
+      } catch (selectorError) {
+        results[fieldName] = {
+          error: `Invalid selector: ${selector}`,
+          message: selectorError.message
+        };
+      }
+    }
+
+    return {
+      content: [{
+        type: 'text',
+        text: JSON.stringify({
+          data: results,
+          selectors_used: selectors,
+          elements_found: Object.keys(results).length,
+          url: response.url
+        }, null, 2)
+      }]
+    };
+  } catch (error) {
+    return {
+      content: [{ type: 'text', text: `Failed to scrape structured data: ${error.message}` }],
+      isError: true
+    };
+  }
+}