npm - crawlforge-mcp-server - Versions diffs - 5.0.5 → 5.2.0 - Mend

crawlforge-mcp-server 5.0.5 → 5.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (41) hide show

package/CLAUDE.md +9 -6
package/README.md +28 -8
package/package.json +6 -5
package/server.js +56 -16
package/src/core/ActionExecutor.js +246 -66
package/src/core/AuthManager.js +1 -0
package/src/core/ChangeTracker.js +215 -22
package/src/core/ResearchOrchestrator.js +9 -3
package/src/core/SamplingClient.js +4 -5
package/src/core/StealthBrowserManager.js +64 -18
package/src/core/cache/CacheManager.js +7 -2
package/src/core/crawlers/BFSCrawler.js +14 -6
package/src/core/llm/LLMManager.js +61 -11
package/src/core/llm/OllamaProvider.js +139 -0
package/src/core/processing/BrowserProcessor.js +28 -2
package/src/schemas/toolOutputSchemas.js +53 -1
package/src/server/requestContext.js +26 -0
package/src/server/transports/streamableHttp.js +54 -11
package/src/server/withAuth.js +24 -6
package/src/skills/agent-skills/crawlforge-deep-research/SKILL.md +26 -3
package/src/skills/agent-skills/crawlforge-getting-started/SKILL.md +5 -4
package/src/skills/agent-skills/crawlforge-getting-started/references/credits.md +1 -0
package/src/skills/agent-skills/crawlforge-structured-extraction/SKILL.md +6 -4
package/src/skills/agent-skills/crawlforge-structured-extraction/references/templates.md +2 -1
package/src/tools/advanced/ScrapeWithActionsTool.js +4 -1
package/src/tools/basic/_fetch.js +8 -2
package/src/tools/basic/fetchUrl.js +4 -1
package/src/tools/crawl/crawlDeep.js +19 -5
package/src/tools/extract/extractStructured.js +16 -4
package/src/tools/extract/extractWithLlm.js +80 -10
package/src/tools/extract/listOllamaModels.js +4 -6
package/src/tools/scrape/_brandingExtractor.js +1 -1
package/src/tools/scrape/unifiedScrape.js +71 -5
package/src/tools/search/adapters/redditOfficialApi.js +196 -0
package/src/tools/search/redditNormalize.js +95 -0
package/src/tools/search/redditSearch.js +326 -0
package/src/tools/templates/ScrapeTemplateTool.js +8 -3
package/src/utils/hiddenContent.js +330 -0
package/src/utils/htmlToMarkdown.js +12 -2
package/src/utils/ollamaConfig.js +121 -0
package/src/tools/templates/TemplateRegistry.js +0 -325

package/src/utils/hiddenContent.js ADDED Viewed

@@ -0,0 +1,330 @@
+/**
+ * hiddenContent -- remove content that a browser would not paint.
+ *
+ * Markup routinely carries text that is present in the DOM but invisible on
+ * screen: screen-reader-only labels, and theme badges that are hidden until a
+ * state class is applied. Serialising that markup to markdown or plain text
+ * drops the CSS, so the hidden text reads as live page content and downstream
+ * LLM extraction believes it.
+ *
+ * Observed on a Shopify Dawn storefront, whose price block ships every badge
+ * unconditionally and hides them in component CSS:
+ *
+ *   <span class="visually-hidden">Regular price</span>   -> clip-rect hidden
+ *   <span class="price__badge-sold-out">Sold out</span>  -> .price .price__badge-sold-out{display:none}
+ *
+ * Extraction read those and reported availability "Sold out" for a product with
+ * 100 units in stock, and a compare-at price the page never displayed.
+ *
+ * Only rules a browser applies unconditionally are honoured:
+ *   - rules inside @media / @supports / @container are ignored, because they
+ *     depend on viewport or capability (a Tailwind `hidden md:inline-block`
+ *     element is visible on desktop and must survive)
+ *   - a hide rule is skipped when the element also matches a rule that puts
+ *     display back to a visible value (Dawn's `.price--sold-out
+ *     .price__badge-sold-out{display:inline-block}`)
+ *   - selectors carrying interaction pseudo-classes (:hover, :focus) or
+ *     pseudo-elements are ignored, since they describe transient state
+ */
+import { load } from 'cheerio';
+/** Class tokens conventionally used to hide text from sighted users. */
+const SCREEN_READER_CLASSES = [
+  'visually-hidden',
+  'visuallyhidden',
+  'sr-only',
+  'screen-reader-text',
+  'screen-reader-only',
+  'a11y-hidden',
+  'hidden-visually'
+];
+/**
+ * Selectors we never act on even if a rule hides them, to stay conservative.
+ * '*' matters in particular: callers may flatten inline style attributes into
+ * synthetic `*{...}` rules, and honouring that would empty the document.
+ */
+const NEVER_REMOVE = new Set(['html', 'body', 'head', 'main', '*', ':root']);
+/**
+ * An element holding more than this share of the page's text is treated as a
+ * JS-revealed wrapper rather than hidden furniture, and is left alone.
+ */
+const MAX_REMOVAL_FRACTION = 0.3;
+/**
+ * Below this much content the share test is meaningless — in a short fragment a
+ * single badge can be a third of the content — so the wrapper guard is skipped.
+ */
+const MIN_TEXT_FOR_BULK_GUARD = 2000;
+const MIN_MARKUP_FOR_BULK_GUARD = 20000;
+/** Strip CSS comments and any at-rule block whose contents are conditional. */
+function stripConditionalBlocks(css) {
+  let out = css.replace(/\/\*[\s\S]*?\*\//g, '');
+  // Remove @media/@supports/@container blocks wholesale, tracking nesting so a
+  // block containing other rules is removed in full.
+  const conditional = /@(?:media|supports|container)[^{]*\{/gi;
+  let match;
+  while ((match = conditional.exec(out)) !== null) {
+    const start = match.index;
+    let depth = 1;
+    let i = conditional.lastIndex;
+    while (i < out.length && depth > 0) {
+      if (out[i] === '{') depth++;
+      else if (out[i] === '}') depth--;
+      i++;
+    }
+    out = out.slice(0, start) + out.slice(i);
+    conditional.lastIndex = start;
+  }
+  return out;
+}
+/** True when cheerio cannot meaningfully evaluate the selector. */
+function isUnsupportedSelector(selector) {
+  return (
+    // Progressive-enhancement rules. Themes ship <html class="no-js"> and swap
+    // the class to "js" on load, so static markup always looks like the no-JS
+    // case and these rules would hide content every real visitor sees.
+    /(^|[\s.#\[])no-js(\b|[.\[])/.test(selector) ||
+    selector.includes('::') ||
+    /:(hover|focus|focus-within|focus-visible|active|target|checked|disabled|placeholder|before|after|root|host|where|is|not\()/i.test(selector) ||
+    selector.includes('@') ||
+    selector.length === 0
+  );
+}
+/**
+ * Approximate CSS specificity as a single comparable number.
+ * Ids weigh most, then classes/attributes/pseudo-classes, then element names.
+ * Enough to settle the cases that matter here, e.g. Dawn's
+ * `.price .price__badge-sold-out` (two classes) beating a bare `.badge`.
+ * @param {string} selector
+ * @returns {number}
+ */
+function specificity(selector) {
+  const ids = (selector.match(/#[\w-]+/g) || []).length;
+  const classes = (selector.match(/\.[\w-]+|\[[^\]]+\]|:[\w-]+/g) || []).length;
+  const elements = (selector.match(/(?:^|[\s>+~])[a-z][\w-]*/gi) || []).length;
+  return ids * 10000 + classes * 100 + elements;
+}
+/**
+ * Parse CSS into rules that hide content and rules that re-show it, each
+ * carrying the specificity and source order a browser would use to resolve
+ * the conflict.
+ * @param {string} css - Concatenated stylesheet text
+ * @returns {{hide: Array<{selector:string,spec:number,order:number,important:boolean}>, show: Array}}
+ */
+export function collectVisibilitySelectors(css) {
+  const hide = [];
+  const show = [];
+  if (!css) return { hide, show };
+  const flat = stripConditionalBlocks(css);
+  const rule = /([^{}]+)\{([^{}]*)\}/g;
+  let m;
+  let order = 0;
+  while ((m = rule.exec(flat)) !== null) {
+    const selectorList = m[1];
+    const body = m[2];
+    order++;
+    const display = /(?:^|[;{\s])display\s*:\s*([a-z-]+)/i.exec(body);
+    const visibility = /(?:^|[;{\s])visibility\s*:\s*(hidden|collapse)/i.exec(body);
+    // The standard visually-hidden recipe: collapsed to a 1px clipped box.
+    const clipped =
+      /clip\s*:\s*rect\(\s*0[\s,]/i.test(body) ||
+      /clip-path\s*:\s*inset\(\s*50%\s*\)/i.test(body);
+    const hides = (display && display[1].toLowerCase() === 'none') || visibility || clipped;
+    const shows = display && display[1].toLowerCase() !== 'none';
+    if (!hides && !shows) continue;
+    const important = /!\s*important/i.test(body);
+    for (const raw of selectorList.split(',')) {
+      const selector = raw.trim();
+      if (isUnsupportedSelector(selector) || NEVER_REMOVE.has(selector)) continue;
+      const entry = { selector, spec: specificity(selector), order, important };
+      if (hides) hide.push(entry);
+      else show.push(entry);
+    }
+  }
+  return { hide, show };
+}
+/**
+ * True when rule `a` beats rule `b` in the cascade: !important first, then
+ * specificity, then source order.
+ */
+function wins(a, b) {
+  if (a.important !== b.important) return a.important;
+  if (a.spec !== b.spec) return a.spec > b.spec;
+  return a.order > b.order;
+}
+/** Elements whose contents a browser never renders as page text. */
+const NON_RENDERED = 'script, style, noscript, template';
+/**
+ * Size of the content a browser would actually render inside an element.
+ *
+ * Script and style payloads must not count: on a commercial storefront they are
+ * an order of magnitude larger than the visible copy, and including them in the
+ * bulk-removal denominator made a wrapper holding the entire product section
+ * look like a minor fragment of the page.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @param {*} el - Element, or a cheerio selection
+ * @returns {{text: number, markup: number}} lengths in characters
+ */
+function renderedSize($, el) {
+  const $clone = $(el).clone();
+  $clone.find(NON_RENDERED).remove();
+  return {
+    text: $clone.text().replace(/\s+/g, ' ').trim().length,
+    markup: ($clone.html() || '').length
+  };
+}
+/** Gather the text of every inline <style> block in the document. */
+export function inlineStyleText($) {
+  const parts = [];
+  $('style').each((_, el) => {
+    const text = $(el).html();
+    if (text) parts.push(text);
+  });
+  return parts.join('\n');
+}
+/**
+ * Remove browser-invisible content from a cheerio document, in place.
+ *
+ * @param {import('cheerio').CheerioAPI} $ - Parsed document
+ * @param {Object} [options]
+ * @param {string} [options.css] - Extra stylesheet text (e.g. linked sheets)
+ * @param {boolean} [options.useInlineStyles=true] - Honour inline <style> blocks
+ * @param {number} [options.maxRemovalFraction=0.3] - Skip elements holding more
+ *   than this share of the page text (JS-revealed wrappers)
+ * @returns {{removed: number, skippedBulk: number}}
+ */
+export function stripHiddenFromDom($, options = {}) {
+  const { css = '', useInlineStyles = true, maxRemovalFraction = MAX_REMOVAL_FRACTION } = options;
+  let removed = 0;
+  let skippedBulk = 0;
+  // Genuinely invisible furniture is small: a badge, a label, a tooltip. An
+  // element holding a large share of the page is a wrapper that JavaScript
+  // reveals on load, and removing it would delete the page. Two real cases:
+  // Shopify's EasyLockdown app ships the whole storefront inside
+  // <div class="easylockdown-content" style="display:none">, and Next.js
+  // App Router streams the rendered page inside <div id="S:0" hidden> before
+  // moving it into place.
+  //
+  // Share is measured by markup as well as text, because the streamed Next.js
+  // wrapper is only ~8% of the page's text but half its markup — the visible
+  // copy is there while the remaining "text" is script payload.
+  //
+  // Both sides of the ratio count rendered content only. Measuring raw text
+  // put ~58KB of inline script into the denominator on a Shopify storefront,
+  // so the EasyLockdown wrapper — which held the whole product section,
+  // price included — scored 0.27 against a 0.3 threshold and was deleted.
+  const document = renderedSize($, 'body');
+  const documentTextLength = document.text || 1;
+  const documentMarkupLength = document.markup || 1;
+  const guardApplies =
+    documentTextLength >= MIN_TEXT_FOR_BULK_GUARD ||
+    documentMarkupLength >= MIN_MARKUP_FOR_BULK_GUARD;
+  const remove = (el) => {
+    const $el = $(el);
+    if (!$el.length || !$el.parent().length) return;
+    if (guardApplies) {
+      const size = renderedSize($, el);
+      const textShare = size.text / documentTextLength;
+      const markupShare = size.markup / documentMarkupLength;
+      if (Math.max(textShare, markupShare) > maxRemovalFraction) {
+        skippedBulk++;
+        return;
+      }
+    }
+    $el.remove();
+    removed++;
+  };
+  // 1. The hidden attribute.
+  $('[hidden]').each((_, el) => remove(el));
+  // 2. Inline display:none / visibility:hidden.
+  $('[style]').each((_, el) => {
+    const style = ($(el).attr('style') || '').toLowerCase();
+    if (/display\s*:\s*none/.test(style) || /visibility\s*:\s*(hidden|collapse)/.test(style)) {
+      remove(el);
+    }
+  });
+  // 3. Conventional screen-reader-only classes.
+  for (const cls of SCREEN_READER_CLASSES) {
+    $(`.${cls}`).each((_, el) => remove(el));
+  }
+  // 4. Rules from the page's own stylesheets.
+  const sheetText = [useInlineStyles ? inlineStyleText($) : '', css].filter(Boolean).join('\n');
+  if (sheetText) {
+    const { hide, show } = collectVisibilitySelectors(sheetText);
+    if (hide.length) {
+      // Query each distinct selector once, keeping the strongest hide rule.
+      const strongest = new Map();
+      for (const rule of hide) {
+        const prev = strongest.get(rule.selector);
+        if (!prev || wins(rule, prev)) strongest.set(rule.selector, rule);
+      }
+      for (const rule of strongest.values()) {
+        let matches;
+        try {
+          matches = $(rule.selector);
+        } catch {
+          continue; // selector cheerio cannot parse
+        }
+        matches.each((_, el) => {
+          // Keep the element only when a re-showing rule actually wins the
+          // cascade. A bare `.badge{display:inline-block}` must not override
+          // `.price .price__badge-sold-out{display:none}`.
+          const reshown = show.some(s => {
+            if (!wins(s, rule)) return false;
+            try { return $(el).is(s.selector); } catch { return false; }
+          });
+          if (!reshown) remove(el);
+        });
+      }
+    }
+  }
+  return { removed, skippedBulk };
+}
+/**
+ * Remove browser-invisible content from an HTML string.
+ *
+ * @param {string} html
+ * @param {Object} [options] - Same options as stripHiddenFromDom
+ * @returns {string} - HTML with hidden content removed
+ */
+export function stripHiddenHtml(html, options = {}) {
+  if (!html) return html;
+  try {
+    const $ = load(html);
+    stripHiddenFromDom($, options);
+    return $.html();
+  } catch {
+    return html; // never let cleanup break the caller
+  }
+}

package/src/utils/htmlToMarkdown.js CHANGED Viewed

@@ -16,6 +16,7 @@
 import TurndownService from 'turndown';
 import { gfm } from 'turndown-plugin-gfm';
+import { stripHiddenHtml } from './hiddenContent.js';
 let _td = null;
@@ -45,12 +46,21 @@ function getTurndown() {
  * Returns an empty string if html is falsy.
  *
  * @param {string} html
+ * @param {Object} [options]
+ * @param {string} [options.css] - Extra stylesheet text used to resolve visibility
+ * @param {boolean} [options.keepHiddenContent] - Skip the hidden-content strip
  * @returns {string}
  */
-export function htmlToMarkdown(html) {
+export function htmlToMarkdown(html, options = {}) {
   if (!html) return '';
   try {
-    return getTurndown().turndown(html).trim();
+    // Drop content a browser would not paint before converting. Turndown keeps
+    // the text of screen-reader-only labels and state-gated badges, which then
+    // reads as live page content once the CSS is gone.
+    const visible = options.keepHiddenContent
+      ? html
+      : stripHiddenHtml(html, { css: options.css });
+    return getTurndown().turndown(visible).trim();
   } catch {
     // Fallback: strip tags, return plain text
     return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

package/src/utils/ollamaConfig.js ADDED Viewed

@@ -0,0 +1,121 @@
+/**
+ * Shared Ollama endpoint configuration.
+ *
+ * OLLAMA_BASE_URL   — where the Ollama API lives (default http://localhost:11434).
+ *                     Set to https://ollama.com for Ollama Cloud, or to a tunnel
+ *                     URL fronting a self-hosted instance.
+ * OLLAMA_API_KEY    — optional bearer token. Ollama Cloud requires it; a plain
+ *                     local instance ignores auth, so leaving it unset keeps the
+ *                     zero-config localhost behavior.
+ *
+ * Every HTTP call to Ollama (extract_with_llm, list_ollama_models, the
+ * SamplingClient fallback chain) must build its URL and headers from here so a
+ * hosted deployment configures the endpoint once.
+ */
+export function ollamaBaseUrl() {
+  return (process.env.OLLAMA_BASE_URL || 'http://localhost:11434').replace(/\/$/, '');
+}
+/**
+ * @param {Record<string, string>} [extra] headers to merge (e.g. Content-Type)
+ * @returns {Record<string, string>}
+ */
+export function ollamaHeaders(extra = {}) {
+  const apiKey = process.env.OLLAMA_API_KEY;
+  return { ...extra, ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}) };
+}
+// ── Model selection ───────────────────────────────────────────────────────────
+/**
+ * Local models ranked by extraction accuracy, measured 2026-08-25 against three
+ * live product pages (books.toscrape, an Amazon listing, a Shopify storefront)
+ * with independently verified ground truth. Score is correct fields out of 18
+ * over two runs; a page with no compare-at price counts a fabricated one as
+ * wrong.
+ *
+ *   gemma3:4b          18/18   1040ms      45/45 over five runs
+ *   gpt-oss:20b        18/18   3464ms
+ *   gemma3:12b         16/18   3652ms      invents a compare-at price
+ *   mistral:7b         16/18   2377ms      misses the rating
+ *   llama3.2           16/18   1179ms      invents a compare-at price, every run
+ *   qwen2.5:3b         16/18   1067ms      misses the title, every run
+ *   dolphin-llama3:8b  12/18   6288ms
+ *
+ * Parameter count did not predict accuracy: the 4B model beat both the 12B and
+ * the 20B, and was three times faster than either.
+ */
+const PREFERRED_MODELS = [
+  'gemma3:4b',
+  'gpt-oss:20b',
+  'gemma3:12b',
+  'mistral:7b',
+  'llama3.2',
+  'qwen2.5:3b'
+];
+/** Used only when Ollama cannot be reached, so the error names a real model. */
+export const FALLBACK_OLLAMA_MODEL = 'llama3.2';
+/** Installed-model list per base URL. Keyed by URL so a changed endpoint re-probes. */
+const _installedByBaseUrl = new Map();
+/** "llama3.2" and "llama3.2:latest" name the same model. */
+function baseName(name) {
+  return name.replace(/:latest$/, '');
+}
+/**
+ * Names of the models installed on the Ollama server, or [] if unreachable.
+ * Cached for the process lifetime — pulling a model mid-session is rare enough
+ * that re-listing on every extraction is not worth the round trip.
+ * @returns {Promise<string[]>}
+ */
+export async function installedOllamaModels() {
+  const url = ollamaBaseUrl();
+  if (!_installedByBaseUrl.has(url)) {
+    _installedByBaseUrl.set(url, (async () => {
+      try {
+        const response = await fetch(`${url}/api/tags`, {
+          headers: ollamaHeaders(),
+          signal: AbortSignal.timeout(3000)
+        });
+        if (!response.ok) return [];
+        const data = await response.json();
+        const models = Array.isArray(data?.models) ? data.models : [];
+        return models.map((m) => m.name).filter(Boolean);
+      } catch {
+        return [];
+      }
+    })());
+  }
+  return _installedByBaseUrl.get(url);
+}
+/**
+ * Choose which Ollama model to extract with.
+ *
+ * The code default used to be llama3.2 regardless of what was installed, and
+ * llama3.2 fabricates values — it invented a compare-at price on a Shopify
+ * product that has none, on every one of five runs. Hardcoding the winner
+ * instead would break anyone who has not pulled it, so the best *installed*
+ * model is chosen, and an explicit OLLAMA_DEFAULT_MODEL always wins.
+ *
+ * @returns {Promise<string>}
+ */
+export async function selectOllamaModel() {
+  const explicit = process.env.OLLAMA_DEFAULT_MODEL;
+  if (explicit) return explicit;
+  const installed = await installedOllamaModels();
+  if (installed.length === 0) return FALLBACK_OLLAMA_MODEL;
+  const byBase = new Map(installed.map((name) => [baseName(name), name]));
+  for (const preferred of PREFERRED_MODELS) {
+    const match = byBase.get(baseName(preferred));
+    if (match) return match;
+  }
+  // Nothing recognised — use whatever is there rather than failing.
+  return installed[0];
+}