crawlforge-mcp-server 6.22.3 → 6.23.0

package/src/tools/search/serpRank.js

@@ -179,7 +179,7 @@ export class SerpRankTool {
       device: meta.device,
       depthScanned: meta.depth,
       organicResults: meta.organicCount,
-      seResultsCount: meta.seResultsCount,
+      seResultsCount: meta.seResultsCount ?? undefined, // omitted when unknown (see the adapter)
       checkUrl: meta.checkUrl, // link to view the real SERP
       cost: meta.cost, // USD charged by DataForSEO for this lookup
       checkedAt: new Date().toISOString(),

package/src/tools/tracking/trackChanges/index.js

@@ -304,6 +304,7 @@ export class TrackChangesTool extends EventEmitter {
       details: comparisonResult.details,
       metrics: comparisonResult.metrics,
       recommendations: comparisonResult.recommendations,
+      ...(comparisonResult.ignoredOptions ? { ignoredOptions: comparisonResult.ignoredOptions } : {}),
       ...(comparisonResult.warnings || fetchWarnings.length
         ? { warnings: [...(comparisonResult.warnings || []), ...fetchWarnings] }
         : {}),

package/src/tools/tracking/trackChanges/schema.js

@@ -41,7 +41,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
     // filled in here is indistinguishable from a value the caller passed, so
     // it overrode the template (R24 2.7). ChangeTracker applies these same
     // defaults when the baseline is created.
-    granularity: z.enum(['page', 'section', 'element', 'text']).optional().describe("Default: section"),
+    granularity: z.enum(['page', 'section', 'element', 'text']).optional().describe("Default: section. Fixed when the baseline is created: a compare always diffs at the baseline's granularity (as with customSelectors and excludeSelectors), and one that asks for a different value gets ignoredOptions:[{option, requested, applied}] plus a warnings entry. Recreate the baseline to change it"),
     trackText: z.boolean().optional().describe("Default: true"),
     trackStructure: z.boolean().optional().describe("Default: true"),
     trackAttributes: z.boolean().default(false),
@@ -49,6 +49,7 @@ export const TRACK_CHANGES_INPUT_SHAPE = {
     trackLinks: z.boolean().optional().describe("Default: true"),
     ignoreWhitespace: z.boolean().optional().describe("Default: true"),
     ignoreCase: z.boolean().optional().describe("Default: false"),
+    includeMarkupChanges: z.boolean().optional().describe("Also return details.markupChanges, a diff of the raw HTML (tags, classes, attributes). details.textChanges always diffs the visible text only. Default: false"),
     customSelectors: z.array(z.string()).optional(),
     excludeSelectors: z.array(z.string()).optional().default([
       'script', 'style', 'noscript', '.advertisement', '.ad', '#comments'

package/src/utils/htmlToMarkdown.js

@@ -52,6 +52,63 @@ export function promoteCornerHeaderCells(html) {
   }
 }
 
+const SKIP_LANGUAGES = new Set(['default', 'none']);
+
+/**
+ * Find a code block's language in the markup that names it: `data-language`,
+ * a `language-x`/`lang-x` class, MDN's `brush: x` class, a Sphinx
+ * `highlight-x` wrapper, or MDN's `.example-header .language-name` label.
+ * @param {import('cheerio').Cheerio<any>} pre
+ * @returns {string}
+ */
+function codeLanguage(pre) {
+  const code = pre.children('code').first();
+  const fromData = pre.attr('data-language') || code.attr('data-language');
+  if (fromData) return fromData;
+  for (const cls of [code.attr('class'), pre.attr('class')]) {
+    const m = /(?:^|\s)(?:language|lang)-([\w+#.-]+)/.exec(cls || '') || /brush:\s*([\w+#.-]+)/.exec(cls || '');
+    if (m) return m[1];
+  }
+  const wrapper = pre.closest('[class*="highlight-"]');
+  const m = /(?:^|\s)highlight-([\w+#.-]+)/.exec(wrapper.attr('class') || '');
+  if (m && !SKIP_LANGUAGES.has(m[1])) return m[1];
+  return pre.closest('.code-example').find('.example-header .language-name').first().text().trim();
+}
+
+/**
+ * Sphinx (docs.python.org) highlights code as `<pre>` full of `<span>`s with
+ * no `<code>`, so Turndown treated it as prose and escaped it (`\# comment`,
+ * `spam \= 1`); MDN puts a "js" language label and copy button in a header
+ * before each `<pre>`, which came out as a stray `js` line (live sweep
+ * 2026-10-20, 2.2). Rewrite every `<pre>` as `<pre><code>` holding its plain
+ * text with the language in `data-language` (Readability strips classes but
+ * keeps data attributes), and drop MDN's example headers. Run it before
+ * Readability as well as here, while the classes that name the language exist.
+ * @param {string} html
+ * @returns {string}
+ */
+export function normalizeCodeBlocks(html) {
+  if (!/<pre[\s>]/i.test(html)) return html;
+  try {
+    const $ = load(html);
+    $('pre').each((_, el) => {
+      const pre = $(el);
+      const language = codeLanguage(pre);
+      pre.find('br').replaceWith('\n');
+      const code = $('<code></code>').text(pre.text());
+      if (language) {
+        pre.attr('data-language', language);
+        code.attr('data-language', language);
+      }
+      pre.empty().append(code);
+    });
+    $('.code-example > .example-header').remove();
+    return $.html();
+  } catch {
+    return html;
+  }
+}
+
 const ROW_CONTAINER = /^(table|thead|tbody|tfoot)$/;
 
 /**
@@ -152,6 +209,25 @@ function getTurndown() {
       replacement: content => (content.trim() ? content.trim() + '\n' : '')
     });
 
+    // Every <pre> is a fenced block of its plain text, never escaped prose;
+    // the language comes from normalizeCodeBlocks' data-language.
+    _td.addRule('codeBlock', {
+      filter: 'pre',
+      replacement: (_content, node) => {
+        const text = node.textContent.replace(/\n$/, '');
+        if (!text.trim()) return '';
+        const code = node.firstChild && node.firstChild.nodeName === 'CODE' ? node.firstChild : null;
+        const language =
+          node.getAttribute('data-language') ||
+          (code && (code.getAttribute('data-language') ||
+            (/(?:^|\s)language-(\S+)/.exec(code.getAttribute('class') || '') || [])[1])) ||
+          '';
+        const longestRun = Math.max(0, ...(text.match(/`+/g) || []).map(r => r.length));
+        const fence = '`'.repeat(Math.max(3, longestRun + 1));
+        return '\n\n' + fence + language + '\n' + text + '\n' + fence + '\n\n';
+      }
+    });
+
     // Remove boilerplate elements before converting
     _td.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
   }
@@ -178,7 +254,7 @@ export function htmlToMarkdown(html, options = {}) {
     const visible = options.keepHiddenContent
       ? rows
       : stripHiddenHtml(rows, { css: options.css });
-    return getTurndown().turndown(promoteCornerHeaderCells(visible)).trim();
+    return getTurndown().turndown(promoteCornerHeaderCells(normalizeCodeBlocks(visible))).trim();
   } catch {
     // Fallback: strip tags, return plain text
     return html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();

package/src/utils/localFilePolicy.js

@@ -0,0 +1,65 @@
+/**
+ * Which local files process_document may read (sourceType 'file' / 'pdf_file').
+ *
+ * A prompt-injected page can talk an agent into reading any path it names, so
+ * a local read is held to two rules, both checked on the path's realpath (a
+ * symlink is judged by where it points):
+ *  - the extension must be one of the document types the tool reads
+ *    (pdf, docx, txt, md, html/htm);
+ *  - no path segment may start with "." (~/.ssh, ~/.aws, ~/.config,
+ *    ~/.crawlforge, ...).
+ *
+ * CRAWLFORGE_ALLOWED_DIRS (a path.delimiter-separated list of directories)
+ * narrows reads to those directories instead: when set, a path is allowed only
+ * if its realpath sits inside one of them, and inside them the dot-segment rule
+ * does not apply. The extension rule always applies.
+ */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+export const ALLOWED_EXTENSIONS = ['.pdf', '.docx', '.txt', '.md', '.html', '.htm'];
+
+async function realpathOr(p) {
+  try {
+    return await fs.realpath(p);
+  } catch {
+    return p; // a missing path is judged as given; the read itself reports it
+  }
+}
+
+/**
+ * Resolves `source` and refuses it when the policy above does not allow it.
+ * @param {string} source the path the caller passed
+ * @param {NodeJS.ProcessEnv} [env]
+ * @returns {Promise<string>} the realpath to read
+ */
+export async function assertLocalFileAllowed(source, env = process.env) {
+  const real = await realpathOr(path.resolve(source));
+
+  const dirs = (env.CRAWLFORGE_ALLOWED_DIRS || '').split(path.delimiter).filter((d) => d.trim());
+  if (dirs.length > 0) {
+    const roots = await Promise.all(dirs.map((dir) => realpathOr(path.resolve(dir.trim()))));
+    if (!roots.some((root) => real.startsWith(root.endsWith(path.sep) ? root : root + path.sep))) {
+      throw new Error(
+        `Local file refused: '${source}' is outside CRAWLFORGE_ALLOWED_DIRS (${dirs.join(path.delimiter)}). ` +
+          'Add its directory to CRAWLFORGE_ALLOWED_DIRS to allow it.'
+      );
+    }
+  } else if (real.split(path.sep).some((segment) => segment.startsWith('.'))) {
+    throw new Error(
+      `Local file refused: '${source}' resolves to a path inside a dot-directory (${real}), ` +
+        'which process_document does not read (they hold credentials and config such as ~/.ssh and ~/.crawlforge). ' +
+        'To allow it, list its directory in CRAWLFORGE_ALLOWED_DIRS.'
+    );
+  }
+
+  const ext = path.extname(real).toLowerCase();
+  if (!ALLOWED_EXTENSIONS.includes(ext)) {
+    throw new Error(
+      `Local file refused: '${source}' resolves to a ${ext ? `'${ext}'` : 'extensionless'} file, and ` +
+        `process_document only reads ${ALLOWED_EXTENSIONS.join(', ')} files from disk.`
+    );
+  }
+
+  return real;
+}

package/src/utils/provenance.js

@@ -91,6 +91,20 @@ const VERSION_PREFIX = /^v(?=\d)/i;
 const MARKUP_BETWEEN_DIGITS = /(\d)(?:(?:\s*<[^>]{0,120}>)+\s*)([\d.,])/g;
 const MAX_WELD_PASSES = 3;
 
+/**
+ * Whether `sep` can be a digit-grouping separator in `t`: Western 3-digit
+ * groups ("1,234,567") or Indian lakh grouping ("12,34,567").
+ *
+ * @param {string} t
+ * @param {string} sep - "." or ","
+ * @returns {boolean}
+ */
+function isGrouped(t, sep) {
+  const s = sep === '.' ? '\\.' : sep;
+  return new RegExp(`^-?\\d{1,3}(?:${s}\\d{3})+$`).test(t)
+    || new RegExp(`^-?\\d{1,2}(?:${s}\\d{2})*${s}\\d{3}$`).test(t);
+}
+
 /**
  * Every numeric reading of one token, canonicalised.
  *
@@ -114,7 +128,9 @@ function readings(token) {
     raw.push(t.split(grouping).join('').replace(decimal, '.'));
   } else if (hasDot || hasComma) {
     const sep = hasDot ? '.' : ',';
-    raw.push(t.split(sep).join(''));                                  // grouping reading
+    // A grouping reading only when the groups are real thousands groups, so
+    // "12.34" is never read as 1234 (nor 1234 found in a page's "12.34").
+    if (isGrouped(t, sep)) raw.push(t.split(sep).join(''));           // grouping reading
     if (t.split(sep).length === 2) raw.push(t.replace(sep, '.'));     // decimal reading
   } else {
     raw.push(t);
@@ -250,6 +266,79 @@ function embeddedVersionReadings(value) {
   });
 }
 
+/** An ISO calendar date, the form a model is asked to convert dates into. */
+const ISO_DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
+
+/** Lowercase, strip accents and fold NBSPs, so "Mär" and "mar" compare equal. */
+const foldText = (s) => s.normalize('NFD').replace(/\p{M}/gu, '').toLowerCase();
+
+/**
+ * Month name -> month number (1-12), long and short forms, from Intl for the
+ * languages a page is most often written in. A name two languages assign to
+ * different months would be ambiguous and is dropped.
+ */
+const MONTHS = (() => {
+  const map = new Map();
+  const clash = new Set();
+  for (const locale of ['en', 'fr', 'de', 'es', 'it', 'pt', 'nl']) {
+    for (const month of ['long', 'short']) {
+      const fmt = new Intl.DateTimeFormat(locale, { month, timeZone: 'UTC' });
+      for (let m = 1; m <= 12; m++) {
+        const name = foldText(fmt.format(new Date(Date.UTC(2026, m - 1, 15)))).replace(/\.$/, '');
+        if (map.has(name) && map.get(name) !== m) clash.add(name);
+        map.set(name, m);
+      }
+    }
+  }
+  map.set('sept', 9); // "Sept 14" in English
+  for (const name of clash) map.delete(name);
+  return map;
+})();
+
+/** "2026-10-14" when y/m/d is a real calendar date, else null. */
+function isoDate(y, m, d) {
+  const date = new Date(Date.UTC(y, m - 1, d));
+  if (date.getUTCFullYear() !== y || date.getUTCMonth() !== m - 1 || date.getUTCDate() !== d) return null;
+  return `${y}-${String(m).padStart(2, '0')}-${String(d).padStart(2, '0')}`;
+}
+
+/** "14 octobre 2026", "14. März 2026", "1er octobre 2026", "14 de octubre de 2026". */
+const DAY_MONTH_YEAR = /(?<![\p{L}\d])(\d{1,2})(?:er|st|nd|rd|th|\.)?\s*(?:de\s+)?(\p{L}{3,})\.?,?\s+(?:de\s+)?(\d{4})(?!\d)/gu;
+/** "October 14, 2026", "Oct 14th 2026". */
+const MONTH_DAY_YEAR = /(?<![\p{L}\d])(\p{L}{3,})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})(?!\d)/gu;
+/** "14.10.2026", "14/10/2026", "10/14/2026" — day/month order read both ways. */
+const NUMERIC_DATE = /(?<!\d)(\d{1,2})[./-](\d{1,2})[./-](\d{4})(?!\d)/g;
+
+/**
+ * Every calendar date the source writes in words or as d/m/y digits, as ISO
+ * strings. Only real dates are admitted, and day, month and year must come
+ * from one expression, so a date is never assembled from scattered numbers.
+ *
+ * @param {string} source
+ * @returns {Set<string>}
+ */
+function datesInSource(source) {
+  const found = new Set();
+  const add = (y, m, d) => {
+    const iso = isoDate(Number(y), Number(m), Number(d));
+    if (iso) found.add(iso);
+  };
+  for (const variant of sourceVariants(source)) {
+    const text = foldText(variant);
+    for (const [, d, name, y] of text.matchAll(DAY_MONTH_YEAR)) {
+      if (MONTHS.has(name)) add(y, MONTHS.get(name), d);
+    }
+    for (const [, name, d, y] of text.matchAll(MONTH_DAY_YEAR)) {
+      if (MONTHS.has(name)) add(y, MONTHS.get(name), d);
+    }
+    for (const [, a, b, y] of text.matchAll(NUMERIC_DATE)) {
+      add(y, b, a);
+      add(y, a, b);
+    }
+  }
+  return found;
+}
+
 /**
  * Replace every numeric value that is not present in the source with null.
  *
@@ -278,6 +367,7 @@ export function verifyNumericProvenance(data, source) {
   // Built lazily: most extractions are all numbers and never need it, and
   // lowercasing a multi-megabyte page is not free.
   let haystacks = null;
+  let dates = null;
   const inSource = (spellings) => {
     if (haystacks === null) haystacks = literalHaystacks(source);
     return spellings.some((s) => haystacks.some((h) => h.includes(s)));
@@ -313,6 +403,17 @@ export function verifyNumericProvenance(data, source) {
         verified++;
         return node;
       }
+      // An ISO date the model converted from "14 octobre 2026" is not on the
+      // page literally; it is when the page's own dates are read. A date that
+      // cannot exist ("2027-03-00") never matches.
+      const iso = typeof node === 'string' && node.trim().match(ISO_DATE);
+      if (iso) {
+        if (dates === null) dates = datesInSource(source);
+        if (dates.has(isoDate(Number(iso[1]), Number(iso[2]), Number(iso[3])))) {
+          verified++;
+          return node;
+        }
+      }
       unverified.push({ path: path || '(root)', value: node, reason: 'not_found_in_source' });
       return null;
     }

package/src/utils/ssrfGuard.js

@@ -10,7 +10,7 @@
  *
  * Two levels:
  *  - Stage 1 (default): blocks connections to loopback, link-local /
- *    cloud-metadata (169.254.0.0/16, incl. 169.254.169.254), and 0.0.0.0.
+ *    cloud-metadata (169.254.0.0/16, incl. 169.254.169.254), 0.0.0.0 and `::`.
  *    These are never legitimate public-scrape targets, so impact is ~zero.
  *  - Stage 2 (SSRF_STRICT=true): full private-range enforcement (RFC1918, ULA,
  *    multicast, CGNAT, etc.) via the existing SSRFProtection range logic.
@@ -42,7 +42,8 @@ import { withFetchCause } from './fetchError.js';
 const _ssrf = new SSRFProtection();
 
 // Narrow Stage-1 ranges: things no legitimate public scrape ever targets.
-const STAGE1_RANGES = ['127.0.0.0/8', '169.254.0.0/16', '0.0.0.0/8', '::1/128', 'fe80::/10'];
+// `::` (the unspecified address) connects to the local host, like 0.0.0.0.
+const STAGE1_RANGES = ['127.0.0.0/8', '169.254.0.0/16', '0.0.0.0/8', '::/128', '::1/128', 'fe80::/10'];
 
 // Literal cloud-metadata / service-discovery hostnames blocked even before DNS.
 const METADATA_HOSTS = new Set(['metadata.google.internal', 'metadata.azure.com', 'metadata']);
@@ -179,7 +180,7 @@ function preflightHostCheck(u, sec) {
  * connects to exactly what we checked (rebinding-safe). Allowlisted hostnames
  * are resolved normally, with no address filtering.
  */
-function ssrfLookup(hostname, opts, callback) {
+export function ssrfLookup(hostname, opts, callback) {
   const sec = config.security?.ssrfProtection;
   if (isAllowlisted(hostname.toLowerCase(), sec?.allowedDomains)) {
     return dns.lookup(hostname, opts, callback);

package/src/utils/ssrfProtection.js

@@ -29,7 +29,10 @@ const SSRF_CONFIG = {
     '0.0.0.0/8',       // This network
     '100.64.0.0/10',   // Carrier-grade NAT
     '198.18.0.0/15',   // Benchmark testing
+    '::/128',          // IPv6 unspecified (connects to the local host)
     '::1/128',         // IPv6 localhost
+    '::/96',           // IPv4-compatible IPv6 (deprecated)
+    '64:ff9b::/96',    // NAT64 (embeds an IPv4 address)
     'fc00::/7',        // IPv6 private network
     'fe80::/10',       // IPv6 link-local
     'ff00::/8',        // IPv6 multicast