crawlforge-mcp-server 5.2.9 → 5.3.1

package/src/utils/complianceAudit.js

@@ -0,0 +1,72 @@
+/**
+ * complianceAudit — durable record of the compliance decisions a customer made.
+ *
+ * Ground rule G5 allows `respect_robots: false`, but only as *the customer's
+ * documented decision*. That means the override has to leave a trace: which key
+ * asked, for which URL, from which tool, when. Without the row the override is
+ * a silent product default again, which is the thing G5 exists to prevent.
+ *
+ * Rows go to `logs/compliance-audit.log` as JSONL (one row per line, appended)
+ * and to a small in-memory ring the tools and tests can read back. Writing is
+ * best-effort: an audit sink that throws must never fail a customer's fetch.
+ *
+ * The API key is never stored. `apiKeyId` is a truncated SHA-256 of it — stable
+ * enough to group a key's overrides, useless as a credential.
+ */
+
+import { createHash } from 'crypto';
+import { appendFile, mkdir } from 'fs/promises';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_LOG_PATH = join(__dirname, '../../logs/compliance-audit.log');
+
+const RING_SIZE = 200;
+const ring = [];
+
+let sink = null; // null → the default JSONL file sink
+
+/** Truncated, non-reversible identifier for an API key. */
+export function apiKeyId(apiKey) {
+  if (typeof apiKey !== 'string' || !apiKey) return 'anonymous';
+  return createHash('sha256').update(apiKey).digest('hex').slice(0, 16);
+}
+
+async function defaultSink(row) {
+  await mkdir(dirname(DEFAULT_LOG_PATH), { recursive: true });
+  await appendFile(DEFAULT_LOG_PATH, `${JSON.stringify(row)}\n`, 'utf8');
+}
+
+/**
+ * Record a compliance event. Never throws and never rejects.
+ * @param {{ event: string, url?: string, tool?: string, apiKeyId?: string, [k: string]: unknown }} event
+ * @returns {{ event: string, timestamp: string }} the row as written
+ */
+export function recordComplianceEvent(event = {}) {
+  const row = { timestamp: new Date().toISOString(), ...event };
+  ring.push(row);
+  if (ring.length > RING_SIZE) ring.shift();
+
+  Promise.resolve()
+    .then(() => (sink || defaultSink)(row))
+    .catch(() => { /* an audit sink must never break a fetch */ });
+
+  return row;
+}
+
+/** Most recent audit rows, newest last. Test/diagnostic hook. */
+export function getComplianceAuditRows() {
+  return ring.slice();
+}
+
+/** Replace the persistence sink (tests, or a hosted deployment). */
+export function setComplianceAuditSink(fn) {
+  sink = typeof fn === 'function' ? fn : null;
+}
+
+/** Test hook. */
+export function _resetComplianceAudit() {
+  ring.length = 0;
+  sink = null;
+}

package/src/utils/contentUtils.js

@@ -312,7 +312,18 @@ export class ContentQualityAssessor {
   }
 
   /**
-   * Calculate simple readability metrics
+   * Calculate Flesch Reading-Ease metrics.
+   *
+   * This is the single Flesch implementation behind process_document and
+   * extract_content: both the `readabilityScore` field and
+   * `qualityAssessment.metrics.readability` are derived from here, so the two
+   * can never report different scores for the same text.
+   *
+   * The score is deliberately NOT clamped to [0, 100] — Flesch is unbounded
+   * (very simple text exceeds 100, dense text goes negative), and
+   * assessContentQuality() treats `score > 100` as a quality signal, which a
+   * clamp would make unreachable.
+   *
    * @param {string} text - Text to analyze
    * @returns {Object} - Readability metrics
    */

package/src/utils/domainFilter.js

@@ -209,24 +209,36 @@ export class DomainFilter {
   /**
    * Check if URL is allowed based on all filtering rules
    * @param {string} url - URL to check
+   * @param {Object} [options] - Evaluation options
+   * @param {boolean} [options.isSeed=false] - Treat url as a crawl's start URL, which is
+   *   exempt from the include-pattern gate (the caller asked for it explicitly). Blacklist,
+   *   exclude patterns and whitelist checks are unchanged.
    * @returns {Object} Decision object with allowed status and metadata
    */
-  isAllowed(url) {
+  isAllowed(url, options = {}) {
+    const { isSeed = false } = options;
     try {
       const normalizedUrl = normalizeUrl(url);
-      
+      // Patterns are tested against the raw URL as well, so a pattern written with a
+      // trailing slash ('/docs/') still matches a URL normalizeUrl has stripped it from.
+      // The two forms can therefore reach different decisions and must not share a cache
+      // entry; seed checks skip the cache entirely since they use a different gate.
+      const cacheKey = normalizedUrl === url ? normalizedUrl : `${normalizedUrl}|${url}`;
+
       // Check cache first
-      if (this.cache.has(normalizedUrl)) {
+      if (!isSeed && this.cache.has(cacheKey)) {
         this.cacheHits++;
-        return this.cache.get(normalizedUrl);
+        return this.cache.get(cacheKey);
       }
 
       this.cacheMisses++;
-      const decision = this.evaluateUrl(normalizedUrl);
-      
+      const decision = this.evaluateUrl(normalizedUrl, url, isSeed);
+
       // Cache the decision
-      this.addToCache(normalizedUrl, decision);
-      
+      if (!isSeed) {
+        this.addToCache(cacheKey, decision);
+      }
+
       return decision;
     } catch (error) {
       return {
@@ -241,9 +253,11 @@ export class DomainFilter {
   /**
    * Internal URL evaluation logic
    * @param {string} url - Normalized URL to evaluate
+   * @param {string} [rawUrl=url] - URL as the caller supplied it, before normalization
+   * @param {boolean} [isSeed=false] - Skip the include-pattern gate for a crawl's start URL
    * @returns {Object} Decision object
    */
-  evaluateUrl(url) {
+  evaluateUrl(url, rawUrl = url, isSeed = false) {
     const urlObj = new URL(url);
     const domain = urlObj.hostname;
     const path = urlObj.pathname;
@@ -255,7 +269,7 @@ export class DomainFilter {
     }
 
     // 2. Check exclude patterns
-    const excludePatternResult = this.checkExcludePatterns(url);
+    const excludePatternResult = this.checkExcludePatterns(url, rawUrl);
     if (!excludePatternResult.allowed) {
       return excludePatternResult;
     }
@@ -267,13 +281,16 @@ export class DomainFilter {
     }
 
     // 4. Check include patterns
-    const includePatternResult = this.checkIncludePatterns(url);
+    const includePatternResult = this.checkIncludePatterns(url, rawUrl);
     if (includePatternResult.allowed) {
       return includePatternResult;
     }
 
-    // 5. Default behavior - if no whitelist exists, allow; if whitelist exists, deny
-    const hasWhitelist = this.whitelist.size > 0 || this.patterns.include.length > 0;
+    // 5. Default behavior - if no whitelist exists, allow; if whitelist exists, deny.
+    // A seed URL is not subject to the include-pattern gate: include patterns scope where a
+    // crawl may go, not whether the URL the caller named may be fetched at all.
+    const hasWhitelist = this.whitelist.size > 0 ||
+      (!isSeed && this.patterns.include.length > 0);
     
     return {
       allowed: !hasWhitelist,
@@ -366,12 +383,13 @@ export class DomainFilter {
 
   /**
    * Check exclude patterns
-   * @param {string} url - URL to check
+   * @param {string} url - Normalized URL to check
+   * @param {string} [rawUrl=url] - URL before normalization; tested as well
    * @returns {Object} Decision object
    */
-  checkExcludePatterns(url) {
+  checkExcludePatterns(url, rawUrl = url) {
     for (const patternConfig of this.patterns.exclude) {
-      if (patternConfig.pattern.test(url)) {
+      if (patternConfig.pattern.test(url) || patternConfig.pattern.test(rawUrl)) {
         return {
           allowed: false,
           reason: `Matches exclude pattern: ${patternConfig.rawPattern}`,
@@ -390,12 +408,13 @@ export class DomainFilter {
 
   /**
    * Check include patterns
-   * @param {string} url - URL to check
+   * @param {string} url - Normalized URL to check
+   * @param {string} [rawUrl=url] - URL before normalization; tested as well
    * @returns {Object} Decision object
    */
-  checkIncludePatterns(url) {
+  checkIncludePatterns(url, rawUrl = url) {
     for (const patternConfig of this.patterns.include) {
-      if (patternConfig.pattern.test(url)) {
+      if (patternConfig.pattern.test(url) || patternConfig.pattern.test(rawUrl)) {
         return {
           allowed: true,
           reason: `Matches include pattern: ${patternConfig.rawPattern}`,

package/src/utils/fetchIdentity.js

@@ -0,0 +1,62 @@
+/**
+ * fetchIdentity — the single outbound identity for every CrawlForge fetch.
+ *
+ * Nine tools used to hardcode nine different User-Agents, so nine tools saw nine
+ * different versions of the same page (a Zillow listing served 41 `address`
+ * elements to one tool and 9 to another). That is a correctness bug, not a
+ * disguise problem, and the fix is one honest identity everywhere:
+ *
+ *     CrawlForge/<version> (+https://crawlforge.dev)
+ *
+ * Ground rule G4: identify honestly by default — real product name, real
+ * contact URL — so a site that wants to block us can. Callers that have their
+ * own agreement with a target can pass a per-request `userAgent` override
+ * (G4's escape hatch); the override wins, the canonical UA applies otherwise.
+ *
+ * The literal 'User-Agent' header name lives here and nowhere else outside the
+ * browser paths, so `identityHeaders()` is the only way to spell it.
+ */
+
+import { createRequire } from 'module';
+
+const _require = createRequire(import.meta.url);
+const _pkg = _require('../../package.json');
+
+/** The canonical, honest identity every page fetch sends. */
+export const CRAWLFORGE_USER_AGENT = `CrawlForge/${_pkg.version} (+https://crawlforge.dev)`;
+
+/**
+ * Identity for non-page traffic (webhook delivery, health checks, alerting).
+ * Same product name and contact URL, with the role appended so a receiving
+ * server can tell a webhook POST from a crawl. Falls back to the canonical UA.
+ * @param {string} [role]
+ * @returns {string}
+ */
+export function serviceUserAgent(role) {
+  const trimmed = typeof role === 'string' ? role.trim() : '';
+  return trimmed
+    ? `CrawlForge/${_pkg.version} (+https://crawlforge.dev; ${trimmed})`
+    : CRAWLFORGE_USER_AGENT;
+}
+
+/**
+ * Resolve the User-Agent for a request: a non-empty override wins, otherwise
+ * the canonical identity (optionally role-suffixed).
+ * @param {string} [override]
+ * @param {string} [role]
+ * @returns {string}
+ */
+export function resolveUserAgent(override, role) {
+  const trimmed = typeof override === 'string' ? override.trim() : '';
+  return trimmed || serviceUserAgent(role);
+}
+
+/**
+ * The outbound identity headers for a fetch. Spread this into a headers object
+ * rather than writing the header name at the call site.
+ * @param {{ userAgent?: string, role?: string }} [options]
+ * @returns {{ 'User-Agent': string }}
+ */
+export function identityHeaders(options = {}) {
+  return { 'User-Agent': resolveUserAgent(options.userAgent, options.role) };
+}

package/src/utils/hostBlocklist.js

@@ -0,0 +1,81 @@
+/**
+ * hostBlocklist — the platform's permanent refusals (ground rule G7).
+ *
+ * When a site owner opts out or sends a takedown, the block has to hold at the
+ * platform layer. A blocklist any customer can switch off is not a blocklist,
+ * so nothing here reads a per-request flag: `respect_robots: false` does not
+ * reach it, and neither does a `userAgent` override.
+ *
+ * To add an entry, append to BLOCKED_HOSTS with a dated one-line reason. Blocks
+ * cover the host and all of its subdomains. `CRAWLFORGE_BLOCKED_HOSTS` (comma
+ * separated) adds more at runtime; it can only ever extend the list.
+ */
+
+/** @type {string[]} host → blocked, with the date and reason it was added. */
+const BLOCKED_HOSTS = [
+  // e.g. 'example.com', // 2026-08-27 owner opt-out, ref #123
+];
+
+export class BlockedHostError extends Error {
+  constructor(host) {
+    super(
+      `${host} is on CrawlForge's permanent blocklist (site-owner opt-out or takedown) ` +
+      `and cannot be fetched. This block is not overridable. ` +
+      `If you believe it is in error, contact support@crawlforge.dev.`
+    );
+    this.name = 'BlockedHostError';
+    this.code = 'HOST_BLOCKED';
+    this.host = host;
+  }
+}
+
+let overrideList = null; // tests only
+
+function blockedSet() {
+  if (overrideList) return overrideList;
+  const fromEnv = (process.env.CRAWLFORGE_BLOCKED_HOSTS || '')
+    .split(',')
+    .map((h) => h.trim().toLowerCase())
+    .filter(Boolean);
+  return new Set([...BLOCKED_HOSTS.map((h) => h.toLowerCase()), ...fromEnv]);
+}
+
+/** The hostname of a URL, lowercased, or null if it will not parse. */
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase().replace(/\.$/, '');
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * True when the URL's host, or any parent domain of it, is blocked.
+ * @param {string} url
+ */
+export function isBlockedHost(url) {
+  const host = hostOf(url);
+  if (!host) return false;
+  const blocked = blockedSet();
+  if (blocked.size === 0) return false;
+
+  const labels = host.split('.');
+  for (let i = 0; i < labels.length - 1; i++) {
+    if (blocked.has(labels.slice(i).join('.'))) return true;
+  }
+  return blocked.has(host);
+}
+
+/**
+ * Throw BlockedHostError if the URL's host is blocked. Call before any network
+ * work — the point is that a blocked host never gets a request.
+ * @param {string} url
+ */
+export function assertHostAllowed(url) {
+  if (isBlockedHost(url)) throw new BlockedHostError(hostOf(url));
+}
+
+/** Test hook: replace the effective list. Pass null to restore. */
+export function _setBlockedHostsForTests(hosts) {
+  overrideList = hosts ? new Set(hosts.map((h) => h.toLowerCase())) : null;
+}

package/src/utils/hostRateLimiter.js

@@ -10,6 +10,14 @@
  * effective behaviour), enabled by RATE_LIMIT_PER_DOMAIN (default true). Setting
  * RATE_LIMIT_PER_DOMAIN=false disables the throttle entirely — there is no global
  * cross-host cap, so broad multi-host crawls are never slowed by this.
+ *
+ * Two politeness signals the host itself sends are honoured on top of that
+ * (ground rule G6 — load we impose is load someone pays for):
+ *   - robots.txt `Crawl-delay`, passed in per request by the robots gate;
+ *   - `Retry-After` on a 429/503, recorded by the fetch helpers so the *next*
+ *     request to that host waits instead of retrying straight into the wall.
+ * Both are host-scoped and survive RATE_LIMIT_PER_DOMAIN=false: an operator
+ * turning off our own throttle is not a licence to ignore the site's.
  */
 import { RateLimiter } from './rateLimiter.js';
 import { config } from '../constants/config.js';
@@ -26,13 +34,103 @@ function limiter() {
   return _limiter;
 }
 
+/** host → { lastRequestAt, notBefore } */
+const hostState = new Map();
+
+function hostOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+}
+
+function stateFor(host) {
+  let state = hostState.get(host);
+  if (!state) {
+    state = { lastRequestAt: 0, notBefore: 0 };
+    hostState.set(host, state);
+  }
+  return state;
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+/**
+ * Parse a `Retry-After` header into milliseconds.
+ * Accepts delta-seconds ("2") and an HTTP-date ("Wed, 21 Oct 2026 07:28:00 GMT").
+ * Returns 0 for anything unparseable, negative, or absent.
+ * @param {string|null|undefined} value
+ * @param {number} [now] epoch ms, for deterministic tests
+ * @returns {number}
+ */
+export function parseRetryAfter(value, now = Date.now()) {
+  if (value === null || value === undefined) return 0;
+  const raw = String(value).trim();
+  if (!raw) return 0;
+
+  if (/^\d+$/.test(raw)) return parseInt(raw, 10) * 1000;
+
+  const asDate = Date.parse(raw);
+  if (!Number.isNaN(asDate)) return Math.max(0, asDate - now);
+
+  return 0;
+}
+
+/**
+ * Record a `Retry-After` the host asked for. Subsequent requests to that host
+ * wait it out. Capped at 5 minutes so a hostile or mistaken header cannot pin a
+ * worker indefinitely.
+ * @param {string} url
+ * @param {string|null|undefined} retryAfterHeader
+ * @returns {number} the backoff applied, in ms (0 if none)
+ */
+export function noteRetryAfter(url, retryAfterHeader) {
+  const host = hostOf(url);
+  if (!host) return 0;
+
+  const delayMs = Math.min(parseRetryAfter(retryAfterHeader), 5 * 60 * 1000);
+  if (delayMs <= 0) return 0;
+
+  const state = stateFor(host);
+  state.notBefore = Math.max(state.notBefore, Date.now() + delayMs);
+  return delayMs;
+}
+
+/** Remaining backoff for a host in ms (0 if none). Diagnostic/test hook. */
+export function getHostBackoffMs(url) {
+  const host = hostOf(url);
+  if (!host) return 0;
+  const state = hostState.get(host);
+  return state ? Math.max(0, state.notBefore - Date.now()) : 0;
+}
+
 /**
  * Wait (if necessary) until another request to this URL's host is allowed.
  * Never throws — a limiter failure must not block a legitimate fetch.
  * @param {string} url
+ * @param {{ crawlDelayMs?: number }} [options] `crawlDelayMs` from robots.txt
  */
-export async function throttleHost(url) {
-  if (config.rateLimit.perDomain === false) return; // feature disabled
+export async function throttleHost(url, options = {}) {
+  const host = hostOf(url);
+  const crawlDelayMs = Number(options.crawlDelayMs) > 0 ? Number(options.crawlDelayMs) : 0;
+
+  if (host) {
+    const state = stateFor(host);
+
+    // The host's own signals, honoured whether or not our throttle is enabled.
+    const waits = [];
+    if (state.notBefore > Date.now()) waits.push(state.notBefore - Date.now());
+    if (crawlDelayMs > 0 && state.lastRequestAt > 0) {
+      waits.push(state.lastRequestAt + crawlDelayMs - Date.now());
+    }
+    const wait = Math.max(0, ...waits);
+    if (wait > 0) await sleep(wait);
+
+    state.lastRequestAt = Date.now();
+  }
+
+  if (config.rateLimit.perDomain === false) return; // our own throttle disabled
   try {
     await limiter().checkLimit(url);
   } catch {
@@ -43,4 +141,5 @@ export async function throttleHost(url) {
 /** Test/diagnostic hook. */
 export function _resetHostRateLimiter() {
   _limiter = null;
+  hostState.clear();
 }

package/src/utils/ollamaConfig.js

@@ -55,6 +55,32 @@ const PREFERRED_MODELS = [
   'qwen2.5:3b'
 ];
 
+/**
+ * Models measured fit to JUDGE claims — relevance to a topic, same-meaning
+ * grouping, and contradiction — as opposed to extracting fields. Measured
+ * 2026-08-28 by replaying a live deep_research run's own 136 claims through
+ * each installed model, three runs each:
+ *
+ *   gemma3:12b   0 false contradictions on 27 real pairs, 3/3 planted caught,
+ *                7-9 cross-source groups (the 4B model: 1-2 false, 0-1/3
+ *                caught, 1 group)
+ *   gemma3:4b    the extraction winner, but it scored "Playwright vs Selenium"
+ *                marketing 0.9 relevant to an anti-bot topic and put it in the
+ *                research summary
+ *   gemma4:31b   judged as cleanly as gemma3:12b but only with thinking turned
+ *                off — under the default it spends the whole token budget on
+ *                hidden reasoning and returns empty content — and it grouped so
+ *                strictly that consensus vanished. Not ranked.
+ *   gpt-oss:20b  empty content at these token budgets for the same reason,
+ *                and `think: false` makes it emit nothing at all. Not ranked.
+ *
+ * Membership here is what turns conflict detection on: a model that invents
+ * disagreement between sources that agree is worse than one that reports none,
+ * so a model absent from this list is never asked. When none is installed the
+ * judgement role falls through to the extraction ranking above.
+ */
+export const JUDGEMENT_MODELS = ['gemma3:12b'];
+
 /** Used only when Ollama cannot be reached, so the error names a real model. */
 export const FALLBACK_OLLAMA_MODEL = 'llama3.2';
 
@@ -102,9 +128,11 @@ export async function installedOllamaModels() {
  * instead would break anyone who has not pulled it, so the best *installed*
  * model is chosen, and an explicit OLLAMA_DEFAULT_MODEL always wins.
  *
+ * @param {'default'|'judgement'} [role] 'judgement' tries JUDGEMENT_MODELS
+ *   first and falls through to the extraction ranking when none is installed.
  * @returns {Promise<string>}
  */
-export async function selectOllamaModel() {
+export async function selectOllamaModel(role = 'default') {
   const explicit = process.env.OLLAMA_DEFAULT_MODEL;
   if (explicit) return explicit;
 
@@ -112,10 +140,16 @@ export async function selectOllamaModel() {
   if (installed.length === 0) return FALLBACK_OLLAMA_MODEL;
 
   const byBase = new Map(installed.map((name) => [baseName(name), name]));
-  for (const preferred of PREFERRED_MODELS) {
+  const ranking = role === 'judgement' ? [...JUDGEMENT_MODELS, ...PREFERRED_MODELS] : PREFERRED_MODELS;
+  for (const preferred of ranking) {
     const match = byBase.get(baseName(preferred));
     if (match) return match;
   }
   // Nothing recognised — use whatever is there rather than failing.
   return installed[0];
 }
+
+/** Whether a model name is one measured fit to judge contradictions. */
+export function isJudgementModel(name) {
+  return typeof name === 'string' && JUDGEMENT_MODELS.some((m) => baseName(m) === baseName(name));
+}