crawlforge-mcp-server 6.21.1 → 6.22.1

package/src/core/AuthManager.js

@@ -758,7 +758,10 @@ class AuthManager {
         break;
       }
       case 'extract_with_llm':
-        note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
+        // Only the cloud providers bill you; the default runs on local Ollama.
+        if (params?.provider === 'openai' || params?.provider === 'anthropic') {
+          note = 'External LLM API call billed by your LLM provider, separate from the credit cost.';
+        }
         break;
       case 'stealth_mode':
         note = projected === 1

package/src/core/LLMsTxtAnalyzer.js

@@ -236,7 +236,9 @@ export class LLMsTxtAnalyzer {
         const batch = commonPaths.slice(i, i + PROBE_CONCURRENCY);
         const results = await Promise.allSettled(
           batch.map(async (path) => {
-            const apiUrl = `${baseUrl}${path}`;
+            // Resolved, not concatenated: the URL arrives as typed, and
+            // "https://site.com/" + "/json" was listed as "https://site.com//json" (R25 1.8).
+            const apiUrl = new URL(path, baseUrl).toString();
             const response = await this.fetchWithTimeout(apiUrl, { timeout: 5000 });
             if (response.ok) {
               const contentType = response.headers.get('content-type') || '';
@@ -367,7 +369,7 @@ export class LLMsTxtAnalyzer {
         const batch = sensitivePaths.slice(i, i + PROBE_CONCURRENCY);
         const results = await Promise.allSettled(
           batch.map(async (path) => {
-            const testUrl = `${baseUrl}${path}`;
+            const testUrl = new URL(path, baseUrl).toString();
             const response = await this.fetchWithTimeout(testUrl, { timeout: 3000 });
             if (response.status === 200 || response.status === 302 || response.status === 401) {
               return {
@@ -571,7 +573,6 @@ export class LLMsTxtAnalyzer {
    */
   prioritizeUrls(urls, baseUrl, limit) {
     const sections = new Map();
-    let sawAny = false;
 
     for (const url of urls) {
       let segments;
@@ -580,7 +581,6 @@ export class LLMsTxtAnalyzer {
       } catch {
         continue;
       }
-      sawAny = true;
       // The root arrives as either "https://site.com" or "https://site.com/"
       // (normalizeUrl strips a trailing slash from every path but "/"), so it
       // is recognised by its path and re-added below in one canonical form.
@@ -590,10 +590,9 @@ export class LLMsTxtAnalyzer {
       sections.get(section).push({ url, depth: segments.length });
     }
 
-    if (!sawAny) return [];
-
     // The site root leads: it is always fetched, and a site guide that omits
-    // the homepage is a misleading one.
+    // the homepage is a misleading one. That holds with no other candidate
+    // too: a one-page site was analysed as 0 pages (R25 1.2).
     const ordered = [`${baseUrl}/`];
     const queues = [...sections.keys()].sort().map((name) =>
       sections.get(name).sort((a, b) => a.depth - b.depth || a.url.localeCompare(b.url)));

package/src/core/LocalizationManager.js

@@ -778,15 +778,40 @@ export class LocalizationManager extends EventEmitter {
   }
   
   getFirstDayOfWeek(countryCode) {
-    // 0 = Sunday, 1 = Monday
+    // 0 = Sunday, 1 = Monday. From CLDR 48 supplemental weekData "firstDay";
+    // covers every SUPPORTED_COUNTRIES entry. The fallback is CLDR's world
+    // default (Monday). `??`, not `||`: Sunday is 0 (R25).
     const firstDays = {
       'US': 0,
       'GB': 1,
       'DE': 1,
-      'FR': 1
+      'FR': 1,
+      'JP': 0,
+      'CN': 1,
+      'AU': 1,
+      'CA': 0,
+      'IT': 1,
+      'ES': 1,
+      'RU': 1,
+      'BR': 0,
+      'IN': 0,
+      'KR': 0,
+      'MX': 0,
+      'NL': 1,
+      'SE': 1,
+      'NO': 1,
+      'SA': 0,
+      'AE': 1,
+      'TR': 1,
+      'IL': 0,
+      'TH': 0,
+      'SG': 0,
+      'PL': 1,
+      'ZA': 0,
+      'CH': 1
     };
-    
-    return firstDays[countryCode] || 1;
+
+    return firstDays[countryCode] ?? 1;
   }
   
   generateScreenProperties(countryCode) {

package/src/core/ResearchOrchestrator.js

@@ -63,6 +63,63 @@ export function reportedFindings(keyFindings) {
   return material.length > 0 ? material : keyFindings;
 }
 
+// Citation markers left in extracted text: "(3)", "[12]", "[3, 4]", "[2-5]".
+// At most three digits, so a year such as "(2024)" stays.
+const CITATION_MARKER = /\s*(?:\(\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\)|\[\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\])/g;
+const LIST_MARKER = /^(?:[-*\u2022\u25AA\u2013]|\d{1,2}[.)])\s+/;
+// A lowercase word run straight into a capitalized one: "glanceWhat". A
+// camelCase name starts with a capital ("JavaScript", "DataDome") and never matches.
+const WELDED_WORDS = /\b[a-z]{2,}[A-Z][a-z]/g;
+
+/**
+ * A line that is a table row or headings welded together, not a statement.
+ * extract_content reads a table one row per line, cells joined by " | " (a
+ * two-cell row has just one), and crawlforge-extractors' flattenText joins
+ * inline elements with no separator, so a table of contents of `<a class="block">` links came back as "The six
+ * vendors at a glanceWhat the independent benchmarks showWhy your approach…"
+ * (R25 re-check, 2026-10-19). A sentence ends in punctuation; welded headings
+ * have none.
+ *
+ * @param {string} line
+ * @returns {boolean}
+ */
+function isPageFragment(line) {
+  if (/\s\|\s/.test(line) || (line.match(/\|/g) || []).length >= 2) return true;
+  return line.length <= 300 && !/[.!?](?:\s|$)/.test(line) &&
+    (line.match(WELDED_WORDS) || []).length >= 2;
+}
+
+/**
+ * The statements in one extracted key point. A key point is a sentence as the
+ * splitter found it, so a heading or list item above it arrives in the same
+ * string ("Built-in encryption\nQUIC was built with security in mind.").
+ * Each line becomes its own statement, except a line starting in lowercase,
+ * which continues the one before it (wrapped text). List and citation markers
+ * are removed. Table rows and welded headings are dropped; heading-only lines
+ * are left for isAdmissibleClaim to reject.
+ *
+ * @param {string} point
+ * @returns {string[]}
+ */
+export function claimStatements(point) {
+  const blocks = [];
+  let afterFragment = false;
+  for (const raw of String(point).split('\n')) {
+    const line = raw.trim();
+    if (!line) continue;
+    if (isPageFragment(line)) {
+      afterFragment = true;
+      continue;
+    }
+    if (blocks.length > 0 && !afterFragment && /^[a-z]/.test(line)) blocks[blocks.length - 1] += ` ${line}`;
+    else blocks.push(line);
+    afterFragment = false;
+  }
+  return blocks
+    .map(block => block.replace(LIST_MARKER, '').replace(CITATION_MARKER, '').replace(/\s+/g, ' ').trim())
+    .filter(Boolean);
+}
+
 /**
  * Page text for the plain-fetch fallback, one line per block element. The
  * regex tag-strip it replaces ran every block together on one line.
@@ -1384,7 +1441,8 @@ export class ResearchOrchestrator extends EventEmitter {
         const keyPoints = summary.keypoints || summary.keyPoints || [];
         if (keyPoints.length > 0) {
           const relevance = this.sourceRelevance(source);
-          keyPoints.forEach((point, index) => {
+          let claimIndex = 0;
+          keyPoints.forEach((keyPoint, index) => claimStatements(keyPoint).forEach(point => {
             const credibility = source.overallCredibility || 0.65;
             // A recommendation is not evidence. Two routes to the same flag: a
             // page promoting itself, and — whoever published it — a claim whose
@@ -1392,7 +1450,7 @@ export class ResearchOrchestrator extends EventEmitter {
             const promotional = isVendorSelfPromotion(point, source.link) ||
               isProductRecommendation(point);
             claims.push({
-              id: `${source.link}_claim_${index}`,
+              id: `${source.link}_claim_${claimIndex++}`,
               claim: point,
               source: source.link,
               sourceTitle: source.title,
@@ -1402,7 +1460,7 @@ export class ResearchOrchestrator extends EventEmitter {
               context: summary.supporting?.[index] || '',
               extractedAt: new Date().toISOString()
             });
-          });
+          }));
         }
       } catch (error) {
         this.logger.warn('Claim extraction failed', {
@@ -1464,31 +1522,33 @@ export class ResearchOrchestrator extends EventEmitter {
   /**
    * Keep only claims that can be research findings: prose about the topic, not
    * document front matter (author/affiliation blocks, DOI stubs, "Retrieved
-   * from" lines), not from a source the relevance analysis scored below
-   * MIN_CLAIM_RELEVANCE, and not a sentence the LLM scored as barely about the
-   * topic. Each gate falls back to the previous claim set rather than returning
-   * nothing — no claims means no findings at all.
+   * from" lines), and not a sentence the LLM scored as barely about the topic.
+   * Those two gates fall back to the previous claim set rather than returning
+   * nothing. A claim from an off-topic source — one the relevance analysis
+   * scored below MIN_CLAIM_RELEVANCE — is always dropped: when every source is
+   * off-topic, no findings is the honest result (R25 reported such claims as
+   * findings through the fallback). A source with no score is kept.
    */
   admitClaims(claims) {
     if (claims.length === 0) return claims;
 
-    const substantive = claims.filter(claim => isAdmissibleClaim(claim.claim));
-    const relevant = substantive.filter(
+    const relevant = claims.filter(
       claim => typeof claim.relevance !== 'number' || claim.relevance >= MIN_CLAIM_RELEVANCE
     );
-    const onTopic = relevant.filter(
+    const substantive = relevant.filter(claim => isAdmissibleClaim(claim.claim));
+    const onTopic = substantive.filter(
       claim => typeof claim.topicRelevance !== 'number' ||
         claim.topicRelevance >= MIN_CLAIM_TOPIC_RELEVANCE
     );
 
     const admitted = onTopic.length > 0
       ? onTopic
-      : (relevant.length > 0 ? relevant : (substantive.length > 0 ? substantive : claims));
+      : (substantive.length > 0 ? substantive : relevant);
 
     this.logger.debug('Claim admission', {
       candidates: claims.length,
-      substantive: substantive.length,
       relevant: relevant.length,
+      substantive: substantive.length,
       onTopic: onTopic.length,
       admitted: admitted.length
     });
@@ -2007,8 +2067,10 @@ export class ResearchOrchestrator extends EventEmitter {
           findings.push({
             finding: claim.claim,
             supportingClaims: group.claims.length,
+            supportingSources: group.sourceCount,
             credibility: group.avgCredibility,
-            sources: group.claims.map(c => c.source),
+            // One entry per source: a group holds several claims from one page.
+            sources: [...new Set(group.claims.map(c => c.source))],
             ...(claim.promotional ? { promotional: true } : {}),
             ...(typeof claim.topicRelevance === 'number'
               ? { topicRelevance: claim.topicRelevance }
@@ -2032,6 +2094,7 @@ export class ResearchOrchestrator extends EventEmitter {
         title: source.title,
         url: source.link,
         credibility: source.overallCredibility,
+        relevance: this.sourceRelevance(source),
         evidence: source.extractedContent?.substring(0, 300) + '...'
       }))
       .slice(0, 15);

package/src/core/SamplingClient.js

@@ -15,13 +15,24 @@
 
 import { ollamaBaseUrl as OLLAMA_BASE_URL, ollamaHeaders, selectOllamaModel } from '../utils/ollamaConfig.js';
 
+/**
+ * A provider call's own 30 s bound, cut short by the caller's time budget
+ * when there is one.
+ * @param {AbortSignal|null} [budget]
+ * @returns {AbortSignal}
+ */
+function callSignal(budget) {
+  const own = AbortSignal.timeout(30_000);
+  return budget ? AbortSignal.any([own, budget]) : own;
+}
+
 /**
  * Attempt an Ollama completion.
  * @param {string} prompt
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryOllama(prompt, { model, maxTokens } = {}) {
+async function tryOllama(prompt, { model, maxTokens, budget } = {}) {
   const ollamaModel = model || await selectOllamaModel();
   const url = `${OLLAMA_BASE_URL()}/api/generate`;
   const res = await fetch(url, {
@@ -33,7 +44,7 @@ async function tryOllama(prompt, { model, maxTokens } = {}) {
       stream: false,
       ...(maxTokens ? { options: { num_predict: maxTokens } } : {}),
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`Ollama HTTP ${res.status}`);
   const data = await res.json();
@@ -47,7 +58,7 @@ async function tryOllama(prompt, { model, maxTokens } = {}) {
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryOpenAI(prompt, { model, maxTokens } = {}) {
+async function tryOpenAI(prompt, { model, maxTokens, budget } = {}) {
   const apiKey = process.env.OPENAI_API_KEY;
   if (!apiKey) throw new Error('OPENAI_API_KEY not set');
   const base = (process.env.OPENAI_BASE_URL || 'https://api.openai.com').replace(/\/$/, '');
@@ -62,7 +73,7 @@ async function tryOpenAI(prompt, { model, maxTokens } = {}) {
       messages: [{ role: 'user', content: prompt }],
       ...(maxTokens ? { max_tokens: maxTokens } : {}),
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`OpenAI HTTP ${res.status}`);
   const data = await res.json();
@@ -75,7 +86,7 @@ async function tryOpenAI(prompt, { model, maxTokens } = {}) {
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryAnthropic(prompt, { model, maxTokens } = {}) {
+async function tryAnthropic(prompt, { model, maxTokens, budget } = {}) {
   const apiKey = process.env.ANTHROPIC_API_KEY;
   if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set');
   const base = (process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com').replace(/\/$/, '');
@@ -91,7 +102,7 @@ async function tryAnthropic(prompt, { model, maxTokens } = {}) {
       max_tokens: maxTokens || 1024,
       messages: [{ role: 'user', content: prompt }],
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`Anthropic HTTP ${res.status}`);
   const data = await res.json();
@@ -116,12 +127,18 @@ export class SamplingClient {
    * @param {string} [options.model] - Override model name
    * @param {number} [options.maxTokens] - Max tokens for response
    * @param {string} [options.systemPrompt] - Optional system-level instruction
+   * @param {number} [options.timeoutMs] - Time budget for the whole chain. Each
+   *   provider call stops at its own 30 s bound or when the budget runs out,
+   *   whichever comes first, and the sampling fallback is raced against it.
+   *   An exhausted budget throws an error with code 'LLM_TIMEOUT'.
    * @returns {Promise<{ text: string, provider: string }>}
    */
   async complete(prompt, options = {}) {
     const fullPrompt = options.systemPrompt
       ? `${options.systemPrompt}\n\n${prompt}`
       : prompt;
+    const budget = options.timeoutMs != null ? AbortSignal.timeout(Math.max(0, options.timeoutMs)) : null;
+    options = { ...options, budget };
 
     // 1. Try Ollama (local, no API key)
     try {
@@ -153,11 +170,19 @@ export class SamplingClient {
     // 3. Try MCP sampling (client-side LLM)
     if (this._mcpServer?.server?.createMessage) {
       try {
-        const samplingResult = await this._mcpServer.server.createMessage({
+        const request = this._mcpServer.server.createMessage({
           messages: [{ role: 'user', content: { type: 'text', text: fullPrompt } }],
           maxTokens: options.maxTokens || 1024,
           includeContext: 'none',
-        });
+        }, budget ? { signal: budget } : undefined);
+        // Raced as well: a client that ignores the cancellation would
+        // otherwise hold the caller past its budget.
+        const samplingResult = await (budget
+          ? Promise.race([request, new Promise((_, reject) => {
+            if (budget.aborted) reject(budget.reason);
+            else budget.addEventListener('abort', () => reject(budget.reason), { once: true });
+          })])
+          : request);
         const text = samplingResult?.content?.text || '';
         if (text) {
           // stderr, never stdout — stdout is the JSON-RPC stream on stdio.
@@ -170,6 +195,11 @@ export class SamplingClient {
     }
 
     // 4. All fallbacks exhausted
+    if (budget?.aborted) {
+      const err = new Error(`LLM time budget of ${options.timeoutMs} ms ran out before a provider answered.`);
+      err.code = 'LLM_TIMEOUT';
+      throw err;
+    }
     throw new Error(
       'No LLM available: Ollama is not running, no API keys set (OPENAI_API_KEY / ANTHROPIC_API_KEY), and the MCP client does not support sampling.'
     );

package/src/core/SnapshotManager.js

@@ -12,6 +12,7 @@ import { gzip, gunzip } from 'zlib';
 import { promisify } from 'util';
 import { z } from 'zod';
 import { EventEmitter } from 'events';
+import { calculateSimilarity } from '../tools/tracking/trackChanges/differ.js';
 
 const gzipAsync = promisify(gzip);
 const gunzipAsync = promisify(gunzip);
@@ -362,9 +363,11 @@ export class SnapshotManager extends EventEmitter {
       if (metadata.compression && metadata.compression.enabled) {
         if (metadata.compression.algorithm === 'gzip') {
           content = await gunzipAsync(content);
-          content = content.toString();
         }
       }
+      // readSnapshotFile returns a Buffer; an uncompressed snapshot was
+      // returned as one, not as its text.
+      content = content.toString();
       
       // Reconstruct from delta if needed
       if (metadata.delta && metadata.delta.enabled) {
@@ -502,14 +505,21 @@ export class SnapshotManager extends EventEmitter {
       
       const history = [];
       const snapshotList = snapshots.snapshots;
-      
+
+      // Each snapshot's content is read once and carried to the next pair.
+      let currentContent = snapshotList.length > 1
+        ? await this.loadSnapshotContent(snapshotList[0].id)
+        : null;
+
       for (let i = 0; i < snapshotList.length - 1; i++) {
         const current = snapshotList[i];
         const previous = snapshotList[i + 1];
-        
+        const previousContent = await this.loadSnapshotContent(previous.id);
+
         // Calculate change metrics
-        const changeMetrics = await this.calculateChangeMetrics(previous, current);
-        
+        const changeMetrics = await this.calculateChangeMetrics(previous, current, previousContent, currentContent);
+        currentContent = previousContent;
+
         history.push({
           timestamp: current.metadata.timestamp,
           snapshotId: current.id,
@@ -842,7 +852,17 @@ export class SnapshotManager extends EventEmitter {
     throw new Error('Cannot reconstruct legacy delta snapshot: no diff data was stored for it. Re-fetch or re-create this snapshot from the source.');
   }
 
-  async calculateChangeMetrics(previousSnapshot, currentSnapshot) {
+  /** A stored snapshot's content, or null when it cannot be read back. */
+  async loadSnapshotContent(snapshotId) {
+    try {
+      const snapshot = await this.retrieveSnapshot(snapshotId);
+      return typeof snapshot.content === 'string' ? snapshot.content : null;
+    } catch {
+      return null;
+    }
+  }
+
+  async calculateChangeMetrics(previousSnapshot, currentSnapshot, previousContent = null, currentContent = null) {
     // Calculate various change metrics between snapshots
     const metrics = {
       sizeDelta: currentSnapshot.metadata.contentLength - previousSnapshot.metadata.contentLength,
@@ -852,14 +872,13 @@ export class SnapshotManager extends EventEmitter {
       similarity: 0
     };
     
-    // Calculate content similarity if different hashes
+    // Content similarity, computed from the stored content. A size-based
+    // estimate scored two different pages of the same length 1.0. null when
+    // either snapshot's content could not be read back.
     if (metrics.hashChanged) {
-      // This would require loading both snapshots' content
-      // For now, estimate based on size difference
-      metrics.similarity = 1 - Math.abs(metrics.sizeDelta) / Math.max(
-        currentSnapshot.metadata.contentLength,
-        previousSnapshot.metadata.contentLength
-      );
+      metrics.similarity = typeof previousContent === 'string' && typeof currentContent === 'string'
+        ? calculateSimilarity(previousContent, currentContent)
+        : null;
     } else {
       metrics.similarity = 1.0;
     }

package/src/core/analysis/ContentAnalyzer.js

@@ -10,10 +10,13 @@ import {
   needsWordSegmentation
 } from '../../utils/languageDetection.js';
 import nlp from 'compromise';
+import Sentiment from 'sentiment';
 import { z } from 'zod';
 import { splitSentences } from './sentenceUtils.js';
 import { LANGUAGE_STOP_WORDS } from './languageStopWords.js';
 
+const sentimentAnalyzer = new Sentiment();
+
 const ContentAnalyzerSchema = z.object({
   text: z.string().min(1),
   options: z.object({
@@ -845,25 +848,20 @@ export class ContentAnalyzer {
    */
   async analyzeSentiment(text, options = {}) {
     try {
-      const doc = nlp(text);
-      
-      // Simple sentiment analysis using compromise
-      // This is basic - for production use a dedicated sentiment library
-      const positiveWords = ['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'awesome', 'perfect', 'love', 'like', 'happy', 'pleased', 'satisfied'];
-      const negativeWords = ['bad', 'terrible', 'awful', 'horrible', 'hate', 'dislike', 'angry', 'sad', 'disappointed', 'frustrated', 'annoyed', 'upset'];
-      
-      const words = doc.terms().out('array').map(w => w.toLowerCase());
-      
-      let positiveCount = 0;
-      let negativeCount = 0;
-      
-      words.forEach(word => {
-        if (positiveWords.includes(word)) positiveCount++;
-        if (negativeWords.includes(word)) negativeCount++;
-      });
+      // AFINN-165 lexicon (about 3,400 English words scored -5..+5) with
+      // negation ("not happy" scores negative). A 25-word list used to leave
+      // clearly positive text "neutral" (R25).
+      const { calculation, tokens } = sentimentAnalyzer.analyze(text);
+      // Drop hits that are function words of the detected language: German
+      // "die" is -3 in the English lexicon.
+      const foreignStopWords = LANGUAGE_STOP_WORDS[options.detectedLanguage];
+      const scores = calculation
+        .map(entry => Object.entries(entry)[0])
+        .filter(([word]) => !foreignStopWords?.has(word))
+        .map(([, score]) => score);
+
+      const totalSentimentWords = scores.length;
 
-      const totalSentimentWords = positiveCount + negativeCount;
-      
       if (totalSentimentWords === 0) {
         // The lexicon is English. Finding none of its words in German or
         // Japanese text is not evidence of neutrality, so say the measure does
@@ -879,22 +877,25 @@ export class ContentAnalyzer {
             notApplicable: 'sentiment-lexicon-is-english-only'
           };
         }
+        // No opinion words at all: neutral, but with no evidence behind it.
         return {
           polarity: 0,
           subjectivity: 0,
           label: 'neutral',
-          confidence: 0.5
+          confidence: 0
         };
       }
 
-      const polarity = (positiveCount - negativeCount) / Math.max(words.length, 1);
-      const subjectivity = totalSentimentWords / Math.max(words.length, 1);
-      
+      // Net lexicon score per word, clamped to -1..1.
+      const net = scores.reduce((sum, score) => sum + score, 0);
+      const polarity = Math.max(-1, Math.min(1, net / Math.max(tokens.length, 1)));
+      const subjectivity = totalSentimentWords / Math.max(tokens.length, 1);
+
       let label = 'neutral';
-      if (polarity > 0.1) label = 'positive';
-      else if (polarity < -0.1) label = 'negative';
+      if (polarity > 0.05) label = 'positive';
+      else if (polarity < -0.05) label = 'negative';
 
-      const confidence = Math.min(1, totalSentimentWords / 10);
+      const confidence = Math.min(1, totalSentimentWords / 5);
 
       return {
         polarity: Math.round(polarity * 100) / 100,