crawlforge-mcp-server 6.21.1 → 6.22.0

package/src/core/ResearchOrchestrator.js

@@ -63,6 +63,35 @@ export function reportedFindings(keyFindings) {
   return material.length > 0 ? material : keyFindings;
 }
 
+// Citation markers left in extracted text: "(3)", "[12]", "[3, 4]", "[2-5]".
+// At most three digits, so a year such as "(2024)" stays.
+const CITATION_MARKER = /\s*(?:\(\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\)|\[\d{1,3}(?:\s*[,\u2013-]\s*\d{1,3})*\])/g;
+const LIST_MARKER = /^(?:[-*\u2022\u25AA\u2013]|\d{1,2}[.)])\s+/;
+
+/**
+ * The statements in one extracted key point. A key point is a sentence as the
+ * splitter found it, so a heading or list item above it arrives in the same
+ * string ("Built-in encryption\nQUIC was built with security in mind.").
+ * Each line becomes its own statement, except a line starting in lowercase,
+ * which continues the one before it (wrapped text). List and citation markers
+ * are removed. Heading-only lines are left for isAdmissibleClaim to reject.
+ *
+ * @param {string} point
+ * @returns {string[]}
+ */
+export function claimStatements(point) {
+  const blocks = [];
+  for (const raw of String(point).split('\n')) {
+    const line = raw.trim();
+    if (!line) continue;
+    if (blocks.length > 0 && /^[a-z]/.test(line)) blocks[blocks.length - 1] += ` ${line}`;
+    else blocks.push(line);
+  }
+  return blocks
+    .map(block => block.replace(LIST_MARKER, '').replace(CITATION_MARKER, '').replace(/\s+/g, ' ').trim())
+    .filter(Boolean);
+}
+
 /**
  * Page text for the plain-fetch fallback, one line per block element. The
  * regex tag-strip it replaces ran every block together on one line.
@@ -1384,7 +1413,8 @@ export class ResearchOrchestrator extends EventEmitter {
         const keyPoints = summary.keypoints || summary.keyPoints || [];
         if (keyPoints.length > 0) {
           const relevance = this.sourceRelevance(source);
-          keyPoints.forEach((point, index) => {
+          let claimIndex = 0;
+          keyPoints.forEach((keyPoint, index) => claimStatements(keyPoint).forEach(point => {
             const credibility = source.overallCredibility || 0.65;
             // A recommendation is not evidence. Two routes to the same flag: a
             // page promoting itself, and — whoever published it — a claim whose
@@ -1392,7 +1422,7 @@ export class ResearchOrchestrator extends EventEmitter {
             const promotional = isVendorSelfPromotion(point, source.link) ||
               isProductRecommendation(point);
             claims.push({
-              id: `${source.link}_claim_${index}`,
+              id: `${source.link}_claim_${claimIndex++}`,
               claim: point,
               source: source.link,
               sourceTitle: source.title,
@@ -1402,7 +1432,7 @@ export class ResearchOrchestrator extends EventEmitter {
               context: summary.supporting?.[index] || '',
               extractedAt: new Date().toISOString()
             });
-          });
+          }));
         }
       } catch (error) {
         this.logger.warn('Claim extraction failed', {
@@ -1464,31 +1494,33 @@ export class ResearchOrchestrator extends EventEmitter {
   /**
    * Keep only claims that can be research findings: prose about the topic, not
    * document front matter (author/affiliation blocks, DOI stubs, "Retrieved
-   * from" lines), not from a source the relevance analysis scored below
-   * MIN_CLAIM_RELEVANCE, and not a sentence the LLM scored as barely about the
-   * topic. Each gate falls back to the previous claim set rather than returning
-   * nothing — no claims means no findings at all.
+   * from" lines), and not a sentence the LLM scored as barely about the topic.
+   * Those two gates fall back to the previous claim set rather than returning
+   * nothing. A claim from an off-topic source — one the relevance analysis
+   * scored below MIN_CLAIM_RELEVANCE — is always dropped: when every source is
+   * off-topic, no findings is the honest result (R25 reported such claims as
+   * findings through the fallback). A source with no score is kept.
    */
   admitClaims(claims) {
     if (claims.length === 0) return claims;
 
-    const substantive = claims.filter(claim => isAdmissibleClaim(claim.claim));
-    const relevant = substantive.filter(
+    const relevant = claims.filter(
       claim => typeof claim.relevance !== 'number' || claim.relevance >= MIN_CLAIM_RELEVANCE
     );
-    const onTopic = relevant.filter(
+    const substantive = relevant.filter(claim => isAdmissibleClaim(claim.claim));
+    const onTopic = substantive.filter(
       claim => typeof claim.topicRelevance !== 'number' ||
         claim.topicRelevance >= MIN_CLAIM_TOPIC_RELEVANCE
     );
 
     const admitted = onTopic.length > 0
       ? onTopic
-      : (relevant.length > 0 ? relevant : (substantive.length > 0 ? substantive : claims));
+      : (substantive.length > 0 ? substantive : relevant);
 
     this.logger.debug('Claim admission', {
       candidates: claims.length,
-      substantive: substantive.length,
       relevant: relevant.length,
+      substantive: substantive.length,
       onTopic: onTopic.length,
       admitted: admitted.length
     });
@@ -2007,8 +2039,10 @@ export class ResearchOrchestrator extends EventEmitter {
           findings.push({
             finding: claim.claim,
             supportingClaims: group.claims.length,
+            supportingSources: group.sourceCount,
             credibility: group.avgCredibility,
-            sources: group.claims.map(c => c.source),
+            // One entry per source: a group holds several claims from one page.
+            sources: [...new Set(group.claims.map(c => c.source))],
             ...(claim.promotional ? { promotional: true } : {}),
             ...(typeof claim.topicRelevance === 'number'
               ? { topicRelevance: claim.topicRelevance }
@@ -2032,6 +2066,7 @@ export class ResearchOrchestrator extends EventEmitter {
         title: source.title,
         url: source.link,
         credibility: source.overallCredibility,
+        relevance: this.sourceRelevance(source),
         evidence: source.extractedContent?.substring(0, 300) + '...'
       }))
       .slice(0, 15);

package/src/core/SamplingClient.js

@@ -15,13 +15,24 @@
 
 import { ollamaBaseUrl as OLLAMA_BASE_URL, ollamaHeaders, selectOllamaModel } from '../utils/ollamaConfig.js';
 
+/**
+ * A provider call's own 30 s bound, cut short by the caller's time budget
+ * when there is one.
+ * @param {AbortSignal|null} [budget]
+ * @returns {AbortSignal}
+ */
+function callSignal(budget) {
+  const own = AbortSignal.timeout(30_000);
+  return budget ? AbortSignal.any([own, budget]) : own;
+}
+
 /**
  * Attempt an Ollama completion.
  * @param {string} prompt
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryOllama(prompt, { model, maxTokens } = {}) {
+async function tryOllama(prompt, { model, maxTokens, budget } = {}) {
   const ollamaModel = model || await selectOllamaModel();
   const url = `${OLLAMA_BASE_URL()}/api/generate`;
   const res = await fetch(url, {
@@ -33,7 +44,7 @@ async function tryOllama(prompt, { model, maxTokens } = {}) {
       stream: false,
       ...(maxTokens ? { options: { num_predict: maxTokens } } : {}),
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`Ollama HTTP ${res.status}`);
   const data = await res.json();
@@ -47,7 +58,7 @@ async function tryOllama(prompt, { model, maxTokens } = {}) {
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryOpenAI(prompt, { model, maxTokens } = {}) {
+async function tryOpenAI(prompt, { model, maxTokens, budget } = {}) {
   const apiKey = process.env.OPENAI_API_KEY;
   if (!apiKey) throw new Error('OPENAI_API_KEY not set');
   const base = (process.env.OPENAI_BASE_URL || 'https://api.openai.com').replace(/\/$/, '');
@@ -62,7 +73,7 @@ async function tryOpenAI(prompt, { model, maxTokens } = {}) {
       messages: [{ role: 'user', content: prompt }],
       ...(maxTokens ? { max_tokens: maxTokens } : {}),
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`OpenAI HTTP ${res.status}`);
   const data = await res.json();
@@ -75,7 +86,7 @@ async function tryOpenAI(prompt, { model, maxTokens } = {}) {
  * @param {object} options
  * @returns {Promise<string>}
  */
-async function tryAnthropic(prompt, { model, maxTokens } = {}) {
+async function tryAnthropic(prompt, { model, maxTokens, budget } = {}) {
   const apiKey = process.env.ANTHROPIC_API_KEY;
   if (!apiKey) throw new Error('ANTHROPIC_API_KEY not set');
   const base = (process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com').replace(/\/$/, '');
@@ -91,7 +102,7 @@ async function tryAnthropic(prompt, { model, maxTokens } = {}) {
       max_tokens: maxTokens || 1024,
       messages: [{ role: 'user', content: prompt }],
     }),
-    signal: AbortSignal.timeout(30_000),
+    signal: callSignal(budget),
   });
   if (!res.ok) throw new Error(`Anthropic HTTP ${res.status}`);
   const data = await res.json();
@@ -116,12 +127,18 @@ export class SamplingClient {
    * @param {string} [options.model] - Override model name
    * @param {number} [options.maxTokens] - Max tokens for response
    * @param {string} [options.systemPrompt] - Optional system-level instruction
+   * @param {number} [options.timeoutMs] - Time budget for the whole chain. Each
+   *   provider call stops at its own 30 s bound or when the budget runs out,
+   *   whichever comes first, and the sampling fallback is raced against it.
+   *   An exhausted budget throws an error with code 'LLM_TIMEOUT'.
    * @returns {Promise<{ text: string, provider: string }>}
    */
   async complete(prompt, options = {}) {
     const fullPrompt = options.systemPrompt
       ? `${options.systemPrompt}\n\n${prompt}`
       : prompt;
+    const budget = options.timeoutMs != null ? AbortSignal.timeout(Math.max(0, options.timeoutMs)) : null;
+    options = { ...options, budget };
 
     // 1. Try Ollama (local, no API key)
     try {
@@ -153,11 +170,19 @@ export class SamplingClient {
     // 3. Try MCP sampling (client-side LLM)
     if (this._mcpServer?.server?.createMessage) {
       try {
-        const samplingResult = await this._mcpServer.server.createMessage({
+        const request = this._mcpServer.server.createMessage({
           messages: [{ role: 'user', content: { type: 'text', text: fullPrompt } }],
           maxTokens: options.maxTokens || 1024,
           includeContext: 'none',
-        });
+        }, budget ? { signal: budget } : undefined);
+        // Raced as well: a client that ignores the cancellation would
+        // otherwise hold the caller past its budget.
+        const samplingResult = await (budget
+          ? Promise.race([request, new Promise((_, reject) => {
+            if (budget.aborted) reject(budget.reason);
+            else budget.addEventListener('abort', () => reject(budget.reason), { once: true });
+          })])
+          : request);
         const text = samplingResult?.content?.text || '';
         if (text) {
           // stderr, never stdout — stdout is the JSON-RPC stream on stdio.
@@ -170,6 +195,11 @@ export class SamplingClient {
     }
 
     // 4. All fallbacks exhausted
+    if (budget?.aborted) {
+      const err = new Error(`LLM time budget of ${options.timeoutMs} ms ran out before a provider answered.`);
+      err.code = 'LLM_TIMEOUT';
+      throw err;
+    }
     throw new Error(
       'No LLM available: Ollama is not running, no API keys set (OPENAI_API_KEY / ANTHROPIC_API_KEY), and the MCP client does not support sampling.'
     );

package/src/core/SnapshotManager.js

@@ -12,6 +12,7 @@ import { gzip, gunzip } from 'zlib';
 import { promisify } from 'util';
 import { z } from 'zod';
 import { EventEmitter } from 'events';
+import { calculateSimilarity } from '../tools/tracking/trackChanges/differ.js';
 
 const gzipAsync = promisify(gzip);
 const gunzipAsync = promisify(gunzip);
@@ -362,9 +363,11 @@ export class SnapshotManager extends EventEmitter {
       if (metadata.compression && metadata.compression.enabled) {
         if (metadata.compression.algorithm === 'gzip') {
           content = await gunzipAsync(content);
-          content = content.toString();
         }
       }
+      // readSnapshotFile returns a Buffer; an uncompressed snapshot was
+      // returned as one, not as its text.
+      content = content.toString();
       
       // Reconstruct from delta if needed
       if (metadata.delta && metadata.delta.enabled) {
@@ -502,14 +505,21 @@ export class SnapshotManager extends EventEmitter {
       
       const history = [];
       const snapshotList = snapshots.snapshots;
-      
+
+      // Each snapshot's content is read once and carried to the next pair.
+      let currentContent = snapshotList.length > 1
+        ? await this.loadSnapshotContent(snapshotList[0].id)
+        : null;
+
       for (let i = 0; i < snapshotList.length - 1; i++) {
         const current = snapshotList[i];
         const previous = snapshotList[i + 1];
-        
+        const previousContent = await this.loadSnapshotContent(previous.id);
+
         // Calculate change metrics
-        const changeMetrics = await this.calculateChangeMetrics(previous, current);
-        
+        const changeMetrics = await this.calculateChangeMetrics(previous, current, previousContent, currentContent);
+        currentContent = previousContent;
+
         history.push({
           timestamp: current.metadata.timestamp,
           snapshotId: current.id,
@@ -842,7 +852,17 @@ export class SnapshotManager extends EventEmitter {
     throw new Error('Cannot reconstruct legacy delta snapshot: no diff data was stored for it. Re-fetch or re-create this snapshot from the source.');
   }
 
-  async calculateChangeMetrics(previousSnapshot, currentSnapshot) {
+  /** A stored snapshot's content, or null when it cannot be read back. */
+  async loadSnapshotContent(snapshotId) {
+    try {
+      const snapshot = await this.retrieveSnapshot(snapshotId);
+      return typeof snapshot.content === 'string' ? snapshot.content : null;
+    } catch {
+      return null;
+    }
+  }
+
+  async calculateChangeMetrics(previousSnapshot, currentSnapshot, previousContent = null, currentContent = null) {
     // Calculate various change metrics between snapshots
     const metrics = {
       sizeDelta: currentSnapshot.metadata.contentLength - previousSnapshot.metadata.contentLength,
@@ -852,14 +872,13 @@ export class SnapshotManager extends EventEmitter {
       similarity: 0
     };
     
-    // Calculate content similarity if different hashes
+    // Content similarity, computed from the stored content. A size-based
+    // estimate scored two different pages of the same length 1.0. null when
+    // either snapshot's content could not be read back.
     if (metrics.hashChanged) {
-      // This would require loading both snapshots' content
-      // For now, estimate based on size difference
-      metrics.similarity = 1 - Math.abs(metrics.sizeDelta) / Math.max(
-        currentSnapshot.metadata.contentLength,
-        previousSnapshot.metadata.contentLength
-      );
+      metrics.similarity = typeof previousContent === 'string' && typeof currentContent === 'string'
+        ? calculateSimilarity(previousContent, currentContent)
+        : null;
     } else {
       metrics.similarity = 1.0;
     }

package/src/core/analysis/ContentAnalyzer.js

@@ -10,10 +10,13 @@ import {
   needsWordSegmentation
 } from '../../utils/languageDetection.js';
 import nlp from 'compromise';
+import Sentiment from 'sentiment';
 import { z } from 'zod';
 import { splitSentences } from './sentenceUtils.js';
 import { LANGUAGE_STOP_WORDS } from './languageStopWords.js';
 
+const sentimentAnalyzer = new Sentiment();
+
 const ContentAnalyzerSchema = z.object({
   text: z.string().min(1),
   options: z.object({
@@ -845,25 +848,20 @@ export class ContentAnalyzer {
    */
   async analyzeSentiment(text, options = {}) {
     try {
-      const doc = nlp(text);
-      
-      // Simple sentiment analysis using compromise
-      // This is basic - for production use a dedicated sentiment library
-      const positiveWords = ['good', 'great', 'excellent', 'amazing', 'wonderful', 'fantastic', 'awesome', 'perfect', 'love', 'like', 'happy', 'pleased', 'satisfied'];
-      const negativeWords = ['bad', 'terrible', 'awful', 'horrible', 'hate', 'dislike', 'angry', 'sad', 'disappointed', 'frustrated', 'annoyed', 'upset'];
-      
-      const words = doc.terms().out('array').map(w => w.toLowerCase());
-      
-      let positiveCount = 0;
-      let negativeCount = 0;
-      
-      words.forEach(word => {
-        if (positiveWords.includes(word)) positiveCount++;
-        if (negativeWords.includes(word)) negativeCount++;
-      });
+      // AFINN-165 lexicon (about 3,400 English words scored -5..+5) with
+      // negation ("not happy" scores negative). A 25-word list used to leave
+      // clearly positive text "neutral" (R25).
+      const { calculation, tokens } = sentimentAnalyzer.analyze(text);
+      // Drop hits that are function words of the detected language: German
+      // "die" is -3 in the English lexicon.
+      const foreignStopWords = LANGUAGE_STOP_WORDS[options.detectedLanguage];
+      const scores = calculation
+        .map(entry => Object.entries(entry)[0])
+        .filter(([word]) => !foreignStopWords?.has(word))
+        .map(([, score]) => score);
+
+      const totalSentimentWords = scores.length;
 
-      const totalSentimentWords = positiveCount + negativeCount;
-      
       if (totalSentimentWords === 0) {
         // The lexicon is English. Finding none of its words in German or
         // Japanese text is not evidence of neutrality, so say the measure does
@@ -879,22 +877,25 @@ export class ContentAnalyzer {
             notApplicable: 'sentiment-lexicon-is-english-only'
           };
         }
+        // No opinion words at all: neutral, but with no evidence behind it.
         return {
           polarity: 0,
           subjectivity: 0,
           label: 'neutral',
-          confidence: 0.5
+          confidence: 0
         };
       }
 
-      const polarity = (positiveCount - negativeCount) / Math.max(words.length, 1);
-      const subjectivity = totalSentimentWords / Math.max(words.length, 1);
-      
+      // Net lexicon score per word, clamped to -1..1.
+      const net = scores.reduce((sum, score) => sum + score, 0);
+      const polarity = Math.max(-1, Math.min(1, net / Math.max(tokens.length, 1)));
+      const subjectivity = totalSentimentWords / Math.max(tokens.length, 1);
+
       let label = 'neutral';
-      if (polarity > 0.1) label = 'positive';
-      else if (polarity < -0.1) label = 'negative';
+      if (polarity > 0.05) label = 'positive';
+      else if (polarity < -0.05) label = 'negative';
 
-      const confidence = Math.min(1, totalSentimentWords / 10);
+      const confidence = Math.min(1, totalSentimentWords / 5);
 
       return {
         polarity: Math.round(polarity * 100) / 100,

package/src/core/crawlers/BFSCrawler.js

@@ -44,6 +44,10 @@ export class BFSCrawler {
     this.sessionContext = sessionContext;
 
     this.visited = new Map(); // normalized key -> the URL as fetched
+    // Normalized keys of the pages fetched, by where each fetch landed after
+    // redirects. Kept apart from visited so the page a redirect lands on is
+    // not fetched again, without spending a second max_pages slot on it.
+    this.landed = new Set();
     this.results = [];
     this.errors = [];
     this.filterDecisions = []; // Track filtering decisions for analysis
@@ -153,7 +157,7 @@ export class BFSCrawler {
 
     // Check if already visited
     const normalizedUrl = normalizeUrl(url);
-    if (this.visited.has(normalizedUrl)) {
+    if (this.visited.has(normalizedUrl) || this.landed.has(normalizedUrl)) {
       return;
     }
     // The form that goes on the wire: a bare origin gains its root "/",
@@ -215,7 +219,7 @@ export class BFSCrawler {
     // top of this task ran before the awaited robots lookup, so concurrent
     // queue tasks may have filled the budget (or claimed this URL) since.
     // This block is synchronous, so the cap is exact.
-    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl)) {
+    if (this.visited.size >= this.maxPages || this.visited.has(normalizedUrl) || this.landed.has(normalizedUrl)) {
       return;
     }
     this.visited.set(normalizedUrl, url);
@@ -263,17 +267,41 @@ export class BFSCrawler {
         if (cacheKey) await this.cache.set(cacheKey, pageData);
       }
 
+      // Links resolve against where the fetch landed, not the URL asked for:
+      // docs.python.org answers /3/tutorial with a redirect to /3/tutorial/,
+      // and appetite.html resolved against the first is /3/appetite.html, a
+      // 404 (R25). A seed that lands on another origin (http -> https, apex
+      // -> www) moves the crawl's origin with it, or every link it holds
+      // would read as external.
+      const pageUrl = pageData.finalUrl || url;
+      const landedKey = normalizeUrl(pageUrl);
+      if (isSeed && landedKey !== normalizedUrl) this.baseUrl = new URL(landedKey);
+      // Another link (one that redirects here, or this URL itself) reached the
+      // same page first: this copy is a duplicate.
+      if (this.landed.has(landedKey)) return;
+      this.landed.add(landedKey);
+
       // Process links for analysis
       if (this.enableLinkAnalysis && this.linkAnalyzer && pageData.links) {
         // Parse the page once and reuse for every link; re-parsing per link is
         // O(links × page size) and starves the event loop on link-dense pages.
         const $page = pageData.originalHtml ? load(pageData.originalHtml) : null;
         const pageBodyText = $page ? $page('body').text() : '';
+        // Index the anchors once by raw href, first one winning: a per-link
+        // `a[href="..."]` query scanned the whole page for every link (16,766
+        // scans of the 7.3 MB python changelog, R25) and blocked other calls.
+        const anchorsByHref = new Map();
+        if ($page) {
+          $page('a[href]').each((_, element) => {
+            const href = $page(element).attr('href');
+            if (!anchorsByHref.has(href)) anchorsByHref.set(href, element);
+          });
+        }
         for (const link of pageData.links) {
-          const absoluteUrl = this.resolveUrl(link, url);
+          const absoluteUrl = this.resolveUrl(link, pageUrl);
           if (absoluteUrl) {
             // Extract anchor text and context from link
-            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText);
+            const linkMetadata = this.extractLinkMetadata(link, $page, pageBodyText, anchorsByHref);
             this.linkAnalyzer.addLink(normalizedUrl, normalizeUrl(absoluteUrl), linkMetadata);
           }
         }
@@ -301,8 +329,9 @@ export class BFSCrawler {
         for (const link of pageData.links) {
           if (this.visited.size >= this.maxPages) break;
           
-          const absoluteUrl = this.resolveUrl(link, url);
-          if (absoluteUrl && !this.visited.has(normalizeUrl(absoluteUrl))) {
+          const absoluteUrl = this.resolveUrl(link, pageUrl);
+          const key = absoluteUrl && normalizeUrl(absoluteUrl);
+          if (key && !this.visited.has(key) && !this.landed.has(key)) {
             // Not awaited: this task already holds a queue slot, so awaiting a
             // child would keep that slot pinned for the rest of the recursive
             // crawl (starving other tasks when concurrency <= depth, and making
@@ -395,7 +424,9 @@ export class BFSCrawler {
       // readBody decodes with the body's real charset — response.text() is
       // UTF-8-only and mangled ISO-8859-1 pages ("portugu�s" on lua.org).
       const html = await readBody(response);
-      return this.parsePage(html, url);
+      // response.url is where the redirects ended; links resolve against it.
+      const finalUrl = response.url || url;
+      return { ...this.parsePage(html, finalUrl), finalUrl };
     } catch (error) {
       clearTimeout(timeoutId);
       throw error;
@@ -656,18 +687,21 @@ export class BFSCrawler {
    * @param {string} href - The href attribute value
    * @param {Object} $ - Cheerio document for the page (parsed once per page)
    * @param {string} bodyText - Pre-computed body text of the page
+   * @param {Map} anchorsByHref - Raw href -> first <a> element with it
    * @returns {Object} Link metadata
    */
-  extractLinkMetadata(href, $, bodyText = '') {
+  extractLinkMetadata(href, $, bodyText = '', anchorsByHref) {
     if (!$) return {};
 
     try {
-      const linkElement = $(`a[href="${href}"]`).first();
+      const element = anchorsByHref.get(href);
 
-      if (linkElement.length === 0) {
+      if (!element) {
         return { href };
       }
 
+      const linkElement = $(element);
+
       const anchorText = linkElement.text().trim();
       const title = linkElement.attr('title');
       const rel = linkElement.attr('rel');
@@ -968,6 +1002,10 @@ export class BFSCrawler {
     const segments = pathname.split('/').filter(s => s);
     
     if (segments.length === 0) return '/';
+    // A file (bugs.html) is a page, not a directory: no trailing "/" or "/...".
+    if (segments.length <= 2 && /.\.[a-z0-9]+$/i.test(segments[segments.length - 1])) {
+      return `/${segments.join('/')}`;
+    }
     if (segments.length === 1) return `/${segments[0]}/`;
     
     // Return first two segments as pattern

package/src/core/processing/ContentProcessor.js

@@ -8,7 +8,7 @@ import { JSDOM } from 'jsdom';
 import * as cheerio from 'cheerio';
 import { z } from 'zod';
 import { ContentQualityAssessor } from '../../utils/contentUtils.js';
-import { recoverDroppedTables } from '../../tools/scrape/_mainContent.js';
+import { recoverDroppedTables, unwrapMediaWikiHeadings, tooLargeForReadability } from '../../tools/scrape/_mainContent.js';
 import { pageTitle } from '../../utils/pageTitle.js';
 import { flattenText } from 'crawlforge-extractors';
 
@@ -73,7 +73,9 @@ export class ContentProcessor {
       const document = dom.window.document;
 
       // Extract main content using Mozilla Readability
-      if (processingOptions.removeBoilerplate) {
+      // A page too large for Readability keeps no `readability` block (R25).
+      if (processingOptions.removeBoilerplate && !tooLargeForReadability(html)) {
+        unwrapMediaWikiHeadings(document);
         const reader = new Readability(document, {
           debug: false,
           maxElemsToDivide: 300,

package/src/core/queue/QueueManager.js

@@ -1,4 +1,4 @@
-import PQueue from 'p-queue';
+import PQueue, { TimeoutError } from 'p-queue';
 
 export class QueueManager {
   constructor(options = {}) {
@@ -46,7 +46,21 @@ export class QueueManager {
 
   async add(fn, options = {}) {
     const { priority = 0 } = options;
-    return this.queue.add(fn, { priority });
+    try {
+      return await this.queue.add(fn, { priority });
+    } catch (error) {
+      // p-queue's message gives the queue state when the task STARTED, which
+      // reads like a queue problem. The task itself ran too long, often
+      // because synchronous work elsewhere held the event loop.
+      if (error instanceof TimeoutError) {
+        throw new Error(
+          `Task ran over its ${this.queue.timeout / 1000} s time budget; ` +
+          'it may have been starved by other work running in this process.',
+          { cause: error }
+        );
+      }
+      throw error;
+    }
   }
 
   async addAll(tasks, options = {}) {