crawlforge-mcp-server 5.0.2 → 5.0.3

package/CLAUDE.md

@@ -62,7 +62,7 @@ These guidelines are working if: fewer unnecessary changes in diffs, fewer rewri
 
 CrawlForge MCP Server - A professional MCP (Model Context Protocol) server providing 27 web scraping, crawling, and content processing tools (5 inline + 22 advanced).
 
-**Current Version:** 5.0.2
+**Current Version:** 5.0.3
 
 ## Development Commands
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "crawlforge-mcp-server",
-  "version": "5.0.2",
+  "version": "5.0.3",
   "mcpName": "io.github.mysleekdesigns/crawlforge-mcp-server",
   "description": "CrawlForge MCP Server - Professional Model Context Protocol server with 27 web scraping, crawling, deep-research, and autonomous-extraction tools. Returns clean Markdown and structured JSON for Claude, Cursor, and any MCP client. Defaults to local Ollama for LLM extraction (no API key needed); OpenAI/Anthropic available as opt-in. Includes a unified multi-format scrape tool, an autonomous agent, pre-built site templates, and Camoufox stealth browsing.",
   "main": "server.js",

package/server.js

@@ -794,6 +794,8 @@ registerToolIfEnabled("scrape_with_actions", {
       distance: z.number().min(0).optional().describe("scroll: pixels to scroll"),
       smooth: z.boolean().optional().describe("scroll: smooth scrolling"),
       toElement: z.string().optional().describe("scroll: selector to scroll to"),
+      x: z.number().min(0).optional().describe("scroll: absolute X coordinate to scroll to (window.scrollTo; with y, takes precedence over direction/distance)"),
+      y: z.number().min(0).optional().describe("scroll: absolute Y coordinate to scroll to (window.scrollTo; with x, takes precedence over direction/distance)"),
       // screenshot
       fullPage: z.boolean().optional().describe("screenshot: capture full page"),
       quality: z.number().min(0).max(100).optional().describe("screenshot: jpeg quality"),
@@ -1299,8 +1301,8 @@ registerToolIfEnabled("localization", {
       extraHTTPHeaders: z.record(z.string()).optional(),
       userAgent: z.string().optional()
     }).optional().describe("Browser context options for locale emulation"),
-    content: z.string().optional().describe("Content for auto-detection of language and locale"),
-    url: z.string().url().optional().describe("URL for geo-blocking detection or auto-detection"),
+    content: z.string().optional().describe("Page content (HTML or plain text) to analyze — required for auto_detect; no fetching is performed"),
+    url: z.string().url().optional().describe("URL — required for handle_geo_blocking; for auto_detect it is optional metadata used only as a TLD country hint (the page is never fetched)"),
     response: z.object({
       status: z.number(),
       body: z.string().optional(),
@@ -1336,7 +1338,7 @@ registerToolIfEnabled("localization", {
         result = await localizationManager.detectGeoBlocking(params.url, params.response);
         break;
       case 'auto_detect':
-        if (!params.content || !params.url) throw new Error('content and url are required for auto_detect operation');
+        if (!params.content) throw new Error('content is required for auto_detect operation');
         result = await localizationManager.autoDetectLocalization(params.content, params.url);
         break;
       case 'get_stats':

package/src/cli/commands/llmstxt.js

@@ -10,17 +10,24 @@ export function register(program) {
     .command('llmstxt <url>')
     .description('Generate llms.txt for a website (AI compliance file)')
     .option('--include-full', 'Also generate llms-full.txt')
-    .option('--max-pages <n>', 'Maximum pages to analyze', '50')
+    .option('--max-pages <n>', 'Maximum pages to analyze, 10-500', '50')
     .action(async (url, opts, cmd) => {
       const globals = cmd.parent.opts();
       const cliFlags = { json: globals.json, pretty: globals.pretty, quiet: globals.quiet };
+      // GenerateLLMsTxtSchema requires analysisOptions.maxPages between 10 and 500 —
+      // validate here so users get a clear message instead of a raw zod error.
+      const maxPages = parseInt(opts.maxPages, 10);
+      if (!Number.isInteger(maxPages) || maxPages < 10 || maxPages > 500) {
+        process.stderr.write('Error: --max-pages must be between 10 and 500\n');
+        process.exit(1);
+      }
       const tool = new GenerateLLMsTxtTool(getToolConfig('generate_llms_txt'));
       // GenerateLLMsTxtSchema expects: url, format ('both'|'llms-txt'|'llms-full-txt'),
       // analysisOptions.maxPages.
       await runTool(tool, {
         url,
         format: opts.includeFull ? 'both' : 'llms-txt',
-        analysisOptions: { maxPages: parseInt(opts.maxPages, 10) }
+        analysisOptions: { maxPages }
       }, cliFlags);
     });
 }

package/src/core/ActionExecutor.js

@@ -72,7 +72,12 @@ const ScrollActionSchema = BaseActionSchema.extend({
   direction: z.enum(['up', 'down', 'left', 'right']).default('down'),
   distance: z.number().min(0).default(100),
   smooth: z.boolean().default(true),
-  toElement: z.string().optional()
+  toElement: z.string().optional(),
+  // Absolute scroll-to coordinates (window.scrollTo). When present they take
+  // precedence over direction/distance. Matches the CLI guide's documented
+  // action-script format: { "type": "scroll", "x": 0, "y": 500 }.
+  x: z.number().min(0).optional(),
+  y: z.number().min(0).optional()
 });
 
 const ScreenshotActionSchema = BaseActionSchema.extend({
@@ -708,6 +713,20 @@ export class ActionExecutor extends EventEmitter {
       return { scrolledToElement: action.toElement };
     }
 
+    // Absolute scroll-to coordinates take precedence over direction/distance.
+    // window.scrollTo (not scrollBy/mouse.wheel, which are relative deltas) is
+    // the standard Playwright pattern for absolute positioning. A missing axis
+    // defaults to 0, matching the plain window.scrollTo(x, y) call form.
+    if (action.x !== undefined || action.y !== undefined) {
+      const targetX = action.x ?? 0;
+      const targetY = action.y ?? 0;
+      await page.evaluate(
+        ([x, y]) => window.scrollTo(x, y),
+        [targetX, targetY]
+      );
+      return { scrolledTo: { x: targetX, y: targetY }, mode: 'absolute' };
+    }
+
     if (humanBehaviorSimulator) {
       // Use human-like scrolling behavior
       await humanBehaviorSimulator.simulateScroll(page, {

package/src/core/AgentOrchestrator.js

@@ -175,9 +175,20 @@ export class AgentOrchestrator {
     let searchQueries = [prompt]; // fallback: use raw prompt as query
     try {
       const planPrompt =
-        `Decompose this research task into 1-3 concise web search queries (one per line, no bullets):\n\n${prompt}`;
+        `Decompose this research task into 1-3 concise web search queries. ` +
+        `Output ONLY the queries, one per line, no preamble or numbering:\n\n${prompt}`;
       const { text } = await this._getSamplingClient().complete(planPrompt, { maxTokens: 200 });
-      const lines = text.split('\n').map(l => l.replace(/^[-*\d.)\s]+/, '').trim()).filter(Boolean);
+      const lines = text.split('\n')
+        .map(l => l.replace(/^[-*\d.)\s]+/, '').trim())
+        .filter(Boolean)
+        // Drop preamble/garbage lines ("Here are 3 concise web search queries:", …)
+        // so they never become search query #1 and poison the URL queue.
+        .filter(l =>
+          !l.endsWith(':') &&
+          !/^here (are|is)\b/i.test(l) &&
+          !/search quer(y|ies)/i.test(l) &&
+          l.length <= 100
+        );
       if (lines.length > 0) searchQueries = lines.slice(0, 3);
     } catch {
       // Sampling unavailable — use raw prompt
@@ -185,6 +196,14 @@ export class AgentOrchestrator {
 
     // ── GATHER (search) ───────────────────────────────────────────────────────
     const urlQueue = [...seedUrls]; // start with any user-provided seeds
+    // Sites named directly in the prompt (full URLs or bare domains like
+    // "news.ycombinator.com") are the most authoritative sources for the task —
+    // queue them ahead of search results.
+    const namedSites = prompt.match(/https?:\/\/[^\s"'<>]+|(?<![\w.@/-])[a-z0-9][\w-]*(?:\.[a-z0-9][\w-]*)*\.[a-z]{2,}(?![\w-])/gi) || [];
+    for (const site of namedSites) {
+      const url = (/^https?:\/\//i.test(site) ? site : `https://${site}`).replace(/[.,;:!?)]+$/, '');
+      if (!urlQueue.includes(url)) urlQueue.push(url);
+    }
     const searchResults = [];
 
     if (urlQueue.length < capUrls) {
@@ -227,8 +246,10 @@ export class AgentOrchestrator {
         const { textContent, finalUrl } = await fetchAndParse(url, { timeoutMs: 10000 });
         if (!isRelevant(textContent, prompt)) continue;
         step++;
+        const sr = searchResults.find(s => s.url === url);
         evidence.push({
           url: finalUrl,
+          title: sr ? sr.title : '',
           text: truncate(textContent),
           step
         });
@@ -236,7 +257,20 @@ export class AgentOrchestrator {
     }
 
     // ── SHAPE ─────────────────────────────────────────────────────────────────
-    const combinedText = evidence.map(e => `--- Source: ${e.url} ---\n${e.text}`).join('\n\n');
+    // Order evidence by simple relevance to the prompt (term overlap with
+    // url+title+text) instead of raw queue order, then give every source a
+    // per-source slice of the synthesis budget so no source is silently cut off.
+    const promptTerms = prompt.toLowerCase().split(/\s+/).filter(t => t.length > 3);
+    const orderedEvidence = evidence
+      .map(e => ({
+        ...e,
+        _score: promptTerms.filter(t => `${e.url} ${e.title || ''} ${e.text}`.toLowerCase().includes(t)).length
+      }))
+      .sort((a, b) => b._score - a._score);
+    const perSourceCap = Math.max(1500, Math.floor(12000 / Math.max(evidence.length, 1)));
+    const combinedText = orderedEvidence
+      .map(e => `--- Source: ${e.url} ---\n${truncate(e.text, perSourceCap)}`)
+      .join('\n\n');
 
     if (!combinedText.trim()) {
       return {
@@ -284,9 +318,14 @@ export class AgentOrchestrator {
 
     try {
       const synthesisPrompt =
-        `You are a research assistant. Based on the sources below, answer this task:\n\n` +
+        `You are a research assistant. Answer this task using ONLY the sources below:\n\n` +
         `Task: ${prompt}\n\n` +
-        `${truncate(combinedText, 12000)}\n\n` +
+        `${combinedText}\n\n` +
+        `Rules:\n` +
+        `- Answer ONLY from the provided sources; do not use outside knowledge.\n` +
+        `- Cite the exact source URL(s) you used.\n` +
+        `- If the sources do not contain the answer, say so explicitly.\n` +
+        `- NEVER invent or guess a URL; cite only URLs that appear in the sources above.\n\n` +
         `Provide a clear, concise answer.`;
 
       const { text } = await this._getSamplingClient().complete(synthesisPrompt, { maxTokens: 1024 });

package/src/core/ChangeTracker.js

@@ -211,12 +211,14 @@ export class ChangeTracker extends EventEmitter {
       // Calculate change significance
       const significance = await this.calculateChangeSignificance(changeAnalysis, baseline.options);
       
-      // Create change record
+      // Create change record. An unchanged compare (significance 'none' ⇒
+      // hasChanges:false) must report a neutral changeType — classifyChangeType
+      // falls through to 'text_change' even when nothing changed at all.
       const changeRecord = {
         url,
         timestamp: Date.now(),
         baselineVersion: baseline.version,
-        changeType: this.classifyChangeType(changeAnalysis),
+        changeType: significance === 'none' ? 'none' : this.classifyChangeType(changeAnalysis),
         significance,
         details: changeAnalysis,
         metrics: {
@@ -742,12 +744,17 @@ export class ChangeTracker extends EventEmitter {
       currentContent = currentContent.toLowerCase();
     }
     
-    // Word-level diff
+    // Word-level diff. Only record an entry when something actually changed
+    // (mirrors line_diff below) — unconditionally pushing an empty word_diff
+    // made every unchanged compare report "Text content changed".
     const wordDiff = diffWords(baselineContent, currentContent);
-    textChanges.push({
-      type: 'word_diff',
-      changes: wordDiff.filter(part => part.added || part.removed)
-    });
+    const wordDiffChanges = wordDiff.filter(part => part.added || part.removed);
+    if (wordDiffChanges.length > 0) {
+      textChanges.push({
+        type: 'word_diff',
+        changes: wordDiffChanges
+      });
+    }
     
     // Line-level diff for structured content
     const lineDiff = diffLines(baselineContent, currentContent);
@@ -903,18 +910,23 @@ export class ChangeTracker extends EventEmitter {
     
     const tagSimilarity = this.calculateTagSimilarity(baselineElements, currentElements);
     const hierarchySimilarity = this.calculateHierarchySimilarity(baseline.hierarchy, current.hierarchy);
-    
-    return (tagSimilarity + hierarchySimilarity) / 2;
+
+    // Clamp defensively — this is a 0-1 metric and must never leave that range.
+    return Math.max(0, Math.min(1, (tagSimilarity + hierarchySimilarity) / 2));
   }
-  
+
   calculateTagSimilarity(baselineElements, currentElements) {
-    const baselineTags = baselineElements.map(el => el.tag);
-    const currentTags = currentElements.map(el => el.tag);
-    
-    const intersection = baselineTags.filter(tag => currentTags.includes(tag));
+    // Jaccard similarity: intersection and union must both operate on SETS.
+    // The old code intersected the raw (duplicate-laden) tag list against a
+    // set union, so repeated tags (div, p, ...) inflated the numerator and
+    // produced impossible scores > 1 (e.g. the observed 1.05).
+    const baselineTags = new Set(baselineElements.map(el => el.tag));
+    const currentTags = new Set(currentElements.map(el => el.tag));
+
+    const intersection = [...baselineTags].filter(tag => currentTags.has(tag));
     const union = new Set([...baselineTags, ...currentTags]);
-    
-    return intersection.length / union.size;
+
+    return union.size === 0 ? 1 : intersection.length / union.size;
   }
   
   calculateHierarchySimilarity(baseline, current) {

package/src/core/LocalizationManager.js

@@ -548,7 +548,7 @@ export class LocalizationManager extends EventEmitter {
   /**
    * Auto-detect appropriate localization from content
    * @param {string} content - Web page content
-   * @param {string} url - Source URL
+   * @param {string} [url] - Optional source URL (used only as a TLD country hint)
    * @returns {Object} - Detected localization settings
    */
   async autoDetectLocalization(content, url) {
@@ -698,14 +698,40 @@ export class LocalizationManager extends EventEmitter {
   }
   
   getDateFormat(countryCode) {
+    // Audited against CLDR short-date patterns / Wikipedia "List of date
+    // formats by country". Covers every SUPPORTED_COUNTRIES entry; the
+    // fallback is DD/MM/YYYY (day-first), which most of the world uses —
+    // MM/DD/YYYY is essentially US-only.
     const formats = {
       'US': 'MM/DD/YYYY',
       'GB': 'DD/MM/YYYY',
       'DE': 'DD.MM.YYYY',
-      'JP': 'YYYY/MM/DD'
+      'FR': 'DD/MM/YYYY',
+      'JP': 'YYYY/MM/DD',
+      'CN': 'YYYY/MM/DD',
+      'AU': 'DD/MM/YYYY',
+      'CA': 'YYYY-MM-DD',
+      'IT': 'DD/MM/YYYY',
+      'ES': 'DD/MM/YYYY',
+      'RU': 'DD.MM.YYYY',
+      'BR': 'DD/MM/YYYY',
+      'IN': 'DD/MM/YYYY',
+      'KR': 'YYYY.MM.DD',
+      'MX': 'DD/MM/YYYY',
+      'NL': 'DD-MM-YYYY',
+      'SE': 'YYYY-MM-DD',
+      'NO': 'DD.MM.YYYY',
+      'SA': 'DD/MM/YYYY',
+      'AE': 'DD/MM/YYYY',
+      'TR': 'DD.MM.YYYY',
+      'IL': 'DD.MM.YYYY',
+      'TH': 'DD/MM/YYYY',
+      'SG': 'DD/MM/YYYY',
+      'PL': 'DD.MM.YYYY',
+      'ZA': 'YYYY/MM/DD'
     };
-    
-    return formats[countryCode] || 'MM/DD/YYYY';
+
+    return formats[countryCode] || 'DD/MM/YYYY';
   }
   
   getNumberFormat(countryCode) {
@@ -1275,7 +1301,17 @@ export class LocalizationManager extends EventEmitter {
       const detectedLang = await this.analyzeTextLanguage(textSample);
       if (detectedLang) {
         detection.evidence.push(`Text analysis: ${detectedLang.language} (${detectedLang.confidence}%)`);
-        detection.confidence += detectedLang.confidence / 100 * 0.2;
+        if (!detection.detectedLanguage) {
+          // Text analysis is the primary (only) language evidence — use it as
+          // the detected language, weighted strongly. Previously the result
+          // was pushed to evidence but detectedLanguage stayed null and the
+          // 0.2 corroboration weight left confidence near zero.
+          detection.detectedLanguage = detectedLang.language;
+          detection.confidence += detectedLang.confidence / 100 * 0.6;
+        } else {
+          // Corroborates (or contradicts) HTML/meta evidence — lower weight
+          detection.confidence += detectedLang.confidence / 100 * 0.2;
+        }
       }
     }
   }
@@ -1347,15 +1383,22 @@ export class LocalizationManager extends EventEmitter {
     
     let bestMatch = null;
     let maxMatches = 0;
-    
+    const wordCount = text.split(/\s+/).filter(Boolean).length;
+
     for (const [lang, pattern] of Object.entries(patterns)) {
       const matches = (text.match(pattern) || []).length;
       if (matches > maxMatches) {
         maxMatches = matches;
-        bestMatch = { language: lang, confidence: Math.min(95, matches * 5) };
+        // Confidence from stop-word DENSITY (matches per word), not the
+        // absolute match count — absolute counts made confidence depend on
+        // sample length (a 500-char English sample scored only ~35%).
+        // Stop-words are ~25-40% of natural text in the matching language,
+        // so density * 250 puts an unambiguous match in the 60-95 range.
+        const density = matches / Math.max(1, wordCount);
+        bestMatch = { language: lang, confidence: Math.min(95, Math.round(density * 250)) };
       }
     }
-    
+
     return bestMatch;
   }
 
@@ -1363,15 +1406,17 @@ export class LocalizationManager extends EventEmitter {
    * Enhanced country detection
    */
   async performCountryDetection(content, url, detection) {
-    // TLD analysis
-    const urlObj = new URL(url);
-    const tldMatch = urlObj.hostname.match(/\.([a-z]{2})$/);
-    if (tldMatch) {
-      const tld = tldMatch[1].toUpperCase();
-      if (SUPPORTED_COUNTRIES[tld]) {
-        detection.detectedCountry = tld;
-        detection.evidence.push(`TLD suggests country: ${tld}`);
-        detection.confidence += 0.2;
+    // TLD analysis — url is optional metadata; content-only detection skips it
+    if (url) {
+      const urlObj = new URL(url);
+      const tldMatch = urlObj.hostname.match(/\.([a-z]{2})$/);
+      if (tldMatch) {
+        const tld = tldMatch[1].toUpperCase();
+        if (SUPPORTED_COUNTRIES[tld]) {
+          detection.detectedCountry = tld;
+          detection.evidence.push(`TLD suggests country: ${tld}`);
+          detection.confidence += 0.2;
+        }
       }
     }
     

package/src/core/MonitorStore.js

@@ -1,18 +1,28 @@
 /**
  * MonitorStore — disk persistence for scheduled change-monitors.
  *
- * One JSON file per monitor under ./monitors/<id>.json. Mirrors JobManager's
- * persistence *pattern* (mkdir-recursive, per-file JSON, randomUUID, load-on-
- * start) but deliberately omits TTL/eviction — scheduled monitors are long-lived
- * and must never be auto-expired.
+ * One JSON file per monitor under <os.homedir()>/.crawlforge/monitors/<id>.json
+ * (same base dir as ~/.crawlforge/config.json and ~/.crawlforge/snapshots —
+ * user state must never depend on process.cwd(), or monitor:create from dir X
+ * and monitor:stop / monitor:run-due from dir Y silently see different stores).
+ * Mirrors JobManager's persistence *pattern* (mkdir-recursive, per-file JSON,
+ * randomUUID, load-on-start) but deliberately omits TTL/eviction — scheduled
+ * monitors are long-lived and must never be auto-expired.
  */
 import { promises as fs } from 'node:fs';
+import os from 'node:os';
 import path from 'node:path';
 import { randomUUID } from 'node:crypto';
 
 export class MonitorStore {
-  constructor({ storageDir = './monitors' } = {}) {
-    this.storageDir = storageDir;
+  constructor({ storageDir, legacyDir } = {}) {
+    this.storageDir = storageDir || path.join(os.homedir(), '.crawlforge', 'monitors');
+    // One-time best-effort migration source for stores created before v5.0.3,
+    // when the default was cwd-relative './monitors'. Only armed when the
+    // caller did NOT override storageDir (i.e. the default upgrade path), so
+    // tests and embedders with explicit dirs never sweep files out of cwd —
+    // unless they opt in by passing legacyDir explicitly.
+    this._legacyDir = legacyDir ?? (storageDir ? null : path.resolve('./monitors'));
     this.monitors = new Map();
     this._loaded = false;
   }
@@ -33,10 +43,49 @@ export class MonitorStore {
     } catch {
       /* dir unavailable — start empty */
     }
+    await this._migrateLegacyDir();
     this._loaded = true;
     return this.monitors;
   }
 
+  /**
+   * Best-effort, idempotent migration of a legacy cwd-relative './monitors'
+   * store into the new home-rooted store. Moves each parseable monitor file
+   * whose id is not already present; skips (and leaves in place) everything
+   * else. Never throws — a failed migration must not break monitor loading.
+   */
+  async _migrateLegacyDir() {
+    if (!this._legacyDir) return;
+    try {
+      if (path.resolve(this._legacyDir) === path.resolve(this.storageDir)) return;
+      const files = await fs.readdir(this._legacyDir); // throws if absent — caught below
+      for (const f of files) {
+        if (!f.endsWith('.json')) continue;
+        const from = path.join(this._legacyDir, f);
+        try {
+          const def = JSON.parse(await fs.readFile(from, 'utf8'));
+          if (!def || !def.id || this.monitors.has(def.id)) continue; // skip existing ids
+          await fs.mkdir(this.storageDir, { recursive: true });
+          const to = path.join(this.storageDir, `${def.id}.json`);
+          try {
+            await fs.rename(from, to);
+          } catch {
+            // rename fails across filesystems (EXDEV) — copy+unlink instead
+            await fs.copyFile(from, to);
+            await fs.unlink(from).catch(() => {});
+          }
+          this.monitors.set(def.id, def);
+        } catch {
+          /* unreadable/corrupt legacy file — leave it, keep going */
+        }
+      }
+      // Tidy up the legacy dir only if the migration emptied it.
+      await fs.rmdir(this._legacyDir).catch(() => {});
+    } catch {
+      /* no legacy dir (the common case) or unreadable — nothing to migrate */
+    }
+  }
+
   newId() {
     return randomUUID();
   }

package/src/core/ResearchOrchestrator.js

@@ -903,8 +903,12 @@ export class ResearchOrchestrator extends EventEmitter {
   /** One stealth navigation. Fresh page/context; judges blocked by rendered content. */
   async _stealthFetchOnce(url) {
     let page;
+    let camoufoxContext;
     if (this._stealthEngineActive === 'camoufox') {
-      page = await this._stealthBrowser.newPage();
+      // viewport:null skips Browser.setDefaultViewport, whose playwright-core
+      // 1.62 payload (isMobile etc.) camoufox's older Firefox build rejects.
+      camoufoxContext = await this._stealthBrowser.newContext({ viewport: null });
+      page = await camoufoxContext.newPage();
     } else {
       const { contextId } = await this._stealthManager.createStealthContext({ level: this.stealthLevel });
       page = await this._stealthManager.createStealthPage(contextId);
@@ -930,6 +934,7 @@ export class ResearchOrchestrator extends EventEmitter {
       return html && html.length > 200 ? html : null;
     } finally {
       await page.close().catch(() => {});
+      if (camoufoxContext) await camoufoxContext.close().catch(() => {});
     }
   }
 

package/src/core/StealthBrowserManager.js

@@ -421,6 +421,20 @@ export class StealthBrowserManager {
       javaScriptEnabled: true
     };
 
+    // camoufox's Firefox build predates the Browser.setDefaultViewport fields
+    // playwright-core 1.62 sends (screenSize, isMobile, ...) and rejects unknown
+    // properties, so any fixed viewport fails. viewport:null skips that protocol
+    // call entirely (deviceScaleFactor/isMobile/hasTouch/screen are invalid or
+    // meaningless without a viewport). window.screen is still spoofed via
+    // addInitScript in applyAdvancedStealthConfigurations.
+    if (this._launchedEngine === 'camoufox') {
+      contextOptions.viewport = null;
+      delete contextOptions.deviceScaleFactor;
+      delete contextOptions.isMobile;
+      delete contextOptions.hasTouch;
+      delete contextOptions.screen;
+    }
+
     const context = await this.browser.newContext(contextOptions);
     const contextId = this.generateContextId();
     
@@ -2001,32 +2015,79 @@ export class BrowserEngine {
 export class CamoufoxAdapter extends BrowserEngine {
   name() { return 'camoufox'; }
 
+  /**
+   * Load camoufox through its CJS entry (dist/index.cjs) via createRequire.
+   * The package's ESM entry (dist/index.js, an esbuild bundle) throws
+   * 'Dynamic require of "events" is not supported' when imported from ESM,
+   * so `await import('camoufox')` fails even when the package IS installed.
+   */
+  async _load() {
+    const { createRequire } = await import('module');
+    const require = createRequire(import.meta.url);
+    return require('camoufox'); // CJS build — ESM build is broken
+  }
+
+  /** True only when the camoufox package itself is absent (vs. present but failing to load). */
+  _isNotInstalled(err) {
+    return err?.code === 'MODULE_NOT_FOUND' && (err.message || '').includes("Cannot find module 'camoufox'");
+  }
+
   async isAvailable() {
     try {
-      await import('camoufox');
+      await this._load();
       return true;
-    } catch {
-      return false;
+    } catch (err) {
+      if (this._isNotInstalled(err)) {
+        return false;
+      }
+      // Installed but broken — surface the real error instead of misreporting "not installed".
+      throw new Error(`camoufox is installed but failed to load: ${err.message}`);
     }
   }
 
   async launch(config = {}) {
     let camoufox;
     try {
-      camoufox = await import('camoufox');
-    } catch {
-      throw new Error(
-        'camoufox is not installed. Run: npm install camoufox. Note: camoufox is MIT-licensed and requires Firefox to be installed.'
-      );
+      camoufox = await this._load();
+    } catch (err) {
+      if (this._isNotInstalled(err)) {
+        throw new Error(
+          'camoufox is not installed. Run: npm install camoufox. Note: camoufox is MIT-licensed and requires Firefox to be installed.'
+        );
+      }
+      throw new Error(`camoufox is installed but failed to load: ${err.message}`);
     }
 
-    // camoufox API mirrors playwright — returns a Browser object
-    const browser = await (camoufox.launch || camoufox.default?.launch)({
+    await this._ensureMacOSLayout(camoufox);
+
+    // camoufox's launcher is Camoufox(options) — the package has no launch()
+    // export. It resolves the fetched Firefox binary (npx camoufox fetch) and
+    // returns a Playwright-compatible Browser. Takes `headless` directly plus
+    // passthrough Playwright Firefox launch options.
+    return camoufox.Camoufox({
       headless: config.headless !== false,
       ...config.launchOptions
     });
+  }
 
-    return browser;
+  /**
+   * macOS packaging fix for camoufox-js: it expects properties.json in
+   * Camoufox.app/Contents/MacOS/, but the .app bundle ships it under
+   * Contents/Resources/. Bridge it so the launcher can boot. Best-effort.
+   * (Same fix as ResearchOrchestrator._ensureCamoufoxLayout.)
+   */
+  async _ensureMacOSLayout(camoufox) {
+    if (process.platform !== 'darwin' || !camoufox?.INSTALL_DIR) return;
+    try {
+      const fs = await import('fs');
+      const path = await import('path');
+      const appDir = path.join(camoufox.INSTALL_DIR, 'Camoufox.app', 'Contents');
+      const target = path.join(appDir, 'MacOS', 'properties.json');
+      const source = path.join(appDir, 'Resources', 'properties.json');
+      if (!fs.existsSync(target) && fs.existsSync(source)) {
+        fs.copyFileSync(source, target);
+      }
+    } catch { /* best-effort; launch surfaces a real error if it matters */ }
   }
 }
 

package/src/core/analysis/ContentAnalyzer.js

@@ -448,8 +448,9 @@ export class ContentAnalyzer {
 
   /**
    * Create extractive summary by scoring pre-split sentences via word
-   * frequency (Luhn-style salience), tokenized with compromise, plus a small
-   * positional bonus for leading/closing sentences. Selects the top N and
+   * frequency (Luhn-style salience), tokenized with compromise, plus a
+   * lead-biased positional bonus (first sentence strongly favored) and a
+   * brevity penalty for very short sentences. Selects the top N and
    * restores original document order.
    * @param {string[]} sentences - Sentences in original document order
    * @param {number} targetSentences - Number of sentences to select
@@ -474,11 +475,22 @@ export class ContentAnalyzer {
 
     const scored = sentences.map((sentence, index) => {
       const words = sentenceWords[index];
+      // Very short sentences carry little information yet the frequency
+      // average below inflates them (few words, each high-frequency), so
+      // dampen their word score (length penalty, Nobata & Sekine 2004).
+      const brevityPenalty = words.length < 5 ? 0.5 : 1;
       const wordScore = words.length > 0
-        ? words.reduce((sum, w) => sum + freq[w] / maxFreq, 0) / words.length
+        ? (words.reduce((sum, w) => sum + freq[w] / maxFreq, 0) / words.length) * brevityPenalty
         : 0;
-      // Leading/closing sentences tend to carry more salience in prose.
-      const positionScore = (index === 0 || index === sentences.length - 1) ? 0.15 : 0;
+      // Lead bias (Edmundson position method): document-leading sentences
+      // carry the definitional payload — Lead-3 remains a near-SOTA
+      // extractive baseline — so favor the first sentence strongly, decay
+      // over the next two, and keep only a token bonus for the closer.
+      let positionScore = 0;
+      if (index === 0) positionScore = 0.4;
+      else if (index === 1) positionScore = 0.15;
+      else if (index === 2) positionScore = 0.08;
+      else if (index === sentences.length - 1) positionScore = 0.05;
       return { sentence, index, score: wordScore + positionScore };
     });
 
@@ -508,17 +520,23 @@ export class ContentAnalyzer {
       const phraseCount = {};
       
       allPhrases.forEach(phrase => {
-        const cleaned = phrase.toLowerCase().trim();
-        if (cleaned.length > 2) {
+        // Strip edge punctuation and skip phrases made only of stop words
+        const cleaned = phrase.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '');
+        const words = cleaned.split(/\s+/);
+        if (cleaned.length > 2 && words.some(w => !this.isStopWord(w))) {
           phraseCount[cleaned] = (phraseCount[cleaned] || 0) + 1;
         }
       });
 
-      // Score and rank topics
+      // Score and rank topics. Confidence is the phrase frequency relative to
+      // the MOST frequent phrase (RAKE-style relative salience) — the old
+      // frequency/totalPhrases normalization collapsed toward 0 on long
+      // documents, so every topic failed minConfidence and topics came back [].
+      const maxFreq = Math.max(1, ...Object.values(phraseCount));
       const topics = Object.entries(phraseCount)
         .map(([topic, frequency]) => ({
           topic,
-          confidence: Math.min(1, frequency / Math.max(allPhrases.length, 1)),
+          confidence: Math.round((frequency / maxFreq) * 100) / 100,
           keywords: topic.split(' ').filter(w => w.length > 2)
         }))
         .filter(topic => topic.confidence >= options.minConfidence)
@@ -543,14 +561,53 @@ export class ContentAnalyzer {
     try {
       const doc = nlp(text);
 
-      const people = doc.people().out('array');
-      const places = doc.places().out('array');
-      const organizations = doc.organizations().out('array');
+      // compromise's out('array') keeps adjoining punctuation ("Craigslist.",
+      // "United States,") — clean edges, drop bare stopword/pronoun tokens,
+      // and dedupe case-insensitively keeping the first casing seen.
+      const clean = (arr) => this.dedupeEntities(
+        arr.map(e => this.cleanEntityText(e))
+          .filter(e => e.length > 1
+            && !this.isBareStopword(e)
+            && !/^(?:inc|corp|ltd|co|llc)\.?$/i.test(e)) // bare corporate-suffix fragment
+      );
+
+      let people = clean(doc.people().out('array'));
+      let places = clean(doc.places().out('array'));
+      let organizations = clean(doc.organizations().out('array'));
       // .dates() needs the compromise-dates plugin (not installed) and threw,
       // aborting ALL entity extraction; #Date+ tag matching is core compromise.
-      const dates = doc.match('#Date+').out('array');
-      const money = doc.money().out('array');
-      let other = doc.topics().out('array').slice(0, 10);
+      const dates = clean(doc.match('#Date+').out('array'));
+      const money = clean(doc.money().out('array'));
+      let other = clean(doc.topics().out('array').slice(0, 10))
+        .filter(e => !this.isSentenceInitialArtifact(e, text));
+
+      // "X v. Y" fragments are legal-case citations, not people/places/orgs —
+      // reclassify them under "other".
+      const legalCases = [];
+      const extractCases = (list) => list.filter(e => {
+        if (this.isLegalCaseFragment(e)) {
+          legalCases.push(e);
+          return false;
+        }
+        return true;
+      });
+      people = extractCases(people);
+      places = extractCases(places);
+      organizations = extractCases(organizations);
+
+      // A bare ALL-CAPS token (UNIX, DOM) is not enough evidence for an
+      // organization — demote to "other" unless the text corroborates it
+      // (corporate suffix or organization-headed alias definition).
+      const demotedAcronyms = [];
+      organizations = organizations.filter(e => {
+        if (/^[A-Z]{2,}$/.test(e) && !this.hasOrganizationEvidence(e, text)) {
+          demotedAcronyms.push(e);
+          return false;
+        }
+        return true;
+      });
+
+      other = [...other, ...legalCases, ...demotedAcronyms];
 
       // Supplement with capitalized proper nouns that compromise may miss
       // (technology names, product names, etc.)
@@ -558,15 +615,27 @@ export class ContentAnalyzer {
         ...people, ...places, ...organizations, ...other
       ].map(e => e.toLowerCase()));
 
-      const properNouns = text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [];
-      const supplemental = [...new Set(properNouns)]
-        .filter(n => !existingEntities.has(n.toLowerCase()) && n.length > 1)
+      const properNouns = (text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [])
+        // Split matches that crossed a sentence boundary ("XPath. In") — a
+        // period after a lowercase letter followed by a capital is a sentence
+        // end, while abbreviations ("U.S. Holdings") stay intact.
+        .flatMap(n => n.split(/(?<=[a-z]\.)\s+(?=[A-Z])/));
+      const supplemental = this.dedupeEntities(properNouns.map(n => this.cleanEntityText(n)))
+        .filter(n => n.length > 1
+          && !existingEntities.has(n.toLowerCase())
+          && !this.isSentenceInitialArtifact(n, text))
         .slice(0, 10);
 
       if (supplemental.length > 0) {
         other = [...other, ...supplemental].slice(0, 15);
       }
 
+      // Keep "other" free of entities already classified more specifically.
+      const classified = new Set(
+        [...people, ...places, ...organizations, ...dates, ...money].map(e => e.toLowerCase())
+      );
+      other = this.dedupeEntities(other).filter(e => !classified.has(e.toLowerCase()));
+
       const allEntities = [...people, ...places, ...organizations, ...dates, ...money, ...other];
       const uniqueEntities = new Set(allEntities.map(e => e.toLowerCase()));
 
@@ -828,6 +897,120 @@ export class ContentAnalyzer {
     return 'Very Difficult';
   }
 
+  /**
+   * Strip leading/trailing punctuation from an entity string while preserving
+   * trailing periods on abbreviations ("Inc.", "U.S.") and internal
+   * punctuation ("Home.dk", "Bidder's Edge").
+   * @param {string} raw - Raw entity string
+   * @returns {string} - Cleaned entity string
+   */
+  cleanEntityText(raw) {
+    let entity = String(raw).trim();
+    let previous;
+    do {
+      previous = entity;
+      entity = entity.replace(/^[\s"'‘’“”`([{<,;:!?«»–—-]+/, '');
+      entity = entity.replace(/[\s"'‘’“”`)\]}>,;:!?«»–—-]+$/, '');
+      if (entity.endsWith('.')) {
+        const lastToken = entity.split(/\s+/).pop();
+        const isAbbreviation = /^(?:[A-Za-z]\.)+$/.test(lastToken)
+          || /^(?:inc|corp|ltd|co|llc|jr|sr|st|mr|mrs|ms|dr|no|vs?)\.$/i.test(lastToken);
+        if (!isAbbreviation) {
+          entity = entity.replace(/\.+$/, '');
+        }
+      }
+    } while (entity !== previous);
+    return entity;
+  }
+
+  /**
+   * Deduplicate entities case-insensitively, keeping the first casing seen
+   * @param {string[]} entities - Entity strings
+   * @returns {string[]} - Deduplicated entity strings
+   */
+  dedupeEntities(entities) {
+    const seen = new Set();
+    const result = [];
+    for (const entity of entities) {
+      const key = entity.toLowerCase();
+      if (!seen.has(key)) {
+        seen.add(key);
+        result.push(entity);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Check whether an entity is a single bare stop word/pronoun token
+   * @param {string} entity - Cleaned entity string
+   * @returns {boolean} - True if a bare stop word
+   */
+  isBareStopword(entity) {
+    return !entity.includes(' ') && this.isStopWord(entity.toLowerCase());
+  }
+
+  /**
+   * True when a single capitalized token is likely just a sentence-initial
+   * common word ("While", "It", "Once"): it is a stop word, the same word
+   * also appears in lowercase elsewhere in the text, or it is only ever
+   * capitalized at the start of a sentence (real proper nouns keep their
+   * capital mid-sentence).
+   * @param {string} entity - Cleaned entity string
+   * @param {string} text - Full source text
+   * @returns {boolean} - True if likely a sentence-initial artifact
+   */
+  isSentenceInitialArtifact(entity, text) {
+    if (entity.includes(' ')) return false;
+    const lower = entity.toLowerCase();
+    if (this.isStopWord(lower)) return true;
+    if (!/^[A-Z][a-z]+$/.test(entity)) return false;
+
+    // Word also appears in lowercase → ordinary word, capitalized only by position
+    if (new RegExp(`(?:^|[^A-Za-z])${lower}(?:[^A-Za-z]|$)`).test(text)) return true;
+
+    // Keep only if at least one occurrence is NOT at a sentence start
+    const occurrence = new RegExp(`\\b${entity}\\b`, 'g');
+    let match;
+    while ((match = occurrence.exec(text)) !== null) {
+      const before = text.slice(0, match.index).replace(/[\s"'‘’“”()[\]]+$/, '');
+      if (before.length > 0 && !/[.!?:]$/.test(before)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Check whether an entity is an "X v. Y" / "X vs. Y" legal-case fragment
+   * @param {string} entity - Cleaned entity string
+   * @returns {boolean} - True if a legal-case citation fragment
+   */
+  isLegalCaseFragment(entity) {
+    return /\s+vs?\.?\s+/i.test(entity);
+  }
+
+  /**
+   * Corroborating evidence that an ALL-CAPS token really is an organization:
+   * a corporate suffix follows it ("IBM Corp"), or it is introduced as the
+   * alias of an organization-headed name ("American Airlines (AA)",
+   * "French Data Protection Authority (CNIL)").
+   * @param {string} acronym - ALL-CAPS candidate (already /^[A-Z]{2,}$/)
+   * @param {string} text - Full source text
+   * @returns {boolean} - True if the text corroborates the org classification
+   */
+  hasOrganizationEvidence(acronym, text) {
+    const corporateSuffix = new RegExp(
+      `\\b${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:[^a-zA-Z]|$)`
+    );
+    if (corporateSuffix.test(text)) return true;
+
+    const orgHeadedAlias = new RegExp(
+      `\\b(?:Airlines?|Authority|Agency|Association|Bureau|Commission|Committee|Corporation|Company|Council|Court|Foundation|Group|Institute|Institution|Organi[sz]ation|Press|Society|Union|University)\\s*\\(["'“‘]?${acronym}["'”’]?\\)`
+    );
+    return orgHeadedAlias.test(text);
+  }
+
   /**
    * Check if word is a stop word
    * @param {string} word - Word to check

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -73,7 +73,11 @@ const ScrollActionSchema = BaseActionSchema.extend({
   direction: z.enum(['up', 'down', 'left', 'right']).default('down'),
   distance: z.number().min(0).default(100),
   smooth: z.boolean().default(true),
-  toElement: z.string().optional()
+  toElement: z.string().optional(),
+  // Absolute scroll-to coordinates (window.scrollTo). When present they take
+  // precedence over direction/distance — see ActionExecutor's ScrollActionSchema.
+  x: z.number().min(0).optional(),
+  y: z.number().min(0).optional()
 });
 
 const ScreenshotActionSchema = BaseActionSchema.extend({

package/src/tools/advanced/batchScrape/worker.js

@@ -8,6 +8,7 @@ import { load } from 'cheerio';
 import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { throttleHost } from '../../../utils/hostRateLimiter.js';
+import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 
 const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
 
@@ -165,7 +166,6 @@ function generateFormats($, html, formats) {
 }
 
 function buildMarkdown($) {
-  let md = '';
   const title = $('title').text().trim();
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
@@ -176,18 +176,17 @@ function buildMarkdown($) {
   }
   if (!$body || $body.length === 0) $body = $('body');
 
+  // Full-fidelity conversion via the shared Turndown helper (same converter
+  // the unified `scrape` tool uses). The previous hand-rolled h1–h3/p/li walk
+  // silently dropped any text living outside those tags (e.g. quotes in
+  // <span>/<small> on quotes.toscrape.com).
+  let md = htmlToMarkdown($.html($body));
+
   // C3: de-dup title — only emit the <title> heading if the page has no <h1>
   // or if the first <h1> text differs from the <title> text (case-insensitive).
   const firstH1 = $body.find('h1').first().text().trim();
   const titleDuplicated = firstH1 && firstH1.toLowerCase() === title.toLowerCase();
-  if (title && !titleDuplicated) md += `# ${title}\n\n`;
-
-  $body.find('h1').each((_, el) => { md += `# ${$(el).text().trim()}\n\n`; });
-  $body.find('h2').each((_, el) => { md += `## ${$(el).text().trim()}\n\n`; });
-  $body.find('h3').each((_, el) => { md += `### ${$(el).text().trim()}\n\n`; });
-  $body.find('p').each((_, el) => { const t = $(el).text().trim(); if (t) md += `${t}\n\n`; });
-  $body.find('ul li').each((_, el) => { md += `- ${$(el).text().trim()}\n`; });
-  $body.find('ol li').each((_, el) => { md += `1. ${$(el).text().trim()}\n`; });
+  if (title && !titleDuplicated) md = `# ${title}\n\n${md}`;
 
   return md.trim();
 }

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -170,6 +170,9 @@ export function buildRecordedEntry(action, timestampMsSinceStart) {
   if (action.value !== undefined) entry.value = action.value;
   if (action.direction !== undefined) entry.direction = action.direction;
   if (action.distance !== undefined) entry.distance = action.distance;
+  // scroll: absolute scroll-to coordinates
+  if (action.x !== undefined) entry.x = action.x;
+  if (action.y !== undefined) entry.y = action.y;
   if (action.description !== undefined) entry.description = action.description;
   // executeJavaScript actions require `script` to replay (ActionExecutor's
   // ActionChainSchema rejects the entry otherwise) — preserve it.

package/src/tools/basic/extractText.js

@@ -89,6 +89,13 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     if (remove_scripts !== false) $('script').remove();
     if (remove_styles !== false) $('style').remove();
 
+    // <noscript> contents are parsed as raw TEXT when scripting is enabled
+    // (cheerio/parse5 default, per the HTML spec), so leaving them in leaks
+    // literal markup into the extracted text — e.g. Wikipedia's
+    // Special:CentralAutoLogin 1x1 <img> tracking pixel. Browsers with JS
+    // enabled never render noscript content, so always strip it.
+    $('noscript').remove();
+
     $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
 
     const result = {

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -1,7 +1,9 @@
 import { z } from 'zod';
+import { load } from 'cheerio';
 import { LLMsTxtAnalyzer } from '../../core/LLMsTxtAnalyzer.js';
 import { Logger } from '../../utils/Logger.js';
 import { getBaseUrl } from '../../utils/urlNormalizer.js';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const logger = new Logger('GenerateLLMsTxtTool');
 
@@ -79,6 +81,14 @@ export class GenerateLLMsTxtTool {
       });
       const analysis = await analyzer.analyzeWebsite(url, analysisOptions);
 
+      // Capture the homepage's <title>/h1 and meta/og description so the
+      // spec-compliant output can carry a real site summary and a named Home
+      // link (llmstxt.org) instead of boilerplate. Best-effort: null on
+      // failure, in which case the generic fallbacks below apply.
+      if (!outputOptions.robotsStyle) {
+        analysis.homePage = await this.fetchHomePageMetadata(baseUrl);
+      }
+
       // Step 2: Generate LLMs.txt Content
       const llmsTxtContent = this.generateLLMsTxt(analysis, outputOptions, complianceLevel);
 
@@ -156,8 +166,10 @@ export class GenerateLLMsTxtTool {
     lines.push(`# ${title}`);
     lines.push('');
 
-    // Blockquote summary (required by spec)
-    const summary = `Site map and key resources for ${baseUrl}, generated to help LLMs locate relevant content.`;
+    // Blockquote summary (required by spec). Prefer the site's own meta /
+    // og:description captured from the homepage over generic boilerplate.
+    const summary = analysis.homePage?.description
+      || `Site map and key resources for ${baseUrl}, generated to help LLMs locate relevant content.`;
     lines.push(`> ${summary}`);
     lines.push('');
 
@@ -177,13 +189,16 @@ export class GenerateLLMsTxtTool {
       lines.push('');
     }
 
-    // Helper: emit a "## Section" with a list of [name](url) links.
+    // Helper: emit a "## Section" with a list of [name](url) links. Link
+    // names prefer the page's actual <title> captured during analysis;
+    // otherwise a humanized full path ("/tag/abilities/page/1" ->
+    // "Tag: abilities — page 1") — never a bare trailing segment like "1".
+    const pageTitles = this.collectPageTitles(analysis);
     const linkLabel = (u) => {
+      const captured = pageTitles.get(this.normalizeTitleKey(u));
+      if (captured) return captured;
       try {
-        const p = new URL(u).pathname.replace(/\/+$/, '');
-        if (!p || p === '') return 'Home';
-        const seg = p.split('/').filter(Boolean).pop() || p;
-        return seg.replace(/[-_]/g, ' ').replace(/\.[a-z0-9]+$/i, '').trim() || p;
+        return this.humanizePath(new URL(u).pathname);
       } catch {
         return u;
       }
@@ -247,6 +262,131 @@ export class GenerateLLMsTxtTool {
     return lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
   }
 
+  /**
+   * Build a URL -> page title map from titles captured during analysis
+   * (classifyPage stores each public page's <title>; the homepage fetch names
+   * the site root). A title shared verbatim by multiple URLs is dropped — it
+   * cannot identify a specific page, so those links fall back to the
+   * humanized path instead.
+   */
+  collectPageTitles(analysis) {
+    const titles = new Map();
+    const counts = new Map();
+    const add = (url, title) => {
+      if (!url || typeof title !== 'string') return;
+      const clean = title.replace(/\s+/g, ' ').trim();
+      if (!clean) return;
+      const key = this.normalizeTitleKey(url);
+      const prev = titles.get(key);
+      if (prev === clean) return;
+      if (prev) counts.set(prev, counts.get(prev) - 1);
+      titles.set(key, clean);
+      counts.set(clean, (counts.get(clean) || 0) + 1);
+    };
+
+    for (const entries of Object.values(analysis.contentTypes || {})) {
+      if (!Array.isArray(entries)) continue;
+      for (const entry of entries) {
+        add(entry?.url, entry?.metadata?.title);
+      }
+    }
+    if (analysis.homePage?.title && analysis.metadata?.baseUrl) {
+      add(analysis.metadata.baseUrl, analysis.homePage.title);
+    }
+
+    for (const [key, title] of titles) {
+      if (counts.get(title) > 1) titles.delete(key);
+    }
+    return titles;
+  }
+
+  normalizeTitleKey(url) {
+    try {
+      return new URL(url).toString();
+    } catch {
+      return url;
+    }
+  }
+
+  /**
+   * Humanize a full URL path into a readable label:
+   *   "/"                      -> "Home"
+   *   "/login"                 -> "Login"
+   *   "/author/Albert-Einstein" -> "Author: Albert Einstein"
+   *   "/tag/abilities/page/1"  -> "Tag: abilities — page 1"
+   * A trailing pagination segment ("page/2", "page-2", bare "2") is folded
+   * into "page N" so a link is never labeled by a bare number.
+   */
+  humanizePath(pathname) {
+    let raw = pathname;
+    try { raw = decodeURIComponent(pathname); } catch { /* keep encoded */ }
+    const segments = raw.split('/').filter(Boolean);
+    if (segments.length === 0) return 'Home';
+
+    // Strip a file extension from the last segment, then de-slug.
+    segments[segments.length - 1] = segments[segments.length - 1].replace(/\.[a-z0-9]+$/i, '');
+    const clean = segments.map((s) => s.replace(/[-_]+/g, ' ').trim()).filter(Boolean);
+    if (clean.length === 0) return 'Home';
+
+    let pageNum = null;
+    const last = clean[clean.length - 1];
+    const pageMatch = last.match(/^page\s*(\d+)$/i);
+    if (pageMatch) {
+      pageNum = pageMatch[1];
+      clean.pop();
+    } else if (/^\d+$/.test(last)) {
+      pageNum = last;
+      clean.pop();
+      if (clean.length > 0 && /^pages?$/i.test(clean[clean.length - 1])) {
+        clean.pop();
+      }
+    }
+    if (clean.length === 0) return pageNum ? `Page ${pageNum}` : 'Home';
+
+    const cap = (s) => s.charAt(0).toUpperCase() + s.slice(1);
+    let label = clean.length === 1
+      ? cap(clean[0])
+      : `${cap(clean[0])}: ${clean.slice(1).join(' / ')}`;
+    if (pageNum) label += ` — page ${pageNum}`;
+    return label;
+  }
+
+  /**
+   * Fetch the site's homepage once and extract naming metadata for the spec
+   * output. Best-effort: returns null on any failure.
+   */
+  async fetchHomePageMetadata(baseUrl) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), Math.min(this.options.timeout, 10000));
+    try {
+      const response = await safeFetch(baseUrl, {
+        signal: controller.signal,
+        headers: { 'User-Agent': this.options.userAgent }
+      });
+      if (!response.ok) return null;
+      return this.extractHomePageMetadata(await response.text());
+    } catch {
+      return null;
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+
+  /**
+   * Extract { title, description } from homepage HTML. Title prefers <title>
+   * then the first <h1>; description prefers meta[name="description"] then
+   * og:description. Returns null when neither is present.
+   */
+  extractHomePageMetadata(html) {
+    const $ = load(html);
+    const clean = (s) => (s || '').replace(/\s+/g, ' ').trim();
+    const title = clean($('title').first().text()) || clean($('h1').first().text());
+    const description = clean($('meta[name="description"]').attr('content'))
+      || clean($('meta[property="og:description"]').attr('content'));
+    if (!title && !description) return null;
+    return { title, description };
+  }
+
   /**
    * Generate legacy robots.txt-style content (opt-in via outputOptions.robotsStyle).
    */

package/src/tools/templates/TemplateRegistry.js

@@ -183,8 +183,11 @@ const TEMPLATES = [
           site: $row.find('.sitebit a').text().trim() || null,
           score: $score.text().replace(' points', '').trim() || null,
           author: $subtext.find('.hnuser').text().trim() || null,
-          posted: $subtext.find('.age a').attr('href') || null,
-          comments: $subtext.find('a[href*="item"]').last().text().trim() || null
+          // ".age a" wraps the relative age string ("3 hours ago"); its href is the item permalink.
+          posted: $subtext.find('.age a').text().trim() || null,
+          // The comments link is also an item?id= link, so exclude the age anchor.
+          // Job posts have no comments link at all -> null.
+          comments: $subtext.find('a[href*="item"]').not('.age a').last().text().trim() || null
         });
       });
       return { stories: stories.slice(0, 30), scraped_at: new Date().toISOString() };

package/src/tools/tracking/trackChanges/index.js

@@ -68,7 +68,10 @@ export class TrackChangesTool extends EventEmitter {
     // Scheduled-monitor subsystem (timers are NOT started here — only the
     // single server-owned instance calls startScheduler()).
     this._mcpServer = null;
-    this.monitorStore = new MonitorStore({ storageDir: this.options.monitorStorageDir || './monitors' });
+    // No storageDir fallback here: MonitorStore itself defaults to
+    // ~/.crawlforge/monitors (cwd-independent, like snapshotStorageDir above)
+    // and runs its legacy ./monitors migration only on that default path.
+    this.monitorStore = new MonitorStore({ storageDir: this.options.monitorStorageDir });
     this.scheduler = new MonitorScheduler({ tool: this, store: this.monitorStore });
 
     // Wired synchronously (no I/O) so no 'error' event emitted by
@@ -200,14 +203,18 @@ export class TrackChangesTool extends EventEmitter {
       snapshotInfo = await this.snapshotManager.storeSnapshot(url, sourceContent, { ...fetchMeta, baseline: true, trackingOptions }, { enableCompression: storageOptions.compressionEnabled });
     }
 
+    // ChangeTracker.createBaseline returns a summary ({contentHash, sections,
+    // elements, createdAt, ...}), NOT the internal baseline object — reading
+    // baseline.analysis?./baseline.timestamp here always yielded
+    // undefined/0/0/undefined regardless of the page.
     return {
       success: true, operation: 'create_baseline', url,
       baseline: {
         version: baseline.version,
-        contentHash: baseline.analysis?.hashes?.page,
-        sections: Object.keys(baseline.analysis?.hashes?.sections || {}).length,
-        elements: Object.keys(baseline.analysis?.hashes?.elements || {}).length,
-        createdAt: baseline.timestamp,
+        contentHash: baseline.contentHash,
+        sections: baseline.sections,
+        elements: baseline.elements,
+        createdAt: baseline.createdAt,
         options: trackingOptions
       },
       snapshot: snapshotInfo, timestamp: Date.now()