crawlforge-mcp-server 5.0.2 → 5.0.4

package/src/core/analysis/ContentAnalyzer.js

@@ -448,8 +448,9 @@ export class ContentAnalyzer {
 
   /**
    * Create extractive summary by scoring pre-split sentences via word
-   * frequency (Luhn-style salience), tokenized with compromise, plus a small
-   * positional bonus for leading/closing sentences. Selects the top N and
+   * frequency (Luhn-style salience), tokenized with compromise, plus a
+   * lead-biased positional bonus (first sentence strongly favored) and a
+   * brevity penalty for very short sentences. Selects the top N and
    * restores original document order.
    * @param {string[]} sentences - Sentences in original document order
    * @param {number} targetSentences - Number of sentences to select
@@ -474,11 +475,22 @@ export class ContentAnalyzer {
 
     const scored = sentences.map((sentence, index) => {
       const words = sentenceWords[index];
+      // Very short sentences carry little information yet the frequency
+      // average below inflates them (few words, each high-frequency), so
+      // dampen their word score (length penalty, Nobata & Sekine 2004).
+      const brevityPenalty = words.length < 5 ? 0.5 : 1;
       const wordScore = words.length > 0
-        ? words.reduce((sum, w) => sum + freq[w] / maxFreq, 0) / words.length
+        ? (words.reduce((sum, w) => sum + freq[w] / maxFreq, 0) / words.length) * brevityPenalty
         : 0;
-      // Leading/closing sentences tend to carry more salience in prose.
-      const positionScore = (index === 0 || index === sentences.length - 1) ? 0.15 : 0;
+      // Lead bias (Edmundson position method): document-leading sentences
+      // carry the definitional payload — Lead-3 remains a near-SOTA
+      // extractive baseline — so favor the first sentence strongly, decay
+      // over the next two, and keep only a token bonus for the closer.
+      let positionScore = 0;
+      if (index === 0) positionScore = 0.4;
+      else if (index === 1) positionScore = 0.15;
+      else if (index === 2) positionScore = 0.08;
+      else if (index === sentences.length - 1) positionScore = 0.05;
       return { sentence, index, score: wordScore + positionScore };
     });
 
@@ -508,17 +520,23 @@ export class ContentAnalyzer {
       const phraseCount = {};
       
       allPhrases.forEach(phrase => {
-        const cleaned = phrase.toLowerCase().trim();
-        if (cleaned.length > 2) {
+        // Strip edge punctuation and skip phrases made only of stop words
+        const cleaned = phrase.toLowerCase().trim().replace(/^[^a-z0-9]+|[^a-z0-9]+$/g, '');
+        const words = cleaned.split(/\s+/);
+        if (cleaned.length > 2 && words.some(w => !this.isStopWord(w))) {
           phraseCount[cleaned] = (phraseCount[cleaned] || 0) + 1;
         }
       });
 
-      // Score and rank topics
+      // Score and rank topics. Confidence is the phrase frequency relative to
+      // the MOST frequent phrase (RAKE-style relative salience) — the old
+      // frequency/totalPhrases normalization collapsed toward 0 on long
+      // documents, so every topic failed minConfidence and topics came back [].
+      const maxFreq = Math.max(1, ...Object.values(phraseCount));
       const topics = Object.entries(phraseCount)
         .map(([topic, frequency]) => ({
           topic,
-          confidence: Math.min(1, frequency / Math.max(allPhrases.length, 1)),
+          confidence: Math.round((frequency / maxFreq) * 100) / 100,
           keywords: topic.split(' ').filter(w => w.length > 2)
         }))
         .filter(topic => topic.confidence >= options.minConfidence)
@@ -543,14 +561,53 @@ export class ContentAnalyzer {
     try {
       const doc = nlp(text);
 
-      const people = doc.people().out('array');
-      const places = doc.places().out('array');
-      const organizations = doc.organizations().out('array');
+      // compromise's out('array') keeps adjoining punctuation ("Craigslist.",
+      // "United States,") — clean edges, drop bare stopword/pronoun tokens,
+      // and dedupe case-insensitively keeping the first casing seen.
+      const clean = (arr) => this.dedupeEntities(
+        arr.map(e => this.cleanEntityText(e))
+          .filter(e => e.length > 1
+            && !this.isBareStopword(e)
+            && !/^(?:inc|corp|ltd|co|llc)\.?$/i.test(e)) // bare corporate-suffix fragment
+      );
+
+      let people = clean(doc.people().out('array'));
+      let places = clean(doc.places().out('array'));
+      let organizations = clean(doc.organizations().out('array'));
       // .dates() needs the compromise-dates plugin (not installed) and threw,
       // aborting ALL entity extraction; #Date+ tag matching is core compromise.
-      const dates = doc.match('#Date+').out('array');
-      const money = doc.money().out('array');
-      let other = doc.topics().out('array').slice(0, 10);
+      const dates = clean(doc.match('#Date+').out('array'));
+      const money = clean(doc.money().out('array'));
+      let other = clean(doc.topics().out('array').slice(0, 10))
+        .filter(e => !this.isSentenceInitialArtifact(e, text));
+
+      // "X v. Y" fragments are legal-case citations, not people/places/orgs —
+      // reclassify them under "other".
+      const legalCases = [];
+      const extractCases = (list) => list.filter(e => {
+        if (this.isLegalCaseFragment(e)) {
+          legalCases.push(e);
+          return false;
+        }
+        return true;
+      });
+      people = extractCases(people);
+      places = extractCases(places);
+      organizations = extractCases(organizations);
+
+      // A bare ALL-CAPS token (UNIX, DOM) is not enough evidence for an
+      // organization — demote to "other" unless the text corroborates it
+      // (corporate suffix or organization-headed alias definition).
+      const demotedAcronyms = [];
+      organizations = organizations.filter(e => {
+        if (/^[A-Z]{2,}$/.test(e) && !this.hasOrganizationEvidence(e, text)) {
+          demotedAcronyms.push(e);
+          return false;
+        }
+        return true;
+      });
+
+      other = [...other, ...legalCases, ...demotedAcronyms];
 
       // Supplement with capitalized proper nouns that compromise may miss
       // (technology names, product names, etc.)
@@ -558,15 +615,27 @@ export class ContentAnalyzer {
         ...people, ...places, ...organizations, ...other
       ].map(e => e.toLowerCase()));
 
-      const properNouns = text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [];
-      const supplemental = [...new Set(properNouns)]
-        .filter(n => !existingEntities.has(n.toLowerCase()) && n.length > 1)
+      const properNouns = (text.match(/\b[A-Z][a-zA-Z.]+(?:\s+[A-Z][a-zA-Z.]+)*/g) || [])
+        // Split matches that crossed a sentence boundary ("XPath. In") — a
+        // period after a lowercase letter followed by a capital is a sentence
+        // end, while abbreviations ("U.S. Holdings") stay intact.
+        .flatMap(n => n.split(/(?<=[a-z]\.)\s+(?=[A-Z])/));
+      const supplemental = this.dedupeEntities(properNouns.map(n => this.cleanEntityText(n)))
+        .filter(n => n.length > 1
+          && !existingEntities.has(n.toLowerCase())
+          && !this.isSentenceInitialArtifact(n, text))
         .slice(0, 10);
 
       if (supplemental.length > 0) {
         other = [...other, ...supplemental].slice(0, 15);
       }
 
+      // Keep "other" free of entities already classified more specifically.
+      const classified = new Set(
+        [...people, ...places, ...organizations, ...dates, ...money].map(e => e.toLowerCase())
+      );
+      other = this.dedupeEntities(other).filter(e => !classified.has(e.toLowerCase()));
+
       const allEntities = [...people, ...places, ...organizations, ...dates, ...money, ...other];
       const uniqueEntities = new Set(allEntities.map(e => e.toLowerCase()));
 
@@ -828,6 +897,120 @@ export class ContentAnalyzer {
     return 'Very Difficult';
   }
 
+  /**
+   * Strip leading/trailing punctuation from an entity string while preserving
+   * trailing periods on abbreviations ("Inc.", "U.S.") and internal
+   * punctuation ("Home.dk", "Bidder's Edge").
+   * @param {string} raw - Raw entity string
+   * @returns {string} - Cleaned entity string
+   */
+  cleanEntityText(raw) {
+    let entity = String(raw).trim();
+    let previous;
+    do {
+      previous = entity;
+      entity = entity.replace(/^[\s"'‘’“”`([{<,;:!?«»–—-]+/, '');
+      entity = entity.replace(/[\s"'‘’“”`)\]}>,;:!?«»–—-]+$/, '');
+      if (entity.endsWith('.')) {
+        const lastToken = entity.split(/\s+/).pop();
+        const isAbbreviation = /^(?:[A-Za-z]\.)+$/.test(lastToken)
+          || /^(?:inc|corp|ltd|co|llc|jr|sr|st|mr|mrs|ms|dr|no|vs?)\.$/i.test(lastToken);
+        if (!isAbbreviation) {
+          entity = entity.replace(/\.+$/, '');
+        }
+      }
+    } while (entity !== previous);
+    return entity;
+  }
+
+  /**
+   * Deduplicate entities case-insensitively, keeping the first casing seen
+   * @param {string[]} entities - Entity strings
+   * @returns {string[]} - Deduplicated entity strings
+   */
+  dedupeEntities(entities) {
+    const seen = new Set();
+    const result = [];
+    for (const entity of entities) {
+      const key = entity.toLowerCase();
+      if (!seen.has(key)) {
+        seen.add(key);
+        result.push(entity);
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Check whether an entity is a single bare stop word/pronoun token
+   * @param {string} entity - Cleaned entity string
+   * @returns {boolean} - True if a bare stop word
+   */
+  isBareStopword(entity) {
+    return !entity.includes(' ') && this.isStopWord(entity.toLowerCase());
+  }
+
+  /**
+   * True when a single capitalized token is likely just a sentence-initial
+   * common word ("While", "It", "Once"): it is a stop word, the same word
+   * also appears in lowercase elsewhere in the text, or it is only ever
+   * capitalized at the start of a sentence (real proper nouns keep their
+   * capital mid-sentence).
+   * @param {string} entity - Cleaned entity string
+   * @param {string} text - Full source text
+   * @returns {boolean} - True if likely a sentence-initial artifact
+   */
+  isSentenceInitialArtifact(entity, text) {
+    if (entity.includes(' ')) return false;
+    const lower = entity.toLowerCase();
+    if (this.isStopWord(lower)) return true;
+    if (!/^[A-Z][a-z]+$/.test(entity)) return false;
+
+    // Word also appears in lowercase → ordinary word, capitalized only by position
+    if (new RegExp(`(?:^|[^A-Za-z])${lower}(?:[^A-Za-z]|$)`).test(text)) return true;
+
+    // Keep only if at least one occurrence is NOT at a sentence start
+    const occurrence = new RegExp(`\\b${entity}\\b`, 'g');
+    let match;
+    while ((match = occurrence.exec(text)) !== null) {
+      const before = text.slice(0, match.index).replace(/[\s"'‘’“”()[\]]+$/, '');
+      if (before.length > 0 && !/[.!?:]$/.test(before)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Check whether an entity is an "X v. Y" / "X vs. Y" legal-case fragment
+   * @param {string} entity - Cleaned entity string
+   * @returns {boolean} - True if a legal-case citation fragment
+   */
+  isLegalCaseFragment(entity) {
+    return /\s+vs?\.?\s+/i.test(entity);
+  }
+
+  /**
+   * Corroborating evidence that an ALL-CAPS token really is an organization:
+   * a corporate suffix follows it ("IBM Corp"), or it is introduced as the
+   * alias of an organization-headed name ("American Airlines (AA)",
+   * "French Data Protection Authority (CNIL)").
+   * @param {string} acronym - ALL-CAPS candidate (already /^[A-Z]{2,}$/)
+   * @param {string} text - Full source text
+   * @returns {boolean} - True if the text corroborates the org classification
+   */
+  hasOrganizationEvidence(acronym, text) {
+    const corporateSuffix = new RegExp(
+      `\\b${acronym},?\\s+(?:Inc|Corp|Corporation|Company|Co|Ltd|LLC|Group)\\.?(?:[^a-zA-Z]|$)`
+    );
+    if (corporateSuffix.test(text)) return true;
+
+    const orgHeadedAlias = new RegExp(
+      `\\b(?:Airlines?|Authority|Agency|Association|Bureau|Commission|Committee|Corporation|Company|Council|Court|Foundation|Group|Institute|Institution|Organi[sz]ation|Press|Society|Union|University)\\s*\\(["'“‘]?${acronym}["'”’]?\\)`
+    );
+    return orgHeadedAlias.test(text);
+  }
+
   /**
    * Check if word is a stop word
    * @param {string} word - Word to check

package/src/tools/advanced/ScrapeWithActionsTool.js

@@ -73,7 +73,11 @@ const ScrollActionSchema = BaseActionSchema.extend({
   direction: z.enum(['up', 'down', 'left', 'right']).default('down'),
   distance: z.number().min(0).default(100),
   smooth: z.boolean().default(true),
-  toElement: z.string().optional()
+  toElement: z.string().optional(),
+  // Absolute scroll-to coordinates (window.scrollTo). When present they take
+  // precedence over direction/distance — see ActionExecutor's ScrollActionSchema.
+  x: z.number().min(0).optional(),
+  y: z.number().min(0).optional()
 });
 
 const ScreenshotActionSchema = BaseActionSchema.extend({

package/src/tools/advanced/batchScrape/worker.js

@@ -8,6 +8,7 @@ import { load } from 'cheerio';
 import { config as appConfig } from '../../../constants/config.js';
 import { ssrfGuard, isSsrfError } from '../../../utils/ssrfGuard.js';
 import { throttleHost } from '../../../utils/hostRateLimiter.js';
+import { htmlToMarkdown } from '../../../utils/htmlToMarkdown.js';
 
 const USER_AGENT = 'MCP-WebScraper-BatchTool/1.0.0';
 
@@ -165,7 +166,6 @@ function generateFormats($, html, formats) {
 }
 
 function buildMarkdown($) {
-  let md = '';
   const title = $('title').text().trim();
 
   const selectors = ['article', 'main', '.content', '#content', '.post-content', '.entry-content'];
@@ -176,18 +176,17 @@ function buildMarkdown($) {
   }
   if (!$body || $body.length === 0) $body = $('body');
 
+  // Full-fidelity conversion via the shared Turndown helper (same converter
+  // the unified `scrape` tool uses). The previous hand-rolled h1–h3/p/li walk
+  // silently dropped any text living outside those tags (e.g. quotes in
+  // <span>/<small> on quotes.toscrape.com).
+  let md = htmlToMarkdown($.html($body));
+
   // C3: de-dup title — only emit the <title> heading if the page has no <h1>
   // or if the first <h1> text differs from the <title> text (case-insensitive).
   const firstH1 = $body.find('h1').first().text().trim();
   const titleDuplicated = firstH1 && firstH1.toLowerCase() === title.toLowerCase();
-  if (title && !titleDuplicated) md += `# ${title}\n\n`;
-
-  $body.find('h1').each((_, el) => { md += `# ${$(el).text().trim()}\n\n`; });
-  $body.find('h2').each((_, el) => { md += `## ${$(el).text().trim()}\n\n`; });
-  $body.find('h3').each((_, el) => { md += `### ${$(el).text().trim()}\n\n`; });
-  $body.find('p').each((_, el) => { const t = $(el).text().trim(); if (t) md += `${t}\n\n`; });
-  $body.find('ul li').each((_, el) => { md += `- ${$(el).text().trim()}\n`; });
-  $body.find('ol li').each((_, el) => { md += `1. ${$(el).text().trim()}\n`; });
+  if (title && !titleDuplicated) md = `# ${title}\n\n${md}`;
 
   return md.trim();
 }

package/src/tools/advanced/scrapeWithActions/recorder.js

@@ -170,6 +170,9 @@ export function buildRecordedEntry(action, timestampMsSinceStart) {
   if (action.value !== undefined) entry.value = action.value;
   if (action.direction !== undefined) entry.direction = action.direction;
   if (action.distance !== undefined) entry.distance = action.distance;
+  // scroll: absolute scroll-to coordinates
+  if (action.x !== undefined) entry.x = action.x;
+  if (action.y !== undefined) entry.y = action.y;
   if (action.description !== undefined) entry.description = action.description;
   // executeJavaScript actions require `script` to replay (ActionExecutor's
   // ActionChainSchema rejects the entry otherwise) — preserve it.

package/src/tools/basic/extractText.js

@@ -89,6 +89,13 @@ export async function extractTextHandler({ url, remove_scripts, remove_styles, o
     if (remove_scripts !== false) $('script').remove();
     if (remove_styles !== false) $('style').remove();
 
+    // <noscript> contents are parsed as raw TEXT when scripting is enabled
+    // (cheerio/parse5 default, per the HTML spec), so leaving them in leaks
+    // literal markup into the extracted text — e.g. Wikipedia's
+    // Special:CentralAutoLogin 1x1 <img> tracking pixel. Browsers with JS
+    // enabled never render noscript content, so always strip it.
+    $('noscript').remove();
+
     $('nav, header, footer, aside, .advertisement, .ad, .sidebar').remove();
 
     const result = {

package/src/tools/extract/_fetchAndParse.js

@@ -96,6 +96,37 @@ function classifyContentType(contentType) {
   return 'binary';
 }
 
+/**
+ * Flatten a parsed document's <body> to text while preserving line structure.
+ *
+ * Cheerio's .text() joins adjacent elements with no separator, which welds
+ * table rows / list items together ("1.Story title329 points") and starves
+ * downstream LLM extraction (AgentOrchestrator, extractStructured,
+ * extractWithLlm) of any structure to parse. Works on a detached clone so the
+ * caller's $ tree is untouched — unifiedScrape reuses $ for other formats.
+ *
+ * @param {import('cheerio').CheerioAPI} $
+ * @returns {string}
+ */
+export function flattenBodyText($) {
+  // Block boundaries are marked with a U+E000 private-use sentinel so the
+  // HTML source's own insignificant newlines can be collapsed to spaces
+  // first, and only the sentinels become line breaks. (NUL won't survive:
+  // .after()/.replaceWith() parse their argument as HTML and the parser
+  // strips NUL; U+E000 passes through and never occurs in real page text.)
+  const $body = $('body').clone();
+  $body.find('br').replaceWith('\uE000');
+  $body.find('td, th').after(' ');
+  $body
+    .find('p, div, li, tr, h1, h2, h3, h4, h5, h6, blockquote, pre, table, ul, ol, dl, section, article, header, footer')
+    .after('\uE000');
+  return $body
+    .text()
+    .replace(/\s+/g, ' ')
+    .replace(/ ?(?:\uE000 ?)+/g, '\n')
+    .trim();
+}
+
 /**
  * Fetch a URL and return parsed HTML via Cheerio.
  *
@@ -149,7 +180,7 @@ export async function fetchAndParse(url, options = {}) {
     $(stripTags.join(', ')).remove();
   }
 
-  const textContent = $('body').text().replace(/\s+/g, ' ').trim();
+  const textContent = flattenBodyText($);
 
   return { html, $, textContent, finalUrl: response.url };
 }

package/src/tools/llmstxt/generateLLMsTxt.js

@@ -1,7 +1,9 @@
 import { z } from 'zod';
+import { load } from 'cheerio';
 import { LLMsTxtAnalyzer } from '../../core/LLMsTxtAnalyzer.js';
 import { Logger } from '../../utils/Logger.js';
 import { getBaseUrl } from '../../utils/urlNormalizer.js';
+import { safeFetch } from '../../utils/ssrfGuard.js';
 
 const logger = new Logger('GenerateLLMsTxtTool');
 
@@ -79,6 +81,14 @@ export class GenerateLLMsTxtTool {
       });
       const analysis = await analyzer.analyzeWebsite(url, analysisOptions);
 
+      // Capture the homepage's <title>/h1 and meta/og description so the
+      // spec-compliant output can carry a real site summary and a named Home
+      // link (llmstxt.org) instead of boilerplate. Best-effort: null on
+      // failure, in which case the generic fallbacks below apply.
+      if (!outputOptions.robotsStyle) {
+        analysis.homePage = await this.fetchHomePageMetadata(baseUrl);
+      }
+
       // Step 2: Generate LLMs.txt Content
       const llmsTxtContent = this.generateLLMsTxt(analysis, outputOptions, complianceLevel);
 
@@ -156,8 +166,10 @@ export class GenerateLLMsTxtTool {
     lines.push(`# ${title}`);
     lines.push('');
 
-    // Blockquote summary (required by spec)
-    const summary = `Site map and key resources for ${baseUrl}, generated to help LLMs locate relevant content.`;
+    // Blockquote summary (required by spec). Prefer the site's own meta /
+    // og:description captured from the homepage over generic boilerplate.
+    const summary = analysis.homePage?.description
+      || `Site map and key resources for ${baseUrl}, generated to help LLMs locate relevant content.`;
     lines.push(`> ${summary}`);
     lines.push('');
 
@@ -177,13 +189,16 @@ export class GenerateLLMsTxtTool {
       lines.push('');
     }
 
-    // Helper: emit a "## Section" with a list of [name](url) links.
+    // Helper: emit a "## Section" with a list of [name](url) links. Link
+    // names prefer the page's actual <title> captured during analysis;
+    // otherwise a humanized full path ("/tag/abilities/page/1" ->
+    // "Tag: abilities — page 1") — never a bare trailing segment like "1".
+    const pageTitles = this.collectPageTitles(analysis);
     const linkLabel = (u) => {
+      const captured = pageTitles.get(this.normalizeTitleKey(u));
+      if (captured) return captured;
       try {
-        const p = new URL(u).pathname.replace(/\/+$/, '');
-        if (!p || p === '') return 'Home';
-        const seg = p.split('/').filter(Boolean).pop() || p;
-        return seg.replace(/[-_]/g, ' ').replace(/\.[a-z0-9]+$/i, '').trim() || p;
+        return this.humanizePath(new URL(u).pathname);
       } catch {
         return u;
       }
@@ -247,6 +262,131 @@ export class GenerateLLMsTxtTool {
     return lines.join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
   }
 
+  /**
+   * Build a URL -> page title map from titles captured during analysis
+   * (classifyPage stores each public page's <title>; the homepage fetch names
+   * the site root). A title shared verbatim by multiple URLs is dropped — it
+   * cannot identify a specific page, so those links fall back to the
+   * humanized path instead.
+   */
+  collectPageTitles(analysis) {
+    const titles = new Map();
+    const counts = new Map();
+    const add = (url, title) => {
+      if (!url || typeof title !== 'string') return;
+      const clean = title.replace(/\s+/g, ' ').trim();
+      if (!clean) return;
+      const key = this.normalizeTitleKey(url);
+      const prev = titles.get(key);
+      if (prev === clean) return;
+      if (prev) counts.set(prev, counts.get(prev) - 1);
+      titles.set(key, clean);
+      counts.set(clean, (counts.get(clean) || 0) + 1);
+    };
+
+    for (const entries of Object.values(analysis.contentTypes || {})) {
+      if (!Array.isArray(entries)) continue;
+      for (const entry of entries) {
+        add(entry?.url, entry?.metadata?.title);
+      }
+    }
+    if (analysis.homePage?.title && analysis.metadata?.baseUrl) {
+      add(analysis.metadata.baseUrl, analysis.homePage.title);
+    }
+
+    for (const [key, title] of titles) {
+      if (counts.get(title) > 1) titles.delete(key);
+    }
+    return titles;
+  }
+
+  normalizeTitleKey(url) {
+    try {
+      return new URL(url).toString();
+    } catch {
+      return url;
+    }
+  }
+
+  /**
+   * Humanize a full URL path into a readable label:
+   *   "/"                      -> "Home"
+   *   "/login"                 -> "Login"
+   *   "/author/Albert-Einstein" -> "Author: Albert Einstein"
+   *   "/tag/abilities/page/1"  -> "Tag: abilities — page 1"
+   * A trailing pagination segment ("page/2", "page-2", bare "2") is folded
+   * into "page N" so a link is never labeled by a bare number.
+   */
+  humanizePath(pathname) {
+    let raw = pathname;
+    try { raw = decodeURIComponent(pathname); } catch { /* keep encoded */ }
+    const segments = raw.split('/').filter(Boolean);
+    if (segments.length === 0) return 'Home';
+
+    // Strip a file extension from the last segment, then de-slug.
+    segments[segments.length - 1] = segments[segments.length - 1].replace(/\.[a-z0-9]+$/i, '');
+    const clean = segments.map((s) => s.replace(/[-_]+/g, ' ').trim()).filter(Boolean);
+    if (clean.length === 0) return 'Home';
+
+    let pageNum = null;
+    const last = clean[clean.length - 1];
+    const pageMatch = last.match(/^page\s*(\d+)$/i);
+    if (pageMatch) {
+      pageNum = pageMatch[1];
+      clean.pop();
+    } else if (/^\d+$/.test(last)) {
+      pageNum = last;
+      clean.pop();
+      if (clean.length > 0 && /^pages?$/i.test(clean[clean.length - 1])) {
+        clean.pop();
+      }
+    }
+    if (clean.length === 0) return pageNum ? `Page ${pageNum}` : 'Home';
+
+    const cap = (s) => s.charAt(0).toUpperCase() + s.slice(1);
+    let label = clean.length === 1
+      ? cap(clean[0])
+      : `${cap(clean[0])}: ${clean.slice(1).join(' / ')}`;
+    if (pageNum) label += ` — page ${pageNum}`;
+    return label;
+  }
+
+  /**
+   * Fetch the site's homepage once and extract naming metadata for the spec
+   * output. Best-effort: returns null on any failure.
+   */
+  async fetchHomePageMetadata(baseUrl) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), Math.min(this.options.timeout, 10000));
+    try {
+      const response = await safeFetch(baseUrl, {
+        signal: controller.signal,
+        headers: { 'User-Agent': this.options.userAgent }
+      });
+      if (!response.ok) return null;
+      return this.extractHomePageMetadata(await response.text());
+    } catch {
+      return null;
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+
+  /**
+   * Extract { title, description } from homepage HTML. Title prefers <title>
+   * then the first <h1>; description prefers meta[name="description"] then
+   * og:description. Returns null when neither is present.
+   */
+  extractHomePageMetadata(html) {
+    const $ = load(html);
+    const clean = (s) => (s || '').replace(/\s+/g, ' ').trim();
+    const title = clean($('title').first().text()) || clean($('h1').first().text());
+    const description = clean($('meta[name="description"]').attr('content'))
+      || clean($('meta[property="og:description"]').attr('content'));
+    if (!title && !description) return null;
+    return { title, description };
+  }
+
   /**
    * Generate legacy robots.txt-style content (opt-in via outputOptions.robotsStyle).
    */

package/src/tools/templates/TemplateRegistry.js

@@ -183,8 +183,11 @@ const TEMPLATES = [
           site: $row.find('.sitebit a').text().trim() || null,
           score: $score.text().replace(' points', '').trim() || null,
           author: $subtext.find('.hnuser').text().trim() || null,
-          posted: $subtext.find('.age a').attr('href') || null,
-          comments: $subtext.find('a[href*="item"]').last().text().trim() || null
+          // ".age a" wraps the relative age string ("3 hours ago"); its href is the item permalink.
+          posted: $subtext.find('.age a').text().trim() || null,
+          // The comments link is also an item?id= link, so exclude the age anchor.
+          // Job posts have no comments link at all -> null.
+          comments: $subtext.find('a[href*="item"]').not('.age a').last().text().trim() || null
         });
       });
       return { stories: stories.slice(0, 30), scraped_at: new Date().toISOString() };

package/src/tools/tracking/trackChanges/index.js

@@ -68,7 +68,10 @@ export class TrackChangesTool extends EventEmitter {
     // Scheduled-monitor subsystem (timers are NOT started here — only the
     // single server-owned instance calls startScheduler()).
     this._mcpServer = null;
-    this.monitorStore = new MonitorStore({ storageDir: this.options.monitorStorageDir || './monitors' });
+    // No storageDir fallback here: MonitorStore itself defaults to
+    // ~/.crawlforge/monitors (cwd-independent, like snapshotStorageDir above)
+    // and runs its legacy ./monitors migration only on that default path.
+    this.monitorStore = new MonitorStore({ storageDir: this.options.monitorStorageDir });
     this.scheduler = new MonitorScheduler({ tool: this, store: this.monitorStore });
 
     // Wired synchronously (no I/O) so no 'error' event emitted by
@@ -200,14 +203,18 @@ export class TrackChangesTool extends EventEmitter {
       snapshotInfo = await this.snapshotManager.storeSnapshot(url, sourceContent, { ...fetchMeta, baseline: true, trackingOptions }, { enableCompression: storageOptions.compressionEnabled });
     }
 
+    // ChangeTracker.createBaseline returns a summary ({contentHash, sections,
+    // elements, createdAt, ...}), NOT the internal baseline object — reading
+    // baseline.analysis?./baseline.timestamp here always yielded
+    // undefined/0/0/undefined regardless of the page.
     return {
       success: true, operation: 'create_baseline', url,
       baseline: {
         version: baseline.version,
-        contentHash: baseline.analysis?.hashes?.page,
-        sections: Object.keys(baseline.analysis?.hashes?.sections || {}).length,
-        elements: Object.keys(baseline.analysis?.hashes?.elements || {}).length,
-        createdAt: baseline.timestamp,
+        contentHash: baseline.contentHash,
+        sections: baseline.sections,
+        elements: baseline.elements,
+        createdAt: baseline.createdAt,
         options: trackingOptions
       },
       snapshot: snapshotInfo, timestamp: Date.now()