crawlforge-mcp-server 6.22.2 → 6.23.0

package/src/core/ChangeTracker.js

@@ -28,6 +28,7 @@ const ChangeTrackingSchema = z.object({
     trackLinks: z.boolean().default(true),
     ignoreWhitespace: z.boolean().default(true),
     ignoreCase: z.boolean().default(false),
+    includeMarkupChanges: z.boolean().default(false),
     customSelectors: z.array(z.string()).optional(),
     excludeSelectors: z.array(z.string()).optional().default([
       'script', 'style', 'noscript', '.advertisement', '.ad'
@@ -344,10 +345,12 @@ export class ChangeTracker extends EventEmitter {
       const currentAnalysis = await this.analyzeContent(currentContent, baseline.options);
       
       // Perform comprehensive change detection
+      // includeMarkupChanges only shapes the response, so unlike the analysis
+      // options it may be asked for at compare time.
       const changeAnalysis = await this.detectChanges(
         baseline.analysis,
         currentAnalysis,
-        baseline.options
+        { ...baseline.options, includeMarkupChanges: options.includeMarkupChanges ?? baseline.options.includeMarkupChanges }
       );
       
       // Calculate change significance
@@ -410,7 +413,16 @@ export class ChangeTracker extends EventEmitter {
         metrics: changeRecord.metrics,
         recommendations: this.generateChangeRecommendations(changeRecord),
         ...(alerts.length ? { alerts } : {}),
+        // ignoredOptions is the structured form of the warning: each entry
+        // names what the caller asked for and what the baseline applied, so
+        // a compare at granularity "text" against a "section" baseline says
+        // so in a field, not only in prose (6.12).
         ...(ignoredOptions.length ? {
+          ignoredOptions: ignoredOptions.map(option => ({
+            option,
+            requested: options[option],
+            applied: baseline.options[option] ?? null
+          })),
           warnings: [
             `${ignoredOptions.join(', ')} passed to this compare ${ignoredOptions.length === 1 ? 'was' : 'were'} ignored — ` +
             `the baseline's options are applied to both sides of the diff. Recreate the baseline to change them.`
@@ -615,13 +627,23 @@ export class ChangeTracker extends EventEmitter {
       changes.modifiedElements.push(...elementChanges.modified);
     }
     
-    // Detect text changes
+    // Detect text changes. Diffed over the visible text, not the raw HTML,
+    // so a class rename or icon markup no longer reads as a text change
+    // (live sweep 4.1). The raw markup diff is kept apart in markupChanges
+    // and only computed when the caller asks for it.
     if (options.trackText) {
       changes.textChanges = await this.detectTextChanges(
-        baseline.originalContent,
-        current.originalContent,
+        this.visibleText(baseline.originalContent),
+        this.visibleText(current.originalContent),
         options
       );
+      if (options.includeMarkupChanges) {
+        changes.markupChanges = await this.detectTextChanges(
+          baseline.originalContent,
+          current.originalContent,
+          options
+        );
+      }
     }
     
     // Detect monetary value changes. Scored by magnitude rather than by how
@@ -1038,6 +1060,32 @@ export class ChangeTracker extends EventEmitter {
     return changes;
   }
   
+  /**
+   * The text a reader sees: text nodes in document order, one per line, with
+   * script, style, noscript and template content dropped. Input with no
+   * markup (a plain-text content param) is returned unchanged.
+   * @param {string} content - HTML or plain text
+   * @returns {string} - Visible text
+   */
+  visibleText(content = '') {
+    if (!/<[a-z!/]/i.test(content)) return content;
+    const $ = load(content);
+    $('script, style, noscript, template').remove();
+    const parts = [];
+    const walk = nodes => {
+      for (const node of nodes) {
+        if (node.type === 'text') {
+          const text = node.data.trim();
+          if (text) parts.push(text);
+        } else if (node.children) {
+          walk(node.children);
+        }
+      }
+    };
+    walk($.root().contents().toArray());
+    return parts.join('\n');
+  }
+
   async detectTextChanges(baselineContent, currentContent, options = {}) {
     const textChanges = [];
     

package/src/core/LLMsTxtAnalyzer.js

@@ -240,8 +240,10 @@ export class LLMsTxtAnalyzer {
             // "https://site.com/" + "/json" was listed as "https://site.com//json" (R25 1.8).
             const apiUrl = new URL(path, baseUrl).toString();
             const response = await this.fetchWithTimeout(apiUrl, { timeout: 5000 });
-            if (response.ok) {
-              const contentType = response.headers.get('content-type') || '';
+            const contentType = response.headers.get('content-type') || '';
+            // A 200 alone is not an API: gov.uk's /api redirects to an HTML
+            // help page (live sweep 4.4). Only machine-readable answers count.
+            if (response.ok && /json|xml/i.test(contentType)) {
               return {
                 url: apiUrl,
                 type: this.determineAPIType(apiUrl, contentType),
@@ -739,23 +741,33 @@ export class LLMsTxtAnalyzer {
       const html = await readBody(response);
       const $ = load(html);
 
-      // Check for forms
-      if ($('form').length > 0) {
+      // Check for forms. A site-wide search box or a hidden feedback form
+      // is page chrome, not the page's purpose: counting them made every
+      // docs.python.org and gov.uk page a "forms" page (live sweep 4.4).
+      const forms = $('form').filter((_, el) => {
+        const form = $(el);
+        return form.attr('hidden') === undefined
+          && form.closest('[role="search"]').length === 0
+          && form.find('input[type="search"]').length === 0;
+      });
+      if (forms.length > 0) {
         return { 
           category: 'forms', 
           type: 'interactive', 
           confidence: 0.9,
-          metadata: { formCount: $('form').length }
+          metadata: { formCount: forms.length, title: pageTitle($) }
         };
       }
 
-      // Check for login/auth indicators
-      if (html.includes('login') || html.includes('password') || $('input[type="password"]').length > 0) {
+      // Check for login/auth indicators. Substring checks for "login" and
+      // "password" matched link hrefs (gov.uk's /student-finance-register-login).
+      if ($('input[type="password"]').length > 0) {
         return { category: 'restricted', type: 'authentication', confidence: 0.8 };
       }
 
-      // Check for dynamic content indicators
-      if (html.includes('application/json') || contentType.includes('json')) {
+      // Check for dynamic content indicators (the response, not a
+      // <script type="application/json"> inside an HTML page)
+      if (contentType.includes('json')) {
         return { category: 'dynamic', type: 'api', confidence: 0.9 };
       }
 

package/src/core/LocalizationManager.js

@@ -45,6 +45,20 @@ const SUPPORTED_COUNTRIES = {
 // "zh-Hans-CN"). "klingon" used to come back as "klingon-DE" (R24).
 const LANGUAGE_TAG = /^[a-z]{2,3}(-[a-z]{4})?(-([a-z]{2}|\d{3}))?$/i;
 
+// A BCP 47 tag that always carries a region, in canonical case: a bare "de"
+// for DE comes back "de-DE" (it used to stay "de" while JP's default answered
+// "ja-JP"), "zh-Hans" for CN becomes "zh-Hans-CN", "EN-gb" becomes "en-GB".
+function withRegion(language, countryCode) {
+  const [lang, ...rest] = String(language).split('-');
+  const script = rest.find(part => part.length === 4);
+  const region = rest.find(part => part.length !== 4) || countryCode;
+  return [
+    lang.toLowerCase(),
+    script && script[0].toUpperCase() + script.slice(1).toLowerCase(),
+    region.toUpperCase()
+  ].filter(Boolean).join('-');
+}
+
 // Common language mappings for Accept-Language headers with cultural preferences
 const LANGUAGE_MAPPINGS = {
   'en': 'en-US,en;q=0.9',
@@ -261,16 +275,17 @@ export class LocalizationManager extends EventEmitter {
     }
     
     const countryData = SUPPORTED_COUNTRIES[validatedInput.countryCode];
+    const language = withRegion(validatedInput.language || countryData.language, validatedInput.countryCode);
     
     // Merge country defaults with custom options
     const localizationConfig = {
       countryCode: validatedInput.countryCode,
-      language: validatedInput.language || countryData.language,
+      language,
       timezone: validatedInput.timezone || countryData.timezone,
       currency: validatedInput.currency || countryData.currency,
       searchDomain: countryData.searchDomain,
       acceptLanguage: validatedInput.acceptLanguage
-        || this.buildAcceptLanguageHeader(validatedInput.language || countryData.language, validatedInput.countryCode),
+        || this.buildAcceptLanguageHeader(language, validatedInput.countryCode),
       customHeaders: validatedInput.customHeaders || {},
       geoLocation: validatedInput.geoLocation,
       proxySettings: validatedInput.proxySettings,
@@ -1291,10 +1306,11 @@ export class LocalizationManager extends EventEmitter {
       return domainMappings[domain];
     }
     
-    // TLD-based suggestions
+    // TLD-based suggestions. ".uk" is the one ccTLD that is not its ISO
+    // code: a .co.uk site used to fall through to the US-first defaults.
     const tldMatch = domain.match(/\.([a-z]{2})$/);
     if (tldMatch) {
-      const tld = tldMatch[1].toUpperCase();
+      const tld = tldMatch[1] === 'uk' ? 'GB' : tldMatch[1].toUpperCase();
       if (SUPPORTED_COUNTRIES[tld]) {
         return [tld];
       }

package/src/core/ResearchOrchestrator.js

@@ -12,6 +12,7 @@ import { safeFetch, safeGoto, assertNavigationAllowed } from '../utils/ssrfGuard
 import { serverStealthProxies } from '../constants/config.js';
 import { preflightFetch, browserPreflight, redirectGate } from '../utils/robotsGate.js';
 import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
+import { guardedProxyFor } from './browser/guardedProxy.js';
 import { noteRetryAfter } from '../utils/hostRateLimiter.js';
 import { load } from 'cheerio';
 import { flattenText } from 'crawlforge-extractors';
@@ -120,6 +121,25 @@ export function claimStatements(point) {
     .filter(Boolean);
 }
 
+/**
+ * The first `max` characters of a page's text, ended at the last sentence or
+ * line end inside them. A plain cut leaves the last sentence half written, and
+ * the summarizer took it as a key point: "…the same hostname can range from no
+ * bot pro" was a finding in the R25 re-check (2026-10-19). Cut as before when
+ * no end falls in the second half.
+ *
+ * @param {string} text
+ * @param {number} max
+ * @returns {string}
+ */
+export function cutAtSentence(text, max) {
+  if (text.length <= max) return text;
+  const cut = text.slice(0, max);
+  const ends = [...cut.matchAll(/[.!?]["'\u201D\u2019)]*(?=\s)|\n/g)];
+  const end = ends.length > 0 ? ends[ends.length - 1].index + ends[ends.length - 1][0].length : 0;
+  return end > max / 2 ? cut.slice(0, end).trimEnd() : cut;
+}
+
 /**
  * Page text for the plain-fetch fallback, one line per block element. The
  * regex tag-strip it replaces ran every block together on one line.
@@ -896,7 +916,7 @@ export class ResearchOrchestrator extends EventEmitter {
                   const textContent = fallbackPageText(html);
                   if (textContent.length > 50) {
                     contentData = {
-                      content: textContent.slice(0, 5000),
+                      content: cutAtSentence(textContent, 5000),
                       metadata: { title: source.title || '' },
                       structuredData: {},
                       fallback: true
@@ -1078,6 +1098,9 @@ export class ResearchOrchestrator extends EventEmitter {
               launchOptions.geoip = true;
             }
             this._stealthBrowser = await camoufox.Camoufox(launchOptions);
+            // Each context goes out through the SSRF-guarded proxy, chained to
+            // this one (src/core/browser/guardedProxy.js).
+            this._stealthUpstream = proxy ? { server: proxy.url } : null;
             this._stealthEngineActive = 'camoufox';
             this.logger.info('Stealth fallback using Camoufox (Firefox) engine', {
               proxy: proxy ? proxy.redacted : null
@@ -1188,7 +1211,10 @@ export class ResearchOrchestrator extends EventEmitter {
     if (this._stealthEngineActive === 'camoufox') {
       // viewport:null skips Browser.setDefaultViewport, whose playwright-core
       // 1.62 payload (isMobile etc.) camoufox's older Firefox build rejects.
-      camoufoxContext = await this._stealthBrowser.newContext({ viewport: null });
+      camoufoxContext = await this._stealthBrowser.newContext({
+        viewport: null,
+        proxy: await guardedProxyFor(this._stealthUpstream)
+      });
       page = await camoufoxContext.newPage();
     } else {
       const { contextId } = await this._stealthManager.createStealthContext({ level: this.stealthLevel });
@@ -1425,7 +1451,7 @@ export class ResearchOrchestrator extends EventEmitter {
       try {
         if (!source.extractedContent) continue;
         
-        const content = source.extractedContent.substring(0, 5000); // Limit content length
+        const content = cutAtSentence(source.extractedContent, 5000); // Limit content length
         
         // Use summarization to extract key points
         const summary = await this.summarizeTool.execute({

package/src/core/StealthBrowserManager.js

@@ -23,6 +23,7 @@ import { guardFirefoxPageErrors } from '../utils/firefoxPageErrorGuard.js';
 import { serverStealthProxies } from '../constants/config.js';
 import { ClearanceJar, sharedClearanceJar } from './ClearanceJar.js';
 import { readWindowState } from './browser/windowState.js';
+import { guardedProxyFor } from './browser/guardedProxy.js';
 
 // Grace given to a document that rendered no title and no text (see _waitOutEmptyDocument).
 export const EMPTY_DOCUMENT_GRACE_MS = 8000;
@@ -873,9 +874,6 @@ export class StealthBrowserManager {
     const proxy = this._launchedEngine === 'camoufox'
       ? (this.browser[CAMOUFOX_PROXY] || null)
       : this.resolveProxy(validatedConfig);
-    if (proxy) {
-      contextOptions.proxy = proxy;
-    }
 
     // The identity a clearance is bound to: engine, the User-Agent this context
     // presents, and the exit it goes out through. Read before the camoufox
@@ -953,6 +951,12 @@ export class StealthBrowserManager {
       }
     }
 
+    // What goes on the context is the SSRF-guarded local proxy, chained to
+    // that one: it checks every connection the page opens — redirect hops,
+    // iframes, WebSockets — where route() never sees them, and keeps the
+    // upstream's credentials itself (src/core/browser/guardedProxy.js).
+    contextOptions.proxy = await guardedProxyFor(proxy);
+
     const context = await this.browser.newContext(contextOptions);
     const contextId = this.generateContextId();
 
@@ -3028,7 +3032,8 @@ export class StealthBrowserManager {
       const response = await safeGoto(page, url, { waitUntil: 'domcontentloaded', onRedirect });
       const landedUrl = page.url();
       if (response && status === null) status = response.status();
-      if (wait_for > 0) await page.waitForTimeout(wait_for);
+      const waitedMs = wait_for > 0 ? wait_for : 0;
+      if (waitedMs > 0) await page.waitForTimeout(waitedMs);
       // Cloudflare's and Vercel's JavaScript challenges solve themselves in a
       // real browser and then reload the page. camoufox on lesswrong.com was
       // read while the "Vercel Security Checkpoint" interstitial was still
@@ -3087,7 +3092,7 @@ export class StealthBrowserManager {
         this._markBlocked(contextId, [url, page.url()]);
       }
 
-      return { success: true, url, title, text, html, screenshot: shot, status, gracedMs, engine: engineUsed, warnings, ...(windowState ? { windowState } : {}) };
+      return { success: true, url, title, text, html, screenshot: shot, status, waitedMs, gracedMs, engine: engineUsed, warnings, ...(windowState ? { windowState } : {}) };
     } finally {
       await this.closeContext(contextId).catch(() => {});
     }

package/src/core/analysis/ContentAnalyzer.js

@@ -255,7 +255,7 @@ export class ContentAnalyzer {
 
       // Readability metrics
       if (analysisOptions.includeReadabilityMetrics) {
-        result.readability = await this.calculateReadability(text);
+        result.readability = await this.calculateReadability(text, analysisOptions);
       }
 
       // Sentiment analysis
@@ -512,6 +512,48 @@ export class ContentAnalyzer {
     return latin.length / letters.length < 0.5;
   }
 
+  /**
+   * Whether the text is confidently not English, so the English-only sentiment
+   * lexicon and Flesch formula must not be applied. A Spanish complaint scored
+   * -0.08 and Spanish and German got English Flesch scores (live sweep
+   * 2026-10-20). franc misfiles short English — "I am so happy today" is
+   * Somali — so a detected language counts only when it has a stop-word list,
+   * the script is not Latin, or the text is long enough to detect reliably.
+   * @param {string} text
+   * @param {{ detectedLanguage?: string|null }} options
+   * @returns {boolean}
+   */
+  isNonEnglishText(text, options = {}) {
+    if (!this.needsSegmentedTerms(text, options)) return false;
+    if (this.isCjkText(text) || !this.isLatinScriptText(text)) return true;
+    return Boolean(LANGUAGE_STOP_WORDS[options.detectedLanguage])
+      || text.split(/\s+/).filter(Boolean).length >= 20;
+  }
+
+  /**
+   * The noun run of a compromise noun phrase: leading determiners, numbers
+   * and adjectives and trailing non-nouns dropped. compromise tags "feel" in
+   * "The switches feel crisp" as a noun, and the whole clause came back as a
+   * topic (sweep 2026-10-20); a plural noun ends an English compound
+   * ("battery life", not "batteries life"), so a noun after it is cut off.
+   * A phrase of only time words ("two weeks", "Monday") is not a topic.
+   * @param {Object} phrase - One compromise noun-phrase match
+   * @returns {string} - The head, or '' when there is none
+   */
+  nounPhraseHead(phrase) {
+    const terms = phrase.terms().json().map(t => t.terms[0]);
+    const isNoun = t => t.tags.includes('Noun') && !t.tags.includes('Pronoun');
+    const first = terms.findIndex(isNoun);
+    if (first === -1) return '';
+    let last = terms.length - 1;
+    while (!isNoun(terms[last])) last--;
+    const head = terms.slice(first, last + 1);
+    const plural = head.findIndex(t => t.tags.includes('Plural'));
+    if (plural !== -1 && plural < head.length - 1 && isNoun(head[plural + 1])) head.length = plural + 1;
+    if (head.every(t => t.tags.includes('Date'))) return '';
+    return head.map(t => t.text).join(' ');
+  }
+
   async extractTopics(text, options = {}) {
     try {
       // compromise noun-phrase matching is English-only: on CJK text it emits
@@ -533,8 +575,8 @@ export class ContentAnalyzer {
 
       const doc = nlp(text);
 
-      // Extract noun phrases as potential topics
-      const nounPhrases = doc.nouns().out('array');
+      // Extract noun-phrase heads as potential topics
+      const nounPhrases = doc.nouns().map(phrase => this.nounPhraseHead(phrase)).filter(Boolean);
       const adjNounPhrases = doc.match('#Adjective+ #Noun+').out('array');
       
       // Combine and count frequency
@@ -597,8 +639,12 @@ export class ContentAnalyzer {
       // compromise's out('array') keeps adjoining punctuation ("Craigslist.",
       // "United States,") — clean edges, drop bare stopword/pronoun tokens,
       // and dedupe case-insensitively keeping the first casing seen.
+      // On other languages compromise's capital-letter heuristics also pick
+      // up function words and titles: "über" as an organisation, "Berlin den",
+      // "Präsidenten Emmanuel Macron" (sweep 2026-10-20).
+      const language = options.detectedLanguage;
       const clean = (arr) => this.dedupeEntities(
-        arr.map(e => this.cleanEntityText(e))
+        arr.map(e => this.trimForeignEntity(this.cleanEntityText(e), language))
           .filter(e => e.length > 1
             && !this.isBareStopword(e)
             && !/^(?:inc|corp|ltd|co|llc)\.?$/i.test(e)) // bare corporate-suffix fragment
@@ -609,7 +655,10 @@ export class ContentAnalyzer {
       let organizations = clean(doc.organizations().out('array'));
       // .dates() needs the compromise-dates plugin (not installed) and threw,
       // aborting ALL entity extraction; #Date+ tag matching is core compromise.
-      const dates = clean(doc.match('#Date+').out('array'));
+      // A duration ("weeks", "an hour") is not a date; a weekday is, in
+      // every language with a list — compromise knows only the English ones.
+      const dates = clean([...doc.match('#Date+').out('array'), ...this.findWeekdays(text, language)])
+        .filter(e => !DURATION_ONLY.test(e));
       const money = clean(doc.money().out('array'));
       let other = clean(doc.topics().out('array').slice(0, 10))
         .filter(e => !this.isSentenceInitialArtifact(e, text));
@@ -651,12 +700,13 @@ export class ContentAnalyzer {
       // Unicode classes, not [A-Z]: the ASCII version cut "Universität" to
       // "Universit" and "Wisłą" to "Wis" and dropped "Środkowej" outright (R16).
       // \b stays ASCII-only even under the u flag, hence the lookbehind.
-      const properNouns = (text.match(/(?<![\p{L}\p{N}])\p{Lu}[\p{L}.]+(?:\s+\p{Lu}[\p{L}.]+)*/gu) || [])
+      // German capitalises every noun, so a capital is no evidence there.
+      const properNouns = language === 'deu' ? [] : (text.match(/(?<![\p{L}\p{N}])\p{Lu}[\p{L}.]+(?:\s+\p{Lu}[\p{L}.]+)*/gu) || [])
         // Split matches that crossed a sentence boundary ("XPath. In") — a
         // period after a lowercase letter followed by a capital is a sentence
         // end, while abbreviations ("U.S. Holdings") stay intact.
         .flatMap(n => n.split(/(?<=\p{Ll}\.)\s+(?=\p{Lu})/u));
-      const supplemental = this.dedupeEntities(properNouns.map(n => this.cleanEntityText(n)))
+      const supplemental = this.dedupeEntities(properNouns.map(n => this.trimForeignEntity(this.cleanEntityText(n), language)))
         .filter(n => n.length > 1
           && !existingEntities.has(n.toLowerCase())
           && !this.isSentenceInitialArtifact(n, text))
@@ -778,9 +828,10 @@ export class ContentAnalyzer {
   /**
    * Calculate readability metrics
    * @param {string} text - Text to analyze
+   * @param {{ detectedLanguage?: string|null }} [options]
    * @returns {Promise<Object>} - Readability metrics
    */
-  async calculateReadability(text) {
+  async calculateReadability(text, options = {}) {
     try {
       const sentences = splitSentences(text);
       const isCjk = this.isCjkText(text);
@@ -824,6 +875,14 @@ export class ContentAnalyzer {
           metrics
         };
       }
+      // The constants are fitted to English; Spanish and German got English
+      // scores (sweep 2026-10-20). No other-language formula is implemented.
+      if (this.isNonEnglishText(text, options)) {
+        return {
+          notApplicable: 'flesch-reading-ease-is-english-only',
+          metrics
+        };
+      }
 
       // Flesch Reading Ease Score
       const fleschScore = 206.835 - (1.015 * avgWordsPerSentence) - (84.6 * avgSyllablesPerWord);
@@ -852,31 +911,27 @@ export class ContentAnalyzer {
       // negation ("not happy" scores negative). A 25-word list used to leave
       // clearly positive text "neutral" (R25).
       const { calculation, tokens } = sentimentAnalyzer.analyze(text);
-      // Drop hits that are function words of the detected language: German
-      // "die" is -3 in the English lexicon.
-      const foreignStopWords = LANGUAGE_STOP_WORDS[options.detectedLanguage];
-      const scores = calculation
-        .map(entry => Object.entries(entry)[0])
-        .filter(([word]) => !foreignStopWords?.has(word))
-        .map(([, score]) => score);
+      const scores = calculation.map(entry => Object.values(entry)[0]);
 
       const totalSentimentWords = scores.length;
 
+      // The lexicon is English. Its hits in Spanish or German text are
+      // accidents ("die" is -3), and finding none is not evidence of
+      // neutrality, so say the measure does not apply (R24, sweep 2026-10-20).
+      // Text franc may have misfiled — "I am so happy today" as Somali — is
+      // still scored when the lexicon matched.
+      if (this.isNonEnglishText(text, options)
+        || (totalSentimentWords === 0 && this.needsSegmentedTerms(text, options))) {
+        return {
+          polarity: 0,
+          subjectivity: 0,
+          label: 'not_applicable',
+          confidence: 0,
+          notApplicable: 'sentiment-lexicon-is-english-only'
+        };
+      }
+
       if (totalSentimentWords === 0) {
-        // The lexicon is English. Finding none of its words in German or
-        // Japanese text is not evidence of neutrality, so say the measure does
-        // not apply instead of reporting "neutral" (R24). Only when nothing
-        // matched: franc puts short English such as "I am so happy today" in
-        // Somali, and those strings must still be scored.
-        if (this.needsSegmentedTerms(text, options)) {
-          return {
-            polarity: 0,
-            subjectivity: 0,
-            label: 'not_applicable',
-            confidence: 0,
-            notApplicable: 'sentiment-lexicon-is-english-only'
-          };
-        }
         // No opinion words at all: neutral, but with no evidence behind it.
         return {
           polarity: 0,
@@ -1005,6 +1060,37 @@ export class ContentAnalyzer {
     return entity;
   }
 
+  /**
+   * Trim an entity found in non-English text: leading and trailing function
+   * words of the detected language ("Berlin den" → "Berlin", "über" → "")
+   * and leading titles ("Präsidenten Emmanuel Macron" → "Emmanuel Macron").
+   * Text with no stop-word list for its language is returned unchanged.
+   * @param {string} entity - Cleaned entity string
+   * @param {string|null} [language] - Detected ISO 639-3 code
+   * @returns {string}
+   */
+  trimForeignEntity(entity, language) {
+    const stopWords = LANGUAGE_STOP_WORDS[language];
+    if (!stopWords) return entity;
+    const tokens = entity.split(/\s+/);
+    while (tokens.length > 0 && (stopWords.has(tokens[0].toLowerCase()) || ENTITY_TITLES.has(tokens[0].toLowerCase()))) tokens.shift();
+    while (tokens.length > 0 && stopWords.has(tokens[tokens.length - 1].toLowerCase())) tokens.pop();
+    return tokens.join(' ');
+  }
+
+  /**
+   * Weekday names of the detected language, as written in the text.
+   * @param {string} text - Full source text
+   * @param {string|null} [language] - Detected ISO 639-3 code
+   * @returns {string[]}
+   */
+  findWeekdays(text, language) {
+    const names = WEEKDAYS[language];
+    if (!names) return [];
+    const pattern = new RegExp(`(?<![\\p{L}\\p{N}])(?:${names.join('|')})(?![\\p{L}\\p{N}])`, 'giu');
+    return text.match(pattern) || [];
+  }
+
   /**
    * Deduplicate entities case-insensitively, keeping the first casing seen
    * @param {string[]} entities - Entity strings
@@ -1156,6 +1242,48 @@ export class ContentAnalyzer {
  * way; every language whose function words are not on this list gets them back
  * as topics.
  */
+/**
+ * A bare duration, optionally counted: "weeks", "an hour", "two years". It
+ * measures time but names no date; compromise tags it #Date (sweep 2026-10-20).
+ */
+const DURATION_ONLY = /^(?:(?:a|an|one|two|three|four|five|six|seven|eight|nine|ten|few|several|many|\d+)\s+)?(?:seconds?|minutes?|hours?|days?|weeks?|fortnights?|months?|years?|decades?|centuries|century)$/i;
+
+/**
+ * Weekday names by ISO 639-3 code, for the languages with a stop-word list.
+ * English is left to compromise. Indonesian "minggu" (Sunday) is left out: it
+ * also means "week", and Portuguese bare "segunda" also means "second".
+ */
+const WEEKDAYS = {
+  deu: ['montag', 'dienstag', 'mittwoch', 'donnerstag', 'freitag', 'samstag', 'sonnabend', 'sonntag'],
+  spa: ['lunes', 'martes', 'miércoles', 'jueves', 'viernes', 'sábado', 'domingo'],
+  fra: ['lundi', 'mardi', 'mercredi', 'jeudi', 'vendredi', 'samedi', 'dimanche'],
+  ita: ['lunedì', 'martedì', 'mercoledì', 'giovedì', 'venerdì', 'sabato', 'domenica'],
+  por: ['segunda-feira', 'terça-feira', 'quarta-feira', 'quinta-feira', 'sexta-feira', 'sábado', 'domingo'],
+  nld: ['maandag', 'dinsdag', 'woensdag', 'donderdag', 'vrijdag', 'zaterdag', 'zondag'],
+  swe: ['måndag', 'tisdag', 'onsdag', 'torsdag', 'fredag', 'lördag', 'söndag'],
+  dan: ['mandag', 'tirsdag', 'onsdag', 'torsdag', 'fredag', 'lørdag', 'søndag'],
+  nob: ['mandag', 'tirsdag', 'onsdag', 'torsdag', 'fredag', 'lørdag', 'søndag'],
+  pol: ['poniedziałek', 'wtorek', 'środa', 'środę', 'czwartek', 'piątek', 'sobota', 'sobotę', 'niedziela', 'niedzielę'],
+  ind: ['senin', 'selasa', 'rabu', 'kamis', 'jumat', 'sabtu'],
+  zlm: ['isnin', 'selasa', 'rabu', 'khamis', 'jumaat', 'sabtu', 'ahad'],
+  zsm: ['isnin', 'selasa', 'rabu', 'khamis', 'jumaat', 'sabtu', 'ahad']
+};
+
+/**
+ * Titles that compromise folds into a following name in non-English text
+ * ("Präsidenten Emmanuel Macron"), lowercased. Stripped only from the front.
+ */
+const ENTITY_TITLES = new Set([
+  'präsident', 'präsidenten', 'präsidentin', 'bundespräsident', 'bundespräsidenten',
+  'bundeskanzler', 'bundeskanzlerin', 'kanzler', 'kanzlerin', 'minister', 'ministerin',
+  'ministerpräsident', 'ministerpräsidentin', 'herr', 'herrn', 'frau', 'könig', 'königin',
+  'presidente', 'presidenta', 'señor', 'señora', 'don', 'doña', 'ministro', 'ministra', 'rey', 'reina',
+  'président', 'présidente', 'monsieur', 'madame', 'ministre', 'roi', 'reine',
+  'signor', 'signore', 'signora', 'senhor', 'senhora',
+  'president', 'premier', 'meneer', 'mevrouw', 'koning', 'koningin',
+  'dr', 'dr.', 'prof', 'prof.', 'professor'
+]);
+
 const STOP_WORDS = new Set([
   // English
   'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by',