feedkeeper 0.26.0 → 0.26.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/items.js

@@ -5,6 +5,8 @@ import { z } from "zod";
 import { requireSession } from "../auth/middleware.js";
 import { listItemsForUser, markAllRead, markItemRead, markItemUnread, bookmarkItem, unbookmarkItem, findItemById, } from "../feeds/repository.js";
 import { loadFullText } from "../feeds/fullText.js";
+import { readerHtml } from "../feeds/articleContent.js";
+import { cleanSnippet } from "../feeds/contentNoise.js";
 import { fetchItemImage, pruneArchive, scheduleArchive, withArchivedImages, withProxiedImages } from "../feeds/archive.js";
 import { ImageCacheBusyError } from "../feeds/imageCache.js";
 export const FULL_TEXT_STATUS = {
@@ -46,6 +48,9 @@ itemsRouter.get("/", (req, res) => {
     const items = parsed.data.offline ? withProxiedImages(archived, origin) : archived;
     res.json(items.map((item) => ({
         ...item,
+        content_snippet: cleanSnippet(item.content_snippet, item.title),
+        content_html: readerHtml(item.content_html, item.title),
+        full_content_html: readerHtml(item.full_content_html, item.title),
         read: Boolean(item.read),
         bookmarked: Boolean(item.bookmarked),
     })));
@@ -121,7 +126,7 @@ itemsRouter.post("/:itemId/extract-content", async (req, res) => {
     const [served] = withArchivedImages([{ id: result.id, link: findItemById(result.id)?.link ?? null, full_content_html: result.fullContentHtml }], requestOrigin(req));
     res.json({
         id: result.id,
-        full_content_html: served.full_content_html,
+        full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title),
         ...(result.cached ? {} : { title: result.title, byline: result.byline }),
         cached: result.cached,
     });

package/server/dist/api/v1/index.js

@@ -25,6 +25,7 @@ const FEATURES = [
     "fulltext",
     "content.revisions",
     "content.prepare",
+    "item.lead",
     "images",
     "muted-keywords",
     "opml",

package/server/dist/api/v1/serialize.js

@@ -1,4 +1,6 @@
-import { contentMetadata } from "../../feeds/articleContent.js";
+import { contentMetadata, readerHtml } from "../../feeds/articleContent.js";
+import { cleanSnippet } from "../../feeds/contentNoise.js";
+import { leadOf } from "../../feeds/lead.js";
 import { db } from "../../db/index.js";
 import { serializeSubscription } from "../../sync/changeLog.js";
 /** A subscription as the list shows it: the synced fields plus counts and polling health. */
@@ -30,11 +32,12 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
         url: row.link,
         publishedAt: row.published_at,
         addedAt: row.created_at,
-        snippet: row.content_snippet,
+        snippet: cleanSnippet(row.content_snippet, row.title),
+        lead: leadOf(row),
         imageUrl: row.image_url,
         hasFullText: Boolean(row.full_content_html),
         content: contentMetadata(row),
-        ...(withContent ? { contentHtml: row.content_html, fullTextHtml: row.full_content_html } : {}),
+        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title), fullTextHtml: readerHtml(row.full_content_html, row.title) } : {}),
         state: {
             read: Boolean(row.read),
             saved: Boolean(row.bookmarked),

package/server/dist/db/migrations/0035_item_lead.sql

@@ -0,0 +1,4 @@
+-- Plain-text beginning of an article without page chrome, derived whenever its content changes. The version says
+-- which cleaning rules produced it, so a rule change is picked up by the background backfill.
+ALTER TABLE items ADD COLUMN content_lead TEXT;
+ALTER TABLE items ADD COLUMN lead_version INTEGER;

package/server/dist/feeds/articleContent.js

@@ -1,6 +1,11 @@
+import { createHash } from "node:crypto";
 import { JSDOM } from "jsdom";
 import { decodeEntities } from "./text.js";
-/** Plain publisher content, shared by readers, search consumers and MCP. */
+import { cleanReaderHtml, cleanSnippet, derivedText } from "./contentNoise.js";
+/**
+ * Plain publisher content, shared by readers, search consumers and MCP. Deliberately not cleaned: length decisions
+ * (which text is the better copy, what retention keeps) must not depend on which phrases the rules know.
+ */
 export function articleText(html) {
     if (!html)
         return "";
@@ -9,13 +14,32 @@ export function articleText(html) {
     fragment.querySelectorAll("p, div, br, li, h1, h2, h3, h4, blockquote").forEach(node => node.append(" "));
     return (fragment.textContent ?? "").replace(/\s+/g, " ").trim();
 }
+const READER_CACHE_LIMIT = 300;
+const readerCache = new Map();
+/**
+ * Article HTML as a reader shows it, without breadcrumbs, repeated headline and similar page chrome. Computed when
+ * served and never stored, so the publisher's original stays intact and rule changes apply to old articles too.
+ */
+export function readerHtml(html, title) {
+    if (!html)
+        return html ?? null;
+    const key = createHash("sha1").update(title ?? "").update("\0").update(html).digest("base64");
+    const cached = readerCache.get(key);
+    if (cached !== undefined)
+        return cached;
+    const cleaned = cleanReaderHtml(html, title);
+    if (readerCache.size >= READER_CACHE_LIMIT)
+        readerCache.delete(readerCache.keys().next().value);
+    readerCache.set(key, cleaned);
+    return cleaned;
+}
 export function contentMetadata(item) {
     const source = item.full_content_html ? "reader" : item.content_html ? "feed" : "snippet";
     return { revision: item.content_revision, source, status: item.extraction_status,
         attemptedAt: item.extraction_attempted_at, retryAt: item.extraction_retry_at };
 }
 export function sourceText(item) {
-    return `${decodeEntities(item.title) ?? ""}\n${articleText(item.full_content_html || item.content_html) || decodeEntities(item.content_snippet) || ""}`.trim();
+    return `${decodeEntities(item.title) ?? ""}\n${derivedText(item.full_content_html || item.content_html, item.title) || cleanSnippet(decodeEntities(item.content_snippet), item.title) || ""}`.trim();
 }
 /** Prefer existing substantive publisher text over a shorter extracted teaser. */
 export function improvesContent(item, extracted) {

package/server/dist/feeds/contentNoise.js

@@ -0,0 +1,419 @@
+import { JSDOM } from "jsdom";
+/**
+ * Publisher pages carry page chrome inside the article body: breadcrumbs, dateline repeats, player leftovers,
+ * recommendation lists, follow prompts. This module removes it in two layers that follow one principle:
+ *
+ * - `cleanReaderHtml` is what a reader shows. It removes only what cannot work in a reader or merely repeats what
+ *   the reader already shows (breadcrumbs, in-page jump links, a date line, the headline, consent placeholders,
+ *   player labels). Attribution, sponsored and affiliate content, disclosures and a publisher's own calls to action
+ *   stay: a source's revenue model is not ours to remove.
+ * - `derivedText` and `cleanSnippet` feed text that is not read as a page (snippets, search, teasers, curation).
+ *   There noise only costs relevance, so credits, ads, calls to action and recommendation lists go as well.
+ *
+ * Rules are structural or generic idioms. A phrase belongs here only when more than one publisher uses it.
+ */
+const BLOCKS = "p, li, h1, h2, h3, h4, h5, h6, blockquote, figcaption, dt, dd, td, th, pre";
+function flat(value) {
+    return (value ?? "").replace(/\s+/g, " ").trim();
+}
+function normalized(value) {
+    return value.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+function leafBlocks(root) {
+    return [...root.querySelectorAll(`${BLOCKS}, div`)].filter(element => !element.querySelector(BLOCKS) && (element.tagName !== "DIV" || !element.querySelector("div")));
+}
+function charsBefore(root, element) {
+    const range = root.ownerDocument.createRange();
+    range.setStart(root, 0);
+    range.setEndBefore(element);
+    return range.toString().length;
+}
+const BREADCRUMB_MARKERS = [
+    "[aria-label*='breadcrumb' i]",
+    "[itemtype*='BreadcrumbList' i]",
+    "[class~='breadcrumb']",
+    "[class~='breadcrumbs']",
+];
+const LANGUAGE_NAMES = ["English", "Deutsch", "Français", "Español", "Italiano", "Português", "Nederlands", "Polski", "Türkçe",
+    "Русский", "日本語", "한국어", "简体字", "繁體字", "简体中文", "繁體中文", "中文", "العربية"];
+const CONSENT_ANCHORS = [
+    /^Empfohlener redaktioneller Inhalt\b/i,
+    /An dieser Stelle findest du einen externen Inhalt/i,
+    /Du kannst ihn dir mit einem Klick anzeigen lassen/i,
+];
+const CONSENT_COMPANIONS = [
+    /\bInhalte? (von \S+ )?erlauben\b/i, /-Inhalte erlauben\b/i, /^Ich bin damit einverstanden, dass mir .+ angezeigt werden/i,
+    /Personenbezogene Daten können an Drittplattformen/i, /^Link zum .+-Inhalt$/i, /^Mehr dazu in unserer Datenschutzerklärung/i,
+];
+function removeReaderNoise(root, title, stats) {
+    let changed = false;
+    const drop = (element, rule) => {
+        if (!element.parentNode)
+            return;
+        element.remove();
+        stats[rule] = (stats[rule] ?? 0) + 1;
+        changed = true;
+    };
+    for (const selector of BREADCRUMB_MARKERS) {
+        try {
+            root.querySelectorAll(selector).forEach(element => drop(element, "breadcrumb"));
+        }
+        catch { /* selector unsupported by the DOM */ }
+    }
+    // Breadcrumbs without markup hints: a short list near the start whose items are single short links.
+    for (const list of root.querySelectorAll("ol, ul")) {
+        const items = [...list.children].filter(child => child.tagName === "LI");
+        if (items.length < 2 || items.length > 8 || charsBefore(root, list) > 300)
+            continue;
+        const onlyLinks = items.every(item => {
+            const links = item.querySelectorAll("a");
+            const label = flat(item.textContent);
+            return links.length === 1 && label.length > 0 && label.length <= 30 && flat(links[0].textContent) === label;
+        });
+        if (onlyLinks)
+            drop(list, "breadcrumb");
+    }
+    const headline = normalized(title ?? "");
+    const seen = new Set();
+    let leading = 0;
+    for (const block of leafBlocks(root)) {
+        const text = flat(block.textContent);
+        if (!text)
+            continue;
+        const position = leading++;
+        const heading = /^H[1-3]$/.test(block.tagName);
+        if (headline && normalized(text) === headline && (position < 4 || (heading && position < 14))) {
+            drop(block, "title-repeat");
+            continue;
+        }
+        if (position < 14 && /^(Audiobeitrag|Videobeitrag)\s*>/.test(text) && headline && normalized(text).endsWith(headline)) {
+            drop(block, "title-repeat");
+            continue;
+        }
+        if (/^[>›»|/·•–—-]+$/.test(text)) {
+            drop(block, "separator");
+            continue;
+        }
+        if (/^(Audiobeitrag|Videobeitrag|Autoplay)$/i.test(text) || (position < 8 && /^Bild$/.test(text))) {
+            drop(block, "player");
+            continue;
+        }
+        if (text.length <= 70 && block.querySelector("time") && charsBefore(root, block) <= 600) {
+            const rest = flat(text.replace(flat(block.querySelector("time")?.textContent), ""));
+            if (/^(stand|zuletzt aktualisiert|aktualisiert|ver(ö|oe)ffentlicht|updated|published|posted)?\s*[:·|•-]*\s*$/i.test(rest)) {
+                drop(block, "dateline");
+                continue;
+            }
+        }
+        const anchors = block.querySelectorAll("a");
+        if (text.length <= 60 && anchors.length === 1 && anchors[0].getAttribute("href")?.startsWith("#") && flat(anchors[0].textContent) === text) {
+            drop(block, "jump-link");
+            continue;
+        }
+        if (text.length <= 200 && /^\d{1,2}:\d{2}\b/.test(text) && /\bAutoplay\b/i.test(text)) {
+            drop(block, "player");
+            continue;
+        }
+        if (text.length <= 160 && LANGUAGE_NAMES.filter(name => text.includes(name)).length >= 4) {
+            drop(block, "language-switch");
+            continue;
+        }
+        if (position < 10 && text.length >= 30) {
+            const key = normalized(text);
+            if (seen.has(key)) {
+                drop(block, "repeat");
+                continue;
+            }
+            seen.add(key);
+        }
+    }
+    // Consent placeholders of embedded third-party content: only the neighbours of a known anchor phrase go.
+    const blocks = leafBlocks(root);
+    blocks.forEach((block, index) => {
+        if (!block.parentNode || flat(block.textContent).length > 400 || !CONSENT_ANCHORS.some(pattern => pattern.test(flat(block.textContent))))
+            return;
+        drop(block, "consent-placeholder");
+        for (const next of blocks.slice(index + 1, index + 7)) {
+            const text = flat(next.textContent);
+            if (!next.parentNode || text.length > 220)
+                break;
+            if (CONSENT_ANCHORS.some(pattern => pattern.test(text)) || CONSENT_COMPANIONS.some(pattern => pattern.test(text)))
+                drop(next, "consent-placeholder");
+        }
+    });
+    return changed;
+}
+const PRESENTATIONAL_TAGS = new Set(["FONT", "BIG", "SMALL", "CENTER", "TT", "U", "STRIKE", "S", "BASEFONT", "BLINK", "NOBR", "SPAN", "MARK"]);
+const PRESENTATIONAL_ATTRIBUTES = ["style", "class", "color", "face", "size", "bgcolor", "align", "valign", "border", "cellpadding",
+    "cellspacing", "background", "hspace", "vspace", "nowrap"];
+const SIZED_TAGS = new Set(["IMG", "VIDEO", "CANVAS", "IFRAME", "EMBED", "OBJECT"]);
+const EMPHASIS_EXEMPT = "h1, h2, h3, h4, h5, h6, th, strong, b";
+function wrapChildren(element, tag) {
+    const wrapper = element.ownerDocument.createElement(tag);
+    while (element.firstChild)
+        wrapper.append(element.firstChild);
+    element.append(wrapper);
+}
+/**
+ * Typography belongs to the client: fonts, sizes, colours, alignment and classes are removed so that only
+ * structure (headings, paragraphs, lists, quotes, tables, links, images) and emphasis (bold, italic) are left.
+ * Emphasis that was only a style (`font-weight: bold` on a span) is kept as strong or em.
+ */
+function removeTypography(root, stats) {
+    let changed = false;
+    root.querySelectorAll("style, link[rel~='stylesheet']").forEach(node => { node.remove(); changed = true; });
+    const targets = new Set([...root.querySelectorAll("a[href^='#']")].map(anchor => (anchor.getAttribute("href") ?? "").slice(1)));
+    for (const element of [...root.querySelectorAll("*")]) {
+        const style = element.getAttribute("style") ?? "";
+        const bold = /font-weight\s*:\s*(bold|bolder|[6-9]00)\b/i.test(style);
+        const italic = /font-style\s*:\s*(italic|oblique)\b/i.test(style);
+        const plain = !element.matches(EMPHASIS_EXEMPT) && !element.closest(EMPHASIS_EXEMPT) && flat(element.textContent) !== "";
+        for (const name of PRESENTATIONAL_ATTRIBUTES) {
+            if (element.hasAttribute(name)) {
+                element.removeAttribute(name);
+                changed = true;
+            }
+        }
+        if (!SIZED_TAGS.has(element.tagName)) {
+            for (const name of ["width", "height"]) {
+                if (element.hasAttribute(name)) {
+                    element.removeAttribute(name);
+                    changed = true;
+                }
+            }
+        }
+        if (plain && bold)
+            wrapChildren(element, "strong");
+        if (plain && italic && !element.closest("em, i"))
+            wrapChildren(element, "em");
+        const anchorTarget = element.id && targets.has(element.id);
+        if (PRESENTATIONAL_TAGS.has(element.tagName) && !anchorTarget) {
+            element.replaceWith(...element.childNodes);
+            changed = true;
+        }
+    }
+    if (changed)
+        stats["typography"] = (stats["typography"] ?? 0) + 1;
+    return changed;
+}
+function serialize(root) {
+    const holder = root.ownerDocument.createElement("div");
+    holder.append(root);
+    return holder.innerHTML;
+}
+/** The article HTML as a reader should show it. Unchanged input is returned as is. */
+export function cleanReaderHtml(html, title, stats = {}) {
+    if (!html)
+        return html;
+    const root = JSDOM.fragment(html);
+    const noise = removeReaderNoise(root, title, stats);
+    const typography = removeTypography(root, stats);
+    return noise || typography ? serialize(root) : html;
+}
+const CREDIT_PATTERNS = [
+    /^(©|\(c\)\s|copyright\b)/i,
+    /^[([]?(Bild|Foto|Fotos|Bildquelle|Bildrechte|Bildnachweis|Titelbild|Beitragsbild|Photo|Image|Credits?|Grafik|Illustration)[)\]]?\s*[:/|]/i,
+    /\|\s*(Image|Photo|Foto|Bild|Credit)s?\s*:/i,
+    /\(\s*(Bildquelle|Bildrechte|Foto|Bild|Photo|Image|Credit|Quelle)\s*:[^)]{1,60}\)\s*\.?$/i,
+    /\bBildrechte:\s*(Bildrechte:\s*)?[^.]{1,60}$/i,
+    /©[^\n]{1,40}$/,
+    /\/\s*(Getty Images|Reuters|AP|AFP|dpa|Imago)\s*$/i,
+    /\bvia (REUTERS|Reuters|AP|AFP|dpa|Getty Images|Imagen)\s*$/,
+    /^Quelle:?\s*\S{0,40}$/i,
+];
+const HEAD_META_PATTERNS = [
+    /^(Stand|Aktualisiert|Zuletzt aktualisiert|Ver(ö|oe)ffentlicht|Updated|Published|Posted)\b.{0,40}\d/i,
+    /^von\s+.{2,40}\s[I|]\s+\d{1,2}\.\d{1,2}\.\d{2,4}/i,
+    /^(\d{1,2}\.\s?\d{1,2}\.\s?\d{2,4}|[A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})([,\s•·|-]+\d{1,2}[:.]\d{2}( Uhr)?)?$/,
+    /^\p{L}[\p{L}\p{N} .&-]{0,30}\s(\d{1,2}\.\s?\d{1,2}\.\s?\d{2,4}|[A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})([,\s•·|-]+\d{1,2}[:.]\d{2}( Uhr)?)?$/u,
+    /^(Kommentare|Kommentieren|Comments|Teilen|Share|Drucken|Print|Merken|Vorlesen)$/i,
+    /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen)\s*:?\s+[^.!?]{0,160}$/i,
+];
+const AD_PATTERNS = [
+    /^(Anzeige|Werbung|Advertisement|Advertorial|Sponsored|Gesponsert|Promoted)\s*:?$/i,
+    /\b(affiliate|partner)[- ]?links?\b/i,
+    /\bwe (may|might) (receive|earn) a commission\b/i,
+    /^FTC:/i,
+    /\bClick Here to Buy\b/i,
+    /^Hurry,/i,
+    /^Worth checking out on Amazon/i,
+    /\bProvision\b.{0,60}\b(Links?|Einkäufe|Käufe)\b/i,
+];
+const SOCIAL_NETWORKS = ["Facebook", "Instagram", "Reddit", "Threads", "Twitter", "Mastodon", "YouTube", "TikTok", "Google News", "Flipboard", "WhatsApp", "LinkedIn"];
+const CALL_TO_ACTION_PATTERNS = [
+    /\b(bevorzugte[nr]?\s+(Google-)?Quelle|bei Google bevorzugen|preferred source)\b/i,
+    /\bWo du uns folgen kannst\b/i,
+    /^Folge[nt]? uns\b/i,
+    /\bFolge uns auch auf\b/i,
+    /\bUns auf Google folgen\b/i,
+    /\bfollow us\b/i,
+    /\bFügt uns doch bei Google\b/i,
+    /\bAuf dem Laufenden bleiben\?/i,
+    /\bNewsletter\b.{0,240}\b(Anmeldung|anmelden|abonnier\w*|subscribe)\b/i,
+    /\b(subscribe|abonnier\w+)\b.{0,60}\b(newsletter|YouTube|channel|Kanal)\b/i,
+    /\bBy subscribing\b/i,
+    /^SUBSCRIBE NOW$/i,
+    /^Hier geht'?s zur Anmeldung!?$/i,
+    /^★.+★$/,
+];
+const COMMENT_PATTERNS = [
+    /\b(let us know|tell us) .{0,40}in the comments\b/i,
+    /\bNur angemeldete Benutzer können kommentieren\b/i,
+];
+const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
+const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;
+const SECTION_LABEL = /^(Auch interessant|Das könnte (dich|Sie|euch) (auch )?interessieren|Mehr zum Thema|Lesen Sie auch|Lies auch|Lest auch|Aktuelle News|Weitere (Artikel|Meldungen|News)|Popular Stories|Related (Stories|Articles|Posts|Reading)|More (Stories|from)|Top Stories|Meistgelesen|Topaktuell|Regeln für Kommentare|Kommentare|Comments?)\b\s*:?/i;
+const MIN_KEPT_CHARACTERS = 150;
+const HEAD_BLOCKS = 6;
+const TAIL_CHARACTERS = 1500;
+const TAIL_CUT_FROM = 0.6;
+function matchesAny(text, patterns) {
+    return patterns.some(pattern => pattern.test(text));
+}
+function isSocialList(text) {
+    return text.length <= 200 && SOCIAL_NETWORKS.filter(name => text.includes(name)).length >= 3;
+}
+function isLinkOnly(block, text) {
+    const anchors = block.querySelectorAll("a");
+    return anchors.length === 1 && flat(anchors[0].textContent) === text;
+}
+/**
+ * Plain, single-line publisher text without page chrome: for snippets, search, teasers and curation, never for
+ * display as a page. Falls back to the plain flattening when cleaning would leave almost nothing.
+ */
+export function derivedText(html, title, stats = {}) {
+    if (!html)
+        return "";
+    const root = JSDOM.fragment(html);
+    root.querySelectorAll("script, style, noscript, nav, footer").forEach(node => node.remove());
+    const plain = () => {
+        root.querySelectorAll("p, div, br, li, h1, h2, h3, h4, blockquote").forEach(node => node.append(" "));
+        return flat(root.textContent);
+    };
+    removeReaderNoise(root, title, stats);
+    root.querySelectorAll("figcaption").forEach(node => node.remove());
+    const entries = leafBlocks(root).map(element => ({ element, text: flat(element.textContent) })).filter(entry => entry.text);
+    const total = entries.reduce((sum, entry) => sum + entry.text.length, 0);
+    if (entries.length === 0 || total < 2 * MIN_KEPT_CHARACTERS)
+        return plain();
+    const dropped = new Set();
+    const mark = (index, rule) => {
+        if (dropped.has(index))
+            return;
+        dropped.add(index);
+        stats[rule] = (stats[rule] ?? 0) + 1;
+    };
+    let before = 0;
+    let cutAt = -1;
+    entries.forEach((entry, index) => {
+        const { text } = entry;
+        const early = index < HEAD_BLOCKS || before <= 600;
+        const late = total - before - text.length <= TAIL_CHARACTERS || index >= entries.length - 8;
+        const position = before / total;
+        before += text.length;
+        if (cutAt >= 0)
+            return;
+        if (SECTION_LABEL.test(text) && text.length <= 400) {
+            if (position >= TAIL_CUT_FROM && before - text.length >= MIN_KEPT_CHARACTERS) {
+                cutAt = index;
+                stats["recommendations"] = (stats["recommendations"] ?? 0) + 1;
+                return;
+            }
+            mark(index, "recommendations");
+            for (let next = index + 1; next < entries.length && isLinkOnly(entries[next].element, entries[next].text); next++)
+                mark(next, "recommendations");
+            return;
+        }
+        if (TAG_LABEL.test(text)) {
+            mark(index, "head-meta");
+            for (let next = index + 1; next < entries.length && isLinkOnly(entries[next].element, entries[next].text); next++)
+                mark(next, "head-meta");
+            return;
+        }
+        if (text.length > 400)
+            return;
+        if (text.length <= 160 && VIDEO_CARD.test(text) && entry.element.querySelector("a"))
+            mark(index, "player");
+        else if (matchesAny(text, AD_PATTERNS))
+            mark(index, "advertising");
+        else if (text.length <= 300 && (early || late) && matchesAny(text, CALL_TO_ACTION_PATTERNS))
+            mark(index, "call-to-action");
+        else if (text.length <= 300 && late && matchesAny(text, COMMENT_PATTERNS))
+            mark(index, "comments");
+        else if (isSocialList(text))
+            mark(index, "call-to-action");
+        else if (early && text.length <= 160 && matchesAny(text, HEAD_META_PATTERNS))
+            mark(index, "head-meta");
+        else if ((early || late) && text.length <= 300 && matchesAny(text, CREDIT_PATTERNS)) {
+            mark(index, "credit");
+            // In the head a credit line follows the caption it belongs to.
+            const caption = entries[index - 1];
+            if (early && caption && index - 1 < HEAD_BLOCKS && caption.text.length <= 200 && /^(©|\(c\)|copyright|Bildrechte|Bildquelle|Foto|Photo|Image|Credit)/i.test(text))
+                mark(index - 1, "credit");
+        }
+    });
+    const kept = entries.filter((_, index) => !dropped.has(index) && (cutAt < 0 || index < cutAt)).map(entry => entry.text);
+    const text = kept.join(" ");
+    if (text.length < Math.min(MIN_KEPT_CHARACTERS, total) && text.length < total * 0.7)
+        return plain();
+    return text;
+}
+const SNIPPET_LINE_PATTERNS = [
+    /^(Der|Die|Das) Beitrag .+ (erschien|wurde) zuerst\b.*$/i,
+    /^The post .+ appeared first on\b.*$/i,
+    /^This article, ".+" first appeared on\b.*$/i,
+    /^Discuss this article in our forums\.?$/i,
+    /^Read (the )?full story( here| at .+)?\.?$/i,
+    /^Zum Beitrag:.*$/i,
+    /^(more…|more\.\.\.|Mehr|Weiterlesen|Read more)$/i,
+    /^(Related Roundup|Related Forum|Tags?|Buyer's Guide):.*$/i,
+    /^★.+★$/,
+    /^Wo du uns folgen kannst:.*$/i,
+    /^Auf dem Laufenden bleiben\?$/i,
+    /^Fügt uns doch bei Google.*$/i,
+];
+const SNIPPET_INLINE_PATTERNS = [
+    /\s*(Der|Die|Das) Beitrag [^\n]+? (erschien|wurde) zuerst (auf|bei|beim) [^\n]*?\.(?=\s|$)(\s*(Folge uns auch auf|Über unseren Newsletter)[^\n]*?\.(?=\s|$))*/gi,
+    /\s*Über unseren Newsletter [^\n]*?\.(?=\s|$)/gi,
+    /\s*The post [^\n]+? appeared first on [^\n]*?\.(?=\s|$)/gi,
+];
+/**
+ * A feed's summary without the footers feed plugins append ("The post … appeared first on …", "Read the full
+ * story at …"), without a repeated headline and with whitespace collapsed to paragraphs.
+ */
+export function cleanSnippet(snippet, title) {
+    if (snippet === null || snippet === undefined)
+        return null;
+    let text = snippet.replace(/\r\n?/g, "\n").replace(/[ \t ]+/g, " ");
+    for (const pattern of SNIPPET_INLINE_PATTERNS)
+        text = text.replace(pattern, "");
+    const headline = normalized(title ?? "");
+    const lines = text.split("\n").map(line => line.trim()).filter((line, index) => line && !SNIPPET_LINE_PATTERNS.some(pattern => pattern.test(line)) && !isSocialList(line) &&
+        !(index === 0 && headline && normalized(line) === headline));
+    return lines.join("\n").trim();
+}
+/** Raised whenever the rules above change what they remove, so stored leads are derived again. */
+export const LEAD_VERSION = 1;
+const LEAD_LENGTH = 600;
+function shorten(text, max) {
+    if (text.length <= max)
+        return text;
+    let slice = text.slice(0, max);
+    if (/[\uD800-\uDBFF]$/.test(slice))
+        slice = slice.slice(0, -1);
+    const sentence = Math.max(slice.lastIndexOf(". "), slice.lastIndexOf("! "), slice.lastIndexOf("? "));
+    if (sentence >= max * 0.5)
+        return slice.slice(0, sentence + 1);
+    const space = slice.lastIndexOf(" ");
+    return (space >= max * 0.5 ? slice.slice(0, space) : slice).replace(/[\s,;:–-]+$/, "") + "…";
+}
+/**
+ * The beginning of an article as one plain line, for teasers: the cleaned body, or the cleaned feed summary when
+ * that says more (a failed download leaves only the summary).
+ */
+export function leadText(parts, max = LEAD_LENGTH) {
+    const body = derivedText(parts.html, parts.title);
+    const summary = flat(cleanSnippet(parts.snippet, parts.title));
+    return shorten(body.length >= summary.length ? body : summary, max);
+}
+//# sourceMappingURL=contentNoise.js.map

package/server/dist/feeds/fullText.js

@@ -5,6 +5,7 @@ import { articleWorkPool, ArticleWorkBusyError, ArticleAccessDeniedError, Articl
 import { requestFullText, authorizedFullTextRequester } from "./fullTextQueue.js";
 import { SsrfBlockedError } from "./ssrfGuard.js";
 import { improvesContent } from "./articleContent.js";
+import { refreshLead } from "./lead.js";
 import { canAccessItem, findItemById, findSubscriptionFullTextMode, updateItemFullContent } from "./repository.js";
 // Kept for integrations importing the old constant; retries now belong to individual articles.
 export const FULL_TEXT_BLOCK_THRESHOLD = 2;
@@ -24,6 +25,7 @@ export function cachedFullText(userId, itemId, force = false) {
     if (item.full_content_html && isCachedConsentSnippet(item.full_content_html)) {
         // Repair cached walls before any caller can mistake them for reader content again.
         db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(itemId);
+        refreshLead(itemId);
         item.full_content_html = null;
     }
     if (!force && item.full_content_html && !isCachedConsentSnippet(item.full_content_html)) {

package/server/dist/feeds/inbox.js

@@ -1,3 +1,4 @@
+import { refreshLead } from "./lead.js";
 import { randomUUID } from "node:crypto";
 import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
@@ -102,6 +103,7 @@ export async function saveToInbox(userId, input) {
        RETURNING id`)
         .get(inbox.id, guid, title, targetUrl, snippet, contentHtml, extractedPartial ? null : contentHtml, imageUrl, now, now, contentHtml && !extractedPartial ? "ready" : extractionStatus, extractedPartial ? new Date(Date.now() + 86400_000).toISOString() : null);
     const itemId = row.id;
+    refreshLead(itemId);
     bookmarkItem(userId, itemId);
     let hasNote = false;
     if (input.note?.trim()) {

package/server/dist/feeds/lead.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { db } from "../db/index.js";
+import { LEAD_VERSION, leadText } from "./contentNoise.js";
+const BACKFILL_BATCH = 50;
+const BACKFILL_PAUSE_MS = 100;
+const COMPUTED_LIMIT = 300;
+const computed = new Map();
+export function computeLead(item) {
+    return leadText({ html: item.full_content_html || item.content_html, snippet: item.content_snippet, title: item.title });
+}
+/** Derive and store the lead of one item; call after its title, summary or content was written. */
+export function refreshLead(itemId) {
+    const item = db.prepare("SELECT title, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
+    if (!item)
+        return;
+    db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?").run(computeLead(item), LEAD_VERSION, itemId);
+}
+/**
+ * The lead to serve: the stored one while its rules are current, otherwise derived on the spot, so an article is
+ * never served without one just because the backfill has not reached it.
+ */
+export function leadOf(item) {
+    if (item.lead_version === LEAD_VERSION)
+        return item.content_lead || null;
+    const key = createHash("sha1").update(item.title ?? "").update("\0").update(item.content_snippet ?? "").update("\0")
+        .update(item.full_content_html || item.content_html || "").digest("base64");
+    let lead = computed.get(key);
+    if (lead === undefined) {
+        lead = computeLead(item);
+        if (computed.size >= COMPUTED_LIMIT)
+            computed.delete(computed.keys().next().value);
+        computed.set(key, lead);
+    }
+    return lead || null;
+}
+/** Derive leads for items that have none or were derived by older rules; returns how many were handled. */
+export function backfillLeads(limit = BACKFILL_BATCH) {
+    const rows = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
+    const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?");
+    db.transaction(() => { for (const row of rows)
+        update.run(computeLead(row), LEAD_VERSION, row.id); })();
+    return rows.length;
+}
+/** Works through the backlog in small steps so polling and requests are never held up for long. */
+export function startLeadBackfill() {
+    let total = 0;
+    const step = () => {
+        const handled = backfillLeads();
+        total += handled;
+        if (handled === BACKFILL_BATCH)
+            setTimeout(step, BACKFILL_PAUSE_MS).unref();
+        else if (total > 0)
+            console.log(`[db] derived the lead of ${total} stored items`);
+    };
+    setTimeout(step, 1000).unref();
+}
+//# sourceMappingURL=lead.js.map

package/server/dist/feeds/poller.js

@@ -88,6 +88,12 @@ function extractImageUrl(item) {
     }
     return null;
 }
+/** The article address: its link, or its guid when the feed gives only that and the guid is a web address. */
+export function itemLink(item) {
+    if (item.link)
+        return item.link;
+    return item.guid && /^https?:\/\//i.test(item.guid) ? item.guid : undefined;
+}
 /** One attempt; the queue owns retries, admission, single-flight and the total deadline. */
 async function pollAttempt(feed, signal) {
     const fetched = await fetchFeed(feed.url, { etag: feed.etag ?? undefined, lastModified: feed.last_modified ?? undefined, signal });
@@ -101,7 +107,7 @@ async function pollAttempt(feed, signal) {
         return { newItems: 0, error: "feed_poll_not_allowed" };
     const items = parsed?.items.map(item => ({
         guid: item.guid ?? item.link ?? item.title ?? crypto.randomUUID(),
-        title: plainTitle(item.title), link: item.link,
+        title: plainTitle(item.title), link: itemLink(item),
         contentSnippet: decodeEntities(item.contentSnippet ?? item.content) ?? undefined,
         contentHtml: item.contentEncoded ?? item["content:encoded"] ?? item.content ?? null,
         publishedAt: normalizePublishedAt(item.isoDate ?? item.pubDate) ?? undefined, imageUrl: extractImageUrl(item),

package/server/dist/feeds/repository.js

@@ -1,5 +1,7 @@
 import { db } from "../db/index.js";
 import { articleText } from "./articleContent.js";
+import { LEAD_VERSION } from "./contentNoise.js";
+import { refreshLead } from "./lead.js";
 import { decodeEntities } from "./text.js";
 export function findFeedByUrl(url) {
     return db.prepare("SELECT * FROM feeds WHERE url = ?").get(url);
@@ -160,14 +162,26 @@ export function upsertItems(feedId, items) {
             const result = insert.run(...values);
             if (result.changes > 0) {
                 inserted++;
+                refreshLead(Number(result.lastInsertRowid));
             }
             else {
-                const previous = db.prepare("SELECT content_html FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
+                const previous = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html, lead_version FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
                 const next = row.contentHtml;
                 // Some publishers replace complete feed bodies with short teasers on a later poll.
                 const retained = previous?.content_html && next && next.length < previous.content_html.length * 0.9 &&
                     articleText(next).length < articleText(previous.content_html).length * 0.9 ? previous.content_html : next ?? null;
-                update.run(row.title ?? null, row.link ?? null, row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
+                // The summary belongs to the body it was sent with: keeping the longer body keeps its summary too, or
+                // a feed that alternates between two variants would change the content on every poll.
+                const keptPrevious = retained !== null && next !== retained;
+                update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
+                // Polling rewrites every item; only a change of what the lead is made from needs it derived again.
+                if (previous) {
+                    const changed = (retained !== null && retained !== previous.content_html) ||
+                        (row.title !== undefined && row.title !== previous.title) ||
+                        (!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet);
+                    if (previous.lead_version !== LEAD_VERSION || (changed && !previous.full_content_html))
+                        refreshLead(previous.id);
+                }
             }
         }
         return inserted;
@@ -410,6 +424,7 @@ export function findItemById(id) {
 }
 export function updateItemFullContent(id, fullContentHtml) {
     db.prepare("UPDATE items SET full_content_html = ? WHERE id = ?").run(fullContentHtml, id);
+    refreshLead(id);
 }
 /** Apply a per-item change to many items atomically; returns how many rows changed. */
 export function applyToItems(itemIds, change) {

package/server/dist/mcp/items.js

@@ -1,4 +1,4 @@
-import { contentMetadata, sourceText } from "../feeds/articleContent.js";
+import { contentMetadata, readerHtml, sourceText } from "../feeds/articleContent.js";
 import { z } from "zod";
 import { findItemForUser, listItemsForUser } from "../feeds/repository.js";
 const cursorSchema = z.object({
@@ -51,7 +51,7 @@ export function getItemForMcp(userId, itemId, maxChars = MAX_CONTENT_LENGTH, off
         return null;
     if (revision !== undefined && revision !== item.content_revision)
         throw new Error("content_revision_conflict");
-    const content = item.full_content_html ?? item.content_html;
+    const content = readerHtml(item.full_content_html ?? item.content_html, item.title);
     const body = format === "text" ? sourceText(item) : content ?? "";
     let end = Math.min(offset + Math.min(maxChars, MAX_CONTENT_LENGTH), body.length);
     if (end < body.length && /[\uD800-\uDBFF]/.test(body[end - 1]))

package/server/dist/server.js

@@ -19,12 +19,14 @@ import { startCleanupScheduler } from "./feeds/cleanup.js";
 import { repairEncodedText } from "./feeds/repository.js";
 import { pruneArchive, scheduleMissingArchives } from "./feeds/archive.js";
 import { startEditionScheduler } from "./native/editions.js";
+import { startLeadBackfill } from "./feeds/lead.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const WEB_DIST = join(__dirname, "../../web/dist");
 runMigrations();
 const repairedItems = repairEncodedText();
 if (repairedItems > 0)
     console.log(`[db] decoded HTML entities in ${repairedItems} stored items`);
+startLeadBackfill();
 pruneArchive();
 const pendingArchives = scheduleMissingArchives();
 if (pendingArchives > 0)

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.26.0",
+  "version": "0.26.1",
   "private": true,
   "type": "module"
 }