feedkeeper 0.26.0 → 0.26.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.26.0",
+  "version": "0.26.2",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/items.js

@@ -5,6 +5,8 @@ import { z } from "zod";
 import { requireSession } from "../auth/middleware.js";
 import { listItemsForUser, markAllRead, markItemRead, markItemUnread, bookmarkItem, unbookmarkItem, findItemById, } from "../feeds/repository.js";
 import { loadFullText } from "../feeds/fullText.js";
+import { readerHtml } from "../feeds/articleContent.js";
+import { cleanSnippet } from "../feeds/contentNoise.js";
 import { fetchItemImage, pruneArchive, scheduleArchive, withArchivedImages, withProxiedImages } from "../feeds/archive.js";
 import { ImageCacheBusyError } from "../feeds/imageCache.js";
 export const FULL_TEXT_STATUS = {
@@ -44,8 +46,11 @@ itemsRouter.get("/", (req, res) => {
     const archived = withArchivedImages(listItemsForUser(req.user.id, parsed.data), origin);
     // Offline copies also get the images of articles that are not archived, through this server.
     const items = parsed.data.offline ? withProxiedImages(archived, origin) : archived;
-    res.json(items.map((item) => ({
+    res.json(items.map(({ content_lead: _lead, lead_version: _leadVersion, ...item }) => ({
         ...item,
+        content_snippet: cleanSnippet(item.content_snippet, item.title),
+        content_html: readerHtml(item.content_html, item.title),
+        full_content_html: readerHtml(item.full_content_html, item.title),
         read: Boolean(item.read),
         bookmarked: Boolean(item.bookmarked),
     })));
@@ -121,7 +126,7 @@ itemsRouter.post("/:itemId/extract-content", async (req, res) => {
     const [served] = withArchivedImages([{ id: result.id, link: findItemById(result.id)?.link ?? null, full_content_html: result.fullContentHtml }], requestOrigin(req));
     res.json({
         id: result.id,
-        full_content_html: served.full_content_html,
+        full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title),
         ...(result.cached ? {} : { title: result.title, byline: result.byline }),
         cached: result.cached,
     });

package/server/dist/api/v1/index.js

@@ -25,6 +25,7 @@ const FEATURES = [
     "fulltext",
     "content.revisions",
     "content.prepare",
+    "item.lead",
     "images",
     "muted-keywords",
     "opml",
@@ -33,7 +34,6 @@ const FEATURES = [
     "folder-icons",
     "notes",
     "edition",
-    "edition.automatic",
     "edition.revisions",
     "native-preferences",
     "native-preferences.edition-size",

package/server/dist/api/v1/resources.js

@@ -18,7 +18,7 @@ import { SsrfBlockedError, normalizeUrlCandidate } from "../../feeds/ssrfGuard.j
 import { buildOverview } from "../../mcp/digest.js";
 import { decodeCursor, encodeCursor } from "../../mcp/items.js";
 import { readingPositions, serializeItem, serializeSubscriptionDetail } from "./serialize.js";
-import { dismissEdition, editionCandidates, generateEdition, generateEditionSchema, getEdition, getEditionState } from "../../native/editions.js";
+import { dismissEdition, editionCandidates, ensureEditionSchema, getEdition, getEditionState } from "../../native/editions.js";
 import { getNativePreferences, patchNativePreferences } from "../../native/preferences.js";
 import { NativeError } from "../../native/requests.js";
 import { capabilityDenial, hasUserCapability, requireCapability } from "../../auth/capabilities.js";
@@ -520,7 +520,7 @@ function editionResponse(req, edition) {
         .filter((row) => Boolean(row));
     const progress = readingPositions(req.user.id, rows.map((r) => r.id));
     const hasNotes = itemsWithNotes(req.user.id, rows.map((r) => r.id));
-    const prepared = prepare(rows, req, "original");
+    const prepared = prepare(rows, req, req.query.images === "proxy" ? "proxy" : "original");
     return {
         ...edition,
         items: prepared.map((row) => serializeItem(row, progress, false, hasNotes.has(row.id))),
@@ -563,14 +563,19 @@ resourcesRouter.get("/edition/candidates", requireCapability("editions"), (req,
     res.setHeader("Cache-Control", "private, no-store");
     res.json({ preferences: getNativePreferences(req.user.id), current: getEditionState(req.user.id), candidates: editionCandidates(req.user.id, new Date(), parsed.data.limit) });
 });
+// Older clients may still ensure an edition before reading. Keep that request read-only.
 resourcesRouter.post("/edition/generate", requireCapability("editions"), (req, res) => {
-    const parsed = generateEditionSchema.safeParse(req.body);
+    const parsed = ensureEditionSchema.safeParse(req.body);
     if (!parsed.success)
         return invalid(res, parsed.error.flatten());
     nativeAction(res, () => {
-        const edition = generateEdition(req.user.id, parsed.data);
+        const current = getEditionState(req.user.id);
+        if (parsed.data.expectedRevision !== undefined && parsed.data.expectedRevision !== current.revision) {
+            throw new NativeError("revision_conflict", 409, current);
+        }
+        const edition = getEdition(req.user.id);
         if (!edition)
-            return void res.status(404).json({ error: "no_edition", current: getEditionState(req.user.id) });
+            return void res.status(404).json({ error: "no_edition", current });
         res.json(editionResponse(req, edition));
     });
 });

package/server/dist/api/v1/serialize.js

@@ -1,4 +1,6 @@
-import { contentMetadata } from "../../feeds/articleContent.js";
+import { contentMetadata, readerHtml } from "../../feeds/articleContent.js";
+import { cleanSnippet } from "../../feeds/contentNoise.js";
+import { leadOf } from "../../feeds/lead.js";
 import { db } from "../../db/index.js";
 import { serializeSubscription } from "../../sync/changeLog.js";
 /** A subscription as the list shows it: the synced fields plus counts and polling health. */
@@ -30,11 +32,12 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
         url: row.link,
         publishedAt: row.published_at,
         addedAt: row.created_at,
-        snippet: row.content_snippet,
+        snippet: cleanSnippet(row.content_snippet, row.title),
+        lead: leadOf(row),
         imageUrl: row.image_url,
         hasFullText: Boolean(row.full_content_html),
         content: contentMetadata(row),
-        ...(withContent ? { contentHtml: row.content_html, fullTextHtml: row.full_content_html } : {}),
+        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title), fullTextHtml: readerHtml(row.full_content_html, row.title) } : {}),
         state: {
             read: Boolean(row.read),
             saved: Boolean(row.bookmarked),

package/server/dist/db/migrations/0035_item_lead.sql

@@ -0,0 +1,4 @@
+-- Plain-text beginning of an article without page chrome, derived whenever its content changes. The version says
+-- which cleaning rules produced it, so a rule change is picked up by the background backfill.
+ALTER TABLE items ADD COLUMN content_lead TEXT;
+ALTER TABLE items ADD COLUMN lead_version INTEGER;

package/server/dist/db/migrations/0036_curated_only_editions.sql

@@ -0,0 +1,5 @@
+-- Retire legacy automatic slots without dismissing the current curation period.
+-- Keep the revision watermark so a late publication cannot silently overwrite newer state.
+UPDATE editions SET status = 'expired', revision = revision + 1,
+  updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')
+WHERE source = 'automatic' AND status = 'active';

package/server/dist/feeds/articleContent.js

@@ -1,6 +1,11 @@
+import { createHash } from "node:crypto";
 import { JSDOM } from "jsdom";
 import { decodeEntities } from "./text.js";
-/** Plain publisher content, shared by readers, search consumers and MCP. */
+import { cleanReaderHtml, cleanSnippet, derivedText } from "./contentNoise.js";
+/**
+ * Plain publisher content, shared by readers, search consumers and MCP. Deliberately not cleaned: length decisions
+ * (which text is the better copy, what retention keeps) must not depend on which phrases the rules know.
+ */
 export function articleText(html) {
     if (!html)
         return "";
@@ -9,13 +14,32 @@ export function articleText(html) {
     fragment.querySelectorAll("p, div, br, li, h1, h2, h3, h4, blockquote").forEach(node => node.append(" "));
     return (fragment.textContent ?? "").replace(/\s+/g, " ").trim();
 }
+const READER_CACHE_LIMIT = 300;
+const readerCache = new Map();
+/**
+ * Article HTML as a reader shows it, without breadcrumbs, repeated headline and similar page chrome. Computed when
+ * served and never stored, so the publisher's original stays intact and rule changes apply to old articles too.
+ */
+export function readerHtml(html, title) {
+    if (!html)
+        return html ?? null;
+    const key = createHash("sha1").update(title ?? "").update("\0").update(html).digest("base64");
+    const cached = readerCache.get(key);
+    if (cached !== undefined)
+        return cached;
+    const cleaned = cleanReaderHtml(html, title);
+    if (readerCache.size >= READER_CACHE_LIMIT)
+        readerCache.delete(readerCache.keys().next().value);
+    readerCache.set(key, cleaned);
+    return cleaned;
+}
 export function contentMetadata(item) {
     const source = item.full_content_html ? "reader" : item.content_html ? "feed" : "snippet";
     return { revision: item.content_revision, source, status: item.extraction_status,
         attemptedAt: item.extraction_attempted_at, retryAt: item.extraction_retry_at };
 }
 export function sourceText(item) {
-    return `${decodeEntities(item.title) ?? ""}\n${articleText(item.full_content_html || item.content_html) || decodeEntities(item.content_snippet) || ""}`.trim();
+    return `${decodeEntities(item.title) ?? ""}\n${derivedText(item.full_content_html || item.content_html, item.title) || cleanSnippet(decodeEntities(item.content_snippet), item.title) || ""}`.trim();
 }
 /** Prefer existing substantive publisher text over a shorter extracted teaser. */
 export function improvesContent(item, extracted) {

package/server/dist/feeds/cleanup.js

@@ -92,7 +92,7 @@ export function runCleanup(custom) {
            AND i.id NOT IN (SELECT item_id FROM item_notes)
            AND i.feed_id NOT IN (SELECT id FROM feeds WHERE is_system_inbox = 1)
            AND NOT EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected
-             WHERE selected.value = i.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now'))
+             WHERE selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed')
            AND i.id IN (SELECT item_id FROM item_reads)
            AND NOT EXISTS (
              SELECT 1 FROM subscriptions s
@@ -110,7 +110,7 @@ export function runCleanup(custom) {
          AND id NOT IN (SELECT item_id FROM item_notes)
          AND feed_id NOT IN (SELECT id FROM feeds WHERE is_system_inbox = 1)
          AND NOT EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected
-           WHERE selected.value = items.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now'))
+           WHERE selected.value = items.id AND e.source = 'curated' AND e.status <> 'dismissed')
          AND COALESCE(julianday(published_at), julianday(created_at)) <= julianday('now', '-' || ? || ' days')`).run(settings.retentionMaxDays);
         deletedOldItems = res.changes;
     }
@@ -121,7 +121,7 @@ export function runCleanup(custom) {
          AND id NOT IN (SELECT item_id FROM item_notes)
          AND feed_id NOT IN (SELECT id FROM feeds WHERE is_system_inbox = 1)
          AND NOT EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected
-           WHERE selected.value = items.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now'))
+           WHERE selected.value = items.id AND e.source = 'curated' AND e.status <> 'dismissed')
          AND id IN (
            SELECT id FROM (
              SELECT id, ROW_NUMBER() OVER (

package/server/dist/feeds/contentNoise.js

@@ -0,0 +1,441 @@
+import { JSDOM } from "jsdom";
+/**
+ * Publisher pages carry page chrome inside the article body: breadcrumbs, dateline repeats, player leftovers,
+ * recommendation lists, follow prompts. This module removes it in two layers that follow one principle:
+ *
+ * - `cleanReaderHtml` is what a reader shows. It removes only what cannot work in a reader or merely repeats what
+ *   the reader already shows (breadcrumbs, in-page jump links, a date line, the headline, consent placeholders,
+ *   player labels). Attribution, sponsored and affiliate content, disclosures and a publisher's own calls to action
+ *   stay: a source's revenue model is not ours to remove.
+ * - `derivedText` and `cleanSnippet` feed text that is not read as a page (snippets, search, teasers, curation).
+ *   There noise only costs relevance, so credits, ads, calls to action and recommendation lists go as well.
+ *
+ * Rules are structural or generic idioms. A phrase belongs here only when more than one publisher uses it.
+ */
+const BLOCKS = "p, li, h1, h2, h3, h4, h5, h6, blockquote, figcaption, dt, dd, td, th, pre";
+function flat(value) {
+    return (value ?? "").replace(/\s+/g, " ").trim();
+}
+function normalized(value) {
+    return value.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+function leafBlocks(root) {
+    return [...root.querySelectorAll(`${BLOCKS}, div`)].filter(element => !element.querySelector(BLOCKS) && (element.tagName !== "DIV" || !element.querySelector("div")));
+}
+function charsBefore(root, element) {
+    const range = root.ownerDocument.createRange();
+    range.setStart(root, 0);
+    range.setEndBefore(element);
+    return range.toString().length;
+}
+const BREADCRUMB_MARKERS = [
+    "[aria-label*='breadcrumb' i]",
+    "[itemtype*='BreadcrumbList' i]",
+    "[class~='breadcrumb']",
+    "[class~='breadcrumbs']",
+];
+const LANGUAGE_NAMES = ["English", "Deutsch", "Français", "Español", "Italiano", "Português", "Nederlands", "Polski", "Türkçe",
+    "Русский", "日本語", "한국어", "简体字", "繁體字", "简体中文", "繁體中文", "中文", "العربية"];
+const CONSENT_ANCHORS = [
+    /^Empfohlener redaktioneller Inhalt\b/i,
+    /An dieser Stelle findest du einen externen Inhalt/i,
+    /Du kannst ihn dir mit einem Klick anzeigen lassen/i,
+];
+const CONSENT_COMPANIONS = [
+    /\bInhalte? (von \S+ )?erlauben\b/i, /-Inhalte erlauben\b/i, /^Ich bin damit einverstanden, dass mir .+ angezeigt werden/i,
+    /Personenbezogene Daten können an Drittplattformen/i, /^Link zum .+-Inhalt$/i, /^Mehr dazu in unserer Datenschutzerklärung/i,
+];
+function removeReaderNoise(root, title, stats) {
+    let changed = false;
+    const drop = (element, rule) => {
+        if (!element.parentNode)
+            return;
+        element.remove();
+        stats[rule] = (stats[rule] ?? 0) + 1;
+        changed = true;
+    };
+    for (const selector of BREADCRUMB_MARKERS) {
+        try {
+            root.querySelectorAll(selector).forEach(element => drop(element, "breadcrumb"));
+        }
+        catch { /* selector unsupported by the DOM */ }
+    }
+    // Breadcrumbs without markup hints: a short list near the start whose items are single short links.
+    for (const list of root.querySelectorAll("ol, ul")) {
+        const items = [...list.children].filter(child => child.tagName === "LI");
+        if (items.length < 2 || items.length > 8 || charsBefore(root, list) > 300)
+            continue;
+        const onlyLinks = items.every(item => {
+            const links = item.querySelectorAll("a");
+            const label = flat(item.textContent);
+            return links.length === 1 && label.length > 0 && label.length <= 30 && flat(links[0].textContent) === label;
+        });
+        if (onlyLinks)
+            drop(list, "breadcrumb");
+    }
+    const headline = normalized(title ?? "");
+    const seen = new Set();
+    let leading = 0;
+    for (const block of leafBlocks(root)) {
+        const text = flat(block.textContent);
+        if (!text)
+            continue;
+        const position = leading++;
+        const heading = /^H[1-3]$/.test(block.tagName);
+        if (headline && normalized(text) === headline && (position < 4 || (heading && position < 14))) {
+            drop(block, "title-repeat");
+            continue;
+        }
+        if (position < 14 && /^(Audiobeitrag|Videobeitrag)\s*>/.test(text) && headline && normalized(text).endsWith(headline)) {
+            drop(block, "title-repeat");
+            continue;
+        }
+        if (/^[>›»|/·•–—-]+$/.test(text)) {
+            drop(block, "separator");
+            continue;
+        }
+        if (/^(Audiobeitrag|Videobeitrag|Autoplay)$/i.test(text) || (position < 8 && /^Bild$/.test(text))) {
+            drop(block, "player");
+            continue;
+        }
+        if (text.length <= 70 && block.querySelector("time") && charsBefore(root, block) <= 600) {
+            const rest = flat(text.replace(flat(block.querySelector("time")?.textContent), ""));
+            if (/^(stand|zuletzt aktualisiert|aktualisiert|ver(ö|oe)ffentlicht|updated|published|posted)?\s*[:·|•-]*\s*$/i.test(rest)) {
+                drop(block, "dateline");
+                continue;
+            }
+        }
+        const anchors = block.querySelectorAll("a");
+        if (text.length <= 60 && anchors.length === 1 && anchors[0].getAttribute("href")?.startsWith("#") && flat(anchors[0].textContent) === text) {
+            drop(block, "jump-link");
+            continue;
+        }
+        if (text.length <= 200 && /^\d{1,2}:\d{2}\b/.test(text) && /\bAutoplay\b/i.test(text)) {
+            drop(block, "player");
+            continue;
+        }
+        if (text.length <= 160 && LANGUAGE_NAMES.filter(name => text.includes(name)).length >= 4) {
+            drop(block, "language-switch");
+            continue;
+        }
+        if (position < 10 && text.length >= 30) {
+            const key = normalized(text);
+            if (seen.has(key)) {
+                drop(block, "repeat");
+                continue;
+            }
+            seen.add(key);
+        }
+    }
+    // Consent placeholders of embedded third-party content: only the neighbours of a known anchor phrase go.
+    const blocks = leafBlocks(root);
+    blocks.forEach((block, index) => {
+        if (!block.parentNode || flat(block.textContent).length > 400 || !CONSENT_ANCHORS.some(pattern => pattern.test(flat(block.textContent))))
+            return;
+        drop(block, "consent-placeholder");
+        for (const next of blocks.slice(index + 1, index + 7)) {
+            const text = flat(next.textContent);
+            if (!next.parentNode || text.length > 220)
+                break;
+            if (CONSENT_ANCHORS.some(pattern => pattern.test(text)) || CONSENT_COMPANIONS.some(pattern => pattern.test(text)))
+                drop(next, "consent-placeholder");
+        }
+    });
+    return changed;
+}
+const PRESENTATIONAL_TAGS = new Set(["FONT", "BIG", "SMALL", "CENTER", "TT", "U", "STRIKE", "S", "BASEFONT", "BLINK", "NOBR", "SPAN", "MARK"]);
+const PRESENTATIONAL_ATTRIBUTES = ["style", "class", "color", "face", "size", "bgcolor", "align", "valign", "border", "cellpadding",
+    "cellspacing", "background", "hspace", "vspace", "nowrap"];
+const SIZED_TAGS = new Set(["IMG", "VIDEO", "CANVAS", "IFRAME", "EMBED", "OBJECT"]);
+// Tracking and scripting hooks no client uses. Lazy-load sources stay: they may be an image's only address.
+const LAZY_SOURCES = new Set(["data-src", "data-srcset", "data-lazy-src", "data-original"]);
+const INERT_ATTRIBUTES = new Set(["role", "tabindex", "itemprop", "itemscope", "itemtype", "itemid", "draggable", "contenteditable"]);
+function isInertAttribute(name) {
+    if (LAZY_SOURCES.has(name))
+        return false;
+    return name.startsWith("data-") || name.startsWith("aria-") || name.startsWith("on") || INERT_ATTRIBUTES.has(name);
+}
+const EMPHASIS_EXEMPT = "h1, h2, h3, h4, h5, h6, th, strong, b";
+function wrapChildren(element, tag) {
+    const wrapper = element.ownerDocument.createElement(tag);
+    while (element.firstChild)
+        wrapper.append(element.firstChild);
+    element.append(wrapper);
+}
+/**
+ * Typography belongs to the client: fonts, sizes, colours, alignment and classes are removed so that only
+ * structure (headings, paragraphs, lists, quotes, tables, links, images) and emphasis (bold, italic) are left.
+ * Emphasis that was only a style (`font-weight: bold` on a span) is kept as strong or em.
+ */
+function removeTypography(root, stats) {
+    let changed = false;
+    root.querySelectorAll("style, link[rel~='stylesheet']").forEach(node => { node.remove(); changed = true; });
+    const targets = new Set([...root.querySelectorAll("a[href^='#']")].map(anchor => (anchor.getAttribute("href") ?? "").slice(1)));
+    for (const element of [...root.querySelectorAll("*")]) {
+        const style = element.getAttribute("style") ?? "";
+        const bold = /font-weight\s*:\s*(bold|bolder|[6-9]00)\b/i.test(style);
+        const italic = /font-style\s*:\s*(italic|oblique)\b/i.test(style);
+        const plain = !element.matches(EMPHASIS_EXEMPT) && !element.closest(EMPHASIS_EXEMPT) && flat(element.textContent) !== "";
+        for (const name of PRESENTATIONAL_ATTRIBUTES) {
+            if (element.hasAttribute(name)) {
+                element.removeAttribute(name);
+                changed = true;
+            }
+        }
+        for (const name of element.getAttributeNames()) {
+            if (isInertAttribute(name)) {
+                element.removeAttribute(name);
+                changed = true;
+            }
+        }
+        if (!SIZED_TAGS.has(element.tagName)) {
+            for (const name of ["width", "height"]) {
+                if (element.hasAttribute(name)) {
+                    element.removeAttribute(name);
+                    changed = true;
+                }
+            }
+        }
+        if (plain && bold)
+            wrapChildren(element, "strong");
+        if (plain && italic && !element.closest("em, i"))
+            wrapChildren(element, "em");
+        const anchorTarget = element.id && targets.has(element.id);
+        if (element.id && !anchorTarget) {
+            element.removeAttribute("id");
+            changed = true;
+        }
+        if (element.tagName === "A" && element.hasAttribute("name") && !targets.has(element.getAttribute("name") ?? "")) {
+            element.removeAttribute("name");
+            changed = true;
+        }
+        if (PRESENTATIONAL_TAGS.has(element.tagName) && !anchorTarget) {
+            element.replaceWith(...element.childNodes);
+            changed = true;
+        }
+    }
+    if (changed)
+        stats["typography"] = (stats["typography"] ?? 0) + 1;
+    return changed;
+}
+function serialize(root) {
+    const holder = root.ownerDocument.createElement("div");
+    holder.append(root);
+    return holder.innerHTML;
+}
+/** The article HTML as a reader should show it. Unchanged input is returned as is. */
+export function cleanReaderHtml(html, title, stats = {}) {
+    if (!html)
+        return html;
+    const root = JSDOM.fragment(html);
+    const noise = removeReaderNoise(root, title, stats);
+    const typography = removeTypography(root, stats);
+    return noise || typography ? serialize(root) : html;
+}
+const CREDIT_PATTERNS = [
+    /^(©|\(c\)\s|copyright\b)/i,
+    /^[([]?(Bild|Foto|Fotos|Bildquelle|Bildrechte|Bildnachweis|Titelbild|Beitragsbild|Photo|Image|Credits?|Grafik|Illustration)[)\]]?\s*[:/|]/i,
+    /\|\s*(Image|Photo|Foto|Bild|Credit)s?\s*:/i,
+    /\(\s*(Bildquelle|Bildrechte|Foto|Bild|Photo|Image|Credit|Quelle)\s*:[^)]{1,60}\)\s*\.?$/i,
+    /\bBildrechte:\s*(Bildrechte:\s*)?[^.]{1,60}$/i,
+    /©[^\n]{1,40}$/,
+    /\/\s*(Getty Images|Reuters|AP|AFP|dpa|Imago)\s*$/i,
+    /\bvia (REUTERS|Reuters|AP|AFP|dpa|Getty Images|Imagen)\s*$/,
+    /^Quelle:?\s*\S{0,40}$/i,
+];
+const HEAD_META_PATTERNS = [
+    /^(Stand|Aktualisiert|Zuletzt aktualisiert|Ver(ö|oe)ffentlicht|Updated|Published|Posted)\b.{0,40}\d/i,
+    /^von\s+.{2,40}\s[I|]\s+\d{1,2}\.\d{1,2}\.\d{2,4}/i,
+    /^(\d{1,2}\.\s?\d{1,2}\.\s?\d{2,4}|[A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})([,\s•·|-]+\d{1,2}[:.]\d{2}( Uhr)?)?$/,
+    /^\p{L}[\p{L}\p{N} .&-]{0,30}\s(\d{1,2}\.\s?\d{1,2}\.\s?\d{2,4}|[A-Z][a-z]{2,8}\.? \d{1,2}, \d{4})([,\s•·|-]+\d{1,2}[:.]\d{2}( Uhr)?)?$/u,
+    /^(Kommentare|Kommentieren|Comments|Teilen|Share|Drucken|Print|Merken|Vorlesen)$/i,
+    /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen)\s*:?\s+[^.!?]{0,160}$/i,
+];
+const AD_PATTERNS = [
+    /^(Anzeige|Werbung|Advertisement|Advertorial|Sponsored|Gesponsert|Promoted)\s*:?$/i,
+    /\b(affiliate|partner)[- ]?links?\b/i,
+    /\bwe (may|might) (receive|earn) a commission\b/i,
+    /^FTC:/i,
+    /\bClick Here to Buy\b/i,
+    /^Hurry,/i,
+    /^Worth checking out on Amazon/i,
+    /\bProvision\b.{0,60}\b(Links?|Einkäufe|Käufe)\b/i,
+];
+const SOCIAL_NETWORKS = ["Facebook", "Instagram", "Reddit", "Threads", "Twitter", "Mastodon", "YouTube", "TikTok", "Google News", "Flipboard", "WhatsApp", "LinkedIn"];
+const CALL_TO_ACTION_PATTERNS = [
+    /\b(bevorzugte[nr]?\s+(Google-)?Quelle|bei Google bevorzugen|preferred source)\b/i,
+    /\bWo du uns folgen kannst\b/i,
+    /^Folge[nt]? uns\b/i,
+    /\bFolge uns auch auf\b/i,
+    /\bUns auf Google folgen\b/i,
+    /\bfollow us\b/i,
+    /\bFügt uns doch bei Google\b/i,
+    /\bAuf dem Laufenden bleiben\?/i,
+    /\bNewsletter\b.{0,240}\b(Anmeldung|anmelden|abonnier\w*|subscribe)\b/i,
+    /\b(subscribe|abonnier\w+)\b.{0,60}\b(newsletter|YouTube|channel|Kanal)\b/i,
+    /\bBy subscribing\b/i,
+    /^SUBSCRIBE NOW$/i,
+    /^Hier geht'?s zur Anmeldung!?$/i,
+    /^★.+★$/,
+];
+const COMMENT_PATTERNS = [
+    /\b(let us know|tell us) .{0,40}in the comments\b/i,
+    /\bNur angemeldete Benutzer können kommentieren\b/i,
+];
+const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
+const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;
+const SECTION_LABEL = /^(Auch interessant|Das könnte (dich|Sie|euch) (auch )?interessieren|Mehr zum Thema|Lesen Sie auch|Lies auch|Lest auch|Aktuelle News|Weitere (Artikel|Meldungen|News)|Popular Stories|Related (Stories|Articles|Posts|Reading)|More (Stories|from)|Top Stories|Meistgelesen|Topaktuell|Regeln für Kommentare|Kommentare|Comments?)\b\s*:?/i;
+const MIN_KEPT_CHARACTERS = 150;
+const HEAD_BLOCKS = 6;
+const TAIL_CHARACTERS = 1500;
+const TAIL_CUT_FROM = 0.6;
+function matchesAny(text, patterns) {
+    return patterns.some(pattern => pattern.test(text));
+}
+function isSocialList(text) {
+    return text.length <= 200 && SOCIAL_NETWORKS.filter(name => text.includes(name)).length >= 3;
+}
+function isLinkOnly(block, text) {
+    const anchors = block.querySelectorAll("a");
+    return anchors.length === 1 && flat(anchors[0].textContent) === text;
+}
+/**
+ * Plain, single-line publisher text without page chrome: for snippets, search, teasers and curation, never for
+ * display as a page. Falls back to the plain flattening when cleaning would leave almost nothing.
+ */
+export function derivedText(html, title, stats = {}) {
+    if (!html)
+        return "";
+    const root = JSDOM.fragment(html);
+    root.querySelectorAll("script, style, noscript, nav, footer").forEach(node => node.remove());
+    const plain = () => {
+        root.querySelectorAll("p, div, br, li, h1, h2, h3, h4, blockquote").forEach(node => node.append(" "));
+        return flat(root.textContent);
+    };
+    removeReaderNoise(root, title, stats);
+    root.querySelectorAll("figcaption").forEach(node => node.remove());
+    const entries = leafBlocks(root).map(element => ({ element, text: flat(element.textContent) })).filter(entry => entry.text);
+    const total = entries.reduce((sum, entry) => sum + entry.text.length, 0);
+    if (entries.length === 0 || total < 2 * MIN_KEPT_CHARACTERS)
+        return plain();
+    const dropped = new Set();
+    const mark = (index, rule) => {
+        if (dropped.has(index))
+            return;
+        dropped.add(index);
+        stats[rule] = (stats[rule] ?? 0) + 1;
+    };
+    let before = 0;
+    let cutAt = -1;
+    entries.forEach((entry, index) => {
+        const { text } = entry;
+        const early = index < HEAD_BLOCKS || before <= 600;
+        const late = total - before - text.length <= TAIL_CHARACTERS || index >= entries.length - 8;
+        const position = before / total;
+        before += text.length;
+        if (cutAt >= 0)
+            return;
+        if (SECTION_LABEL.test(text) && text.length <= 400) {
+            if (position >= TAIL_CUT_FROM && before - text.length >= MIN_KEPT_CHARACTERS) {
+                cutAt = index;
+                stats["recommendations"] = (stats["recommendations"] ?? 0) + 1;
+                return;
+            }
+            mark(index, "recommendations");
+            for (let next = index + 1; next < entries.length && isLinkOnly(entries[next].element, entries[next].text); next++)
+                mark(next, "recommendations");
+            return;
+        }
+        if (TAG_LABEL.test(text)) {
+            mark(index, "head-meta");
+            for (let next = index + 1; next < entries.length && isLinkOnly(entries[next].element, entries[next].text); next++)
+                mark(next, "head-meta");
+            return;
+        }
+        if (text.length > 400)
+            return;
+        if (text.length <= 160 && VIDEO_CARD.test(text) && entry.element.querySelector("a"))
+            mark(index, "player");
+        else if (matchesAny(text, AD_PATTERNS))
+            mark(index, "advertising");
+        else if (text.length <= 300 && (early || late) && matchesAny(text, CALL_TO_ACTION_PATTERNS))
+            mark(index, "call-to-action");
+        else if (text.length <= 300 && late && matchesAny(text, COMMENT_PATTERNS))
+            mark(index, "comments");
+        else if (isSocialList(text))
+            mark(index, "call-to-action");
+        else if (early && text.length <= 160 && matchesAny(text, HEAD_META_PATTERNS))
+            mark(index, "head-meta");
+        else if ((early || late) && text.length <= 300 && matchesAny(text, CREDIT_PATTERNS)) {
+            mark(index, "credit");
+            // In the head a credit line follows the caption it belongs to.
+            const caption = entries[index - 1];
+            if (early && caption && index - 1 < HEAD_BLOCKS && caption.text.length <= 200 && /^(©|\(c\)|copyright|Bildrechte|Bildquelle|Foto|Photo|Image|Credit)/i.test(text))
+                mark(index - 1, "credit");
+        }
+    });
+    const kept = entries.filter((_, index) => !dropped.has(index) && (cutAt < 0 || index < cutAt)).map(entry => entry.text);
+    const text = kept.join(" ");
+    if (text.length < Math.min(MIN_KEPT_CHARACTERS, total) && text.length < total * 0.7)
+        return plain();
+    return text;
+}
+const SNIPPET_LINE_PATTERNS = [
+    /^(Der|Die|Das) Beitrag .+ (erschien|wurde) zuerst\b.*$/i,
+    /^The post .+ appeared first on\b.*$/i,
+    /^This article, ".+" first appeared on\b.*$/i,
+    /^Discuss this article in our forums\.?$/i,
+    /^Read (the )?full story( here| at .+)?\.?$/i,
+    /^Zum Beitrag:.*$/i,
+    /^(more…|more\.\.\.|Mehr|Weiterlesen|Read more)$/i,
+    /^(Related Roundup|Related Forum|Tags?|Buyer's Guide):.*$/i,
+    /^★.+★$/,
+    /^Wo du uns folgen kannst:.*$/i,
+    /^Auf dem Laufenden bleiben\?$/i,
+    /^Fügt uns doch bei Google.*$/i,
+];
+const SNIPPET_INLINE_PATTERNS = [
+    /\s*(Der|Die|Das) Beitrag [^\n]+? (erschien|wurde) zuerst (auf|bei|beim) [^\n]*?\.(?=\s|$)(\s*(Folge uns auch auf|Über unseren Newsletter)[^\n]*?\.(?=\s|$))*/gi,
+    /\s*Über unseren Newsletter [^\n]*?\.(?=\s|$)/gi,
+    /\s*The post [^\n]+? appeared first on [^\n]*?\.(?=\s|$)/gi,
+];
+/**
+ * A feed's summary without the footers feed plugins append ("The post … appeared first on …", "Read the full
+ * story at …"), without a repeated headline and with whitespace collapsed to paragraphs.
+ */
+export function cleanSnippet(snippet, title) {
+    if (snippet === null || snippet === undefined)
+        return null;
+    let text = snippet.replace(/\r\n?/g, "\n").replace(/[ \t\u00a0]+/g, " ");
+    for (const pattern of SNIPPET_INLINE_PATTERNS)
+        text = text.replace(pattern, "");
+    const headline = normalized(title ?? "");
+    const lines = text.split("\n").map(line => line.trim()).filter((line, index) => line && !SNIPPET_LINE_PATTERNS.some(pattern => pattern.test(line)) && !isSocialList(line) &&
+        !(index === 0 && headline && normalized(line) === headline));
+    return lines.join("\n").trim();
+}
+/** Raised whenever the rules above change what they remove, so stored leads are derived again. */
+export const LEAD_VERSION = 1;
+const LEAD_LENGTH = 600;
+function shorten(text, max) {
+    if (text.length <= max)
+        return text;
+    let slice = text.slice(0, max);
+    if (/[\uD800-\uDBFF]$/.test(slice))
+        slice = slice.slice(0, -1);
+    const sentence = Math.max(slice.lastIndexOf(". "), slice.lastIndexOf("! "), slice.lastIndexOf("? "));
+    if (sentence >= max * 0.5)
+        return slice.slice(0, sentence + 1);
+    const space = slice.lastIndexOf(" ");
+    return (space >= max * 0.5 ? slice.slice(0, space) : slice).replace(/[\s,;:–-]+$/, "") + "…";
+}
+/**
+ * The beginning of an article as one plain line, for teasers: the cleaned body, or the cleaned feed summary when
+ * that says more (a failed download leaves only the summary).
+ */
+export function leadText(parts, max = LEAD_LENGTH) {
+    const body = derivedText(parts.html, parts.title);
+    const summary = flat(cleanSnippet(parts.snippet, parts.title));
+    return shorten(body.length >= summary.length ? body : summary, max);
+}
+//# sourceMappingURL=contentNoise.js.map

package/server/dist/feeds/fullText.js

@@ -5,6 +5,7 @@ import { articleWorkPool, ArticleWorkBusyError, ArticleAccessDeniedError, Articl
 import { requestFullText, authorizedFullTextRequester } from "./fullTextQueue.js";
 import { SsrfBlockedError } from "./ssrfGuard.js";
 import { improvesContent } from "./articleContent.js";
+import { refreshLead } from "./lead.js";
 import { canAccessItem, findItemById, findSubscriptionFullTextMode, updateItemFullContent } from "./repository.js";
 // Kept for integrations importing the old constant; retries now belong to individual articles.
 export const FULL_TEXT_BLOCK_THRESHOLD = 2;
@@ -24,6 +25,7 @@ export function cachedFullText(userId, itemId, force = false) {
     if (item.full_content_html && isCachedConsentSnippet(item.full_content_html)) {
         // Repair cached walls before any caller can mistake them for reader content again.
         db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(itemId);
+        refreshLead(itemId);
         item.full_content_html = null;
     }
     if (!force && item.full_content_html && !isCachedConsentSnippet(item.full_content_html)) {

package/server/dist/feeds/inbox.js

@@ -1,3 +1,4 @@
+import { refreshLead } from "./lead.js";
 import { randomUUID } from "node:crypto";
 import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
@@ -102,6 +103,7 @@ export async function saveToInbox(userId, input) {
        RETURNING id`)
         .get(inbox.id, guid, title, targetUrl, snippet, contentHtml, extractedPartial ? null : contentHtml, imageUrl, now, now, contentHtml && !extractedPartial ? "ready" : extractionStatus, extractedPartial ? new Date(Date.now() + 86400_000).toISOString() : null);
     const itemId = row.id;
+    refreshLead(itemId);
     bookmarkItem(userId, itemId);
     let hasNote = false;
     if (input.note?.trim()) {

package/server/dist/feeds/lead.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { db } from "../db/index.js";
+import { LEAD_VERSION, leadText } from "./contentNoise.js";
+const BACKFILL_BATCH = 50;
+const BACKFILL_PAUSE_MS = 100;
+const COMPUTED_LIMIT = 300;
+const computed = new Map();
+export function computeLead(item) {
+    return leadText({ html: item.full_content_html || item.content_html, snippet: item.content_snippet, title: item.title });
+}
+/** Derive and store the lead of one item; call after its title, summary or content was written. */
+export function refreshLead(itemId) {
+    const item = db.prepare("SELECT title, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
+    if (!item)
+        return;
+    db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?").run(computeLead(item), LEAD_VERSION, itemId);
+}
+/**
+ * The lead to serve: the stored one while its rules are current, otherwise derived on the spot, so an article is
+ * never served without one just because the backfill has not reached it.
+ */
+export function leadOf(item) {
+    if (item.lead_version === LEAD_VERSION)
+        return item.content_lead || null;
+    const key = createHash("sha1").update(item.title ?? "").update("\0").update(item.content_snippet ?? "").update("\0")
+        .update(item.full_content_html || item.content_html || "").digest("base64");
+    let lead = computed.get(key);
+    if (lead === undefined) {
+        lead = computeLead(item);
+        if (computed.size >= COMPUTED_LIMIT)
+            computed.delete(computed.keys().next().value);
+        computed.set(key, lead);
+    }
+    return lead || null;
+}
+/** Derive leads for items that have none or were derived by older rules; returns how many were handled. */
+export function backfillLeads(limit = BACKFILL_BATCH) {
+    const rows = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
+    const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?");
+    db.transaction(() => { for (const row of rows)
+        update.run(computeLead(row), LEAD_VERSION, row.id); })();
+    return rows.length;
+}
+/** Works through the backlog in small steps so polling and requests are never held up for long. */
+export function startLeadBackfill() {
+    let total = 0;
+    const step = () => {
+        const handled = backfillLeads();
+        total += handled;
+        if (handled === BACKFILL_BATCH)
+            setTimeout(step, BACKFILL_PAUSE_MS).unref();
+        else if (total > 0)
+            console.log(`[db] derived the lead of ${total} stored items`);
+    };
+    setTimeout(step, 1000).unref();
+}
+//# sourceMappingURL=lead.js.map

package/server/dist/feeds/poller.js

@@ -88,6 +88,12 @@ function extractImageUrl(item) {
     }
     return null;
 }
+/** The article address: its link, or its guid when the feed gives only that and the guid is a web address. */
+export function itemLink(item) {
+    if (item.link)
+        return item.link;
+    return item.guid && /^https?:\/\//i.test(item.guid) ? item.guid : undefined;
+}
 /** One attempt; the queue owns retries, admission, single-flight and the total deadline. */
 async function pollAttempt(feed, signal) {
     const fetched = await fetchFeed(feed.url, { etag: feed.etag ?? undefined, lastModified: feed.last_modified ?? undefined, signal });
@@ -101,7 +107,7 @@ async function pollAttempt(feed, signal) {
         return { newItems: 0, error: "feed_poll_not_allowed" };
     const items = parsed?.items.map(item => ({
         guid: item.guid ?? item.link ?? item.title ?? crypto.randomUUID(),
-        title: plainTitle(item.title), link: item.link,
+        title: plainTitle(item.title), link: itemLink(item),
         contentSnippet: decodeEntities(item.contentSnippet ?? item.content) ?? undefined,
         contentHtml: item.contentEncoded ?? item["content:encoded"] ?? item.content ?? null,
         publishedAt: normalizePublishedAt(item.isoDate ?? item.pubDate) ?? undefined, imageUrl: extractImageUrl(item),

package/server/dist/feeds/repository.js

@@ -1,5 +1,7 @@
 import { db } from "../db/index.js";
 import { articleText } from "./articleContent.js";
+import { LEAD_VERSION } from "./contentNoise.js";
+import { refreshLead } from "./lead.js";
 import { decodeEntities } from "./text.js";
 export function findFeedByUrl(url) {
     return db.prepare("SELECT * FROM feeds WHERE url = ?").get(url);
@@ -41,7 +43,7 @@ export function removeFeedIfUnused(feedId) {
         return;
     db.prepare(`DELETE FROM items WHERE feed_id = ? AND id NOT IN (SELECT item_id FROM item_bookmarks) AND id NOT IN (SELECT item_id FROM item_notes)
       AND NOT EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected
-        WHERE selected.value = items.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now'))`).run(feedId);
+        WHERE selected.value = items.id AND e.source = 'curated' AND e.status <> 'dismissed')`).run(feedId);
     db.prepare("DELETE FROM feeds WHERE id = ? AND is_system_inbox = 0 AND NOT EXISTS (SELECT 1 FROM items WHERE feed_id = ?)").run(feedId, feedId);
 }
 /** Feeds kept only for saved articles or annotated articles whose last bookmark and note are gone. */
@@ -52,7 +54,7 @@ export function removeUnusedFeeds() {
        AND NOT EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = feeds.id)
        AND NOT EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = feeds.id)
        AND NOT EXISTS (SELECT 1 FROM items i, editions e, json_each(e.item_ids) selected
-         WHERE i.feed_id = feeds.id AND selected.value = i.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now'))`).run().changes;
+         WHERE i.feed_id = feeds.id AND selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed')`).run().changes;
 }
 /** Subscribers can open every item of their feeds; anyone can open what they saved or added a note to. */
 export function canAccessItem(userId, itemId) {
@@ -62,7 +64,7 @@ export function canAccessItem(userId, itemId) {
            OR EXISTS (SELECT 1 FROM item_bookmarks b WHERE b.item_id = i.id AND b.user_id = ?)
            OR EXISTS (SELECT 1 FROM item_notes n WHERE n.item_id = i.id AND n.user_id = ?)
            OR EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected WHERE e.user_id = ?
-             AND selected.value = i.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now')))`).get(itemId, userId, userId, userId, userId));
+             AND selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed'))`).get(itemId, userId, userId, userId, userId));
 }
 export function listSubscriptionsForUser(userId) {
     return db
@@ -160,14 +162,26 @@ export function upsertItems(feedId, items) {
             const result = insert.run(...values);
             if (result.changes > 0) {
                 inserted++;
+                refreshLead(Number(result.lastInsertRowid));
             }
             else {
-                const previous = db.prepare("SELECT content_html FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
+                const previous = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html, lead_version FROM items WHERE feed_id = ? AND guid = ?").get(feedId, row.guid);
                 const next = row.contentHtml;
                 // Some publishers replace complete feed bodies with short teasers on a later poll.
                 const retained = previous?.content_html && next && next.length < previous.content_html.length * 0.9 &&
                     articleText(next).length < articleText(previous.content_html).length * 0.9 ? previous.content_html : next ?? null;
-                update.run(row.title ?? null, row.link ?? null, row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
+                // The summary belongs to the body it was sent with: keeping the longer body keeps its summary too, or
+                // a feed that alternates between two variants would change the content on every poll.
+                const keptPrevious = retained !== null && next !== retained;
+                update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
+                // Polling rewrites every item; only a change of what the lead is made from needs it derived again.
+                if (previous) {
+                    const changed = (retained !== null && retained !== previous.content_html) ||
+                        (row.title !== undefined && row.title !== previous.title) ||
+                        (!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet);
+                    if (previous.lead_version !== LEAD_VERSION || (changed && !previous.full_content_html))
+                        refreshLead(previous.id);
+                }
             }
         }
         return inserted;
@@ -278,7 +292,7 @@ export function findItemForUser(userId, itemId) {
      LEFT JOIN item_notes n ON n.item_id = i.id AND n.user_id = ?
      WHERE i.id = ? AND (s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.item_id IS NOT NULL OR
        EXISTS (SELECT 1 FROM editions e, json_each(e.item_ids) selected WHERE e.user_id = ?
-         AND selected.value = i.id AND e.status = 'active' AND julianday(e.expires_at) > julianday('now')))`).get(userId, userId, userId, userId, itemId, userId);
+         AND selected.value = i.id AND e.source = 'curated' AND e.status <> 'dismissed'))`).get(userId, userId, userId, userId, itemId, userId);
 }
 export function updateSubscriptionLabel(userId, feedId, label) {
     db.prepare("UPDATE subscriptions SET label = ? WHERE user_id = ? AND feed_id = ?").run(label?.trim() || null, userId, feedId);
@@ -410,6 +424,7 @@ export function findItemById(id) {
 }
 export function updateItemFullContent(id, fullContentHtml) {
     db.prepare("UPDATE items SET full_content_html = ? WHERE id = ?").run(fullContentHtml, id);
+    refreshLead(id);
 }
 /** Apply a per-item change to many items atomically; returns how many rows changed. */
 export function applyToItems(itemIds, change) {

package/server/dist/mcp/items.js

@@ -1,4 +1,4 @@
-import { contentMetadata, sourceText } from "../feeds/articleContent.js";
+import { contentMetadata, readerHtml, sourceText } from "../feeds/articleContent.js";
 import { z } from "zod";
 import { findItemForUser, listItemsForUser } from "../feeds/repository.js";
 const cursorSchema = z.object({
@@ -37,7 +37,7 @@ export function listItemsPage(userId, options) {
     });
     const hasMore = rows.length > limit;
     const page = rows.slice(0, limit);
-    const items = page.map(({ content_html: _contentHtml, full_content_html: _fullContentHtml, ...item }) => item);
+    const items = page.map(({ content_html: _contentHtml, full_content_html: _fullContentHtml, content_lead: _lead, lead_version: _leadVersion, ...item }) => item);
     return {
         items,
         newestCursor: page[0] ? encodeCursor(page[0]) : null,
@@ -51,15 +51,16 @@ export function getItemForMcp(userId, itemId, maxChars = MAX_CONTENT_LENGTH, off
         return null;
     if (revision !== undefined && revision !== item.content_revision)
         throw new Error("content_revision_conflict");
-    const content = item.full_content_html ?? item.content_html;
+    const content = readerHtml(item.full_content_html ?? item.content_html, item.title);
     const body = format === "text" ? sourceText(item) : content ?? "";
     let end = Math.min(offset + Math.min(maxChars, MAX_CONTENT_LENGTH), body.length);
     if (end < body.length && /[\uD800-\uDBFF]/.test(body[end - 1]))
         end--;
     if (end === offset && offset < body.length)
         end = Math.min(offset + 2, body.length);
+    const { content_lead: _lead, lead_version: _leadVersion, ...visible } = item;
     return {
-        ...item,
+        ...visible,
         content_html: format === "html" ? body.slice(offset, end) : undefined,
         content_truncated: end < body.length,
         full_content_html: undefined,

package/server/dist/mcp/promptsAndResources.js

@@ -29,7 +29,7 @@ export function registerPromptsAndResources(server, userId, scope) {
     server.registerResource("feeds", "feedkeeper://feeds", { title: "Subscribed feeds", description: "All subscriptions with unread counts and health.", mimeType: "application/json" }, (uri) => ({ contents: [{ uri: uri.href, mimeType: "application/json", text: JSON.stringify(listSubscriptionsForUser(userId), null, 2) }] }));
     server.registerResource("opml", "feedkeeper://opml", { title: "Subscriptions as OPML", description: "The subscription list in OPML 2.0, for backup or migration.", mimeType: "text/x-opml" }, (uri) => ({ contents: [{ uri: uri.href, mimeType: "text/x-opml", text: generateOpml(userId) }] }));
     server.registerResource("saved", "feedkeeper://saved", { title: "Saved articles", description: "The newest 100 saved articles, without their content.", mimeType: "application/json" }, (uri) => {
-        const items = listItemsForUser(userId, { bookmarkedOnly: true, limit: 100 }).map(({ content_html: _html, full_content_html: _full, ...item }) => item);
+        const items = listItemsForUser(userId, { bookmarkedOnly: true, limit: 100 }).map(({ content_html: _html, full_content_html: _full, content_lead: _lead, lead_version: _leadVersion, ...item }) => item);
         return { contents: [{ uri: uri.href, mimeType: "application/json", text: JSON.stringify(items, null, 2) }] };
     });
     server.registerResource("digest", "feedkeeper://digest", { title: "Unread digest", description: "Unread articles of the last 24 hours, grouped by feed.", mimeType: "application/json" }, (uri) => ({ contents: [{ uri: uri.href, mimeType: "application/json", text: JSON.stringify(buildDigest(userId, { hours: 24 }), null, 2) }] }));

package/server/dist/mcp/server.js

@@ -15,7 +15,7 @@ import { discoverFeeds } from "../feeds/discovery.js";
 import { findUserById } from "../auth/users.js";
 import { getDatabaseStats, runCleanup } from "../feeds/cleanup.js";
 import { APP_VERSION } from "../version.js";
-import { dismissEdition, editionCandidates, generateEdition, generateEditionSchema, getEditionState, publishCuratedEdition, publishEditionFields } from "../native/editions.js";
+import { dismissEdition, editionCandidates, ensureEditionSchema, getEdition, getEditionState, publishCuratedEdition, publishEditionFields } from "../native/editions.js";
 import { getNativePreferences, patchNativePreferences, preferencePatchSchema } from "../native/preferences.js";
 import { NativeError } from "../native/requests.js";
 import { capabilityDenial, hasUserCapability } from "../auth/capabilities.js";
@@ -254,7 +254,7 @@ export function createMcpServerForUser(userId, scope) {
         },
     }, async ({ feedId, folderId, search, bookmarkedOnly, since, until, limit, includeContent, snippetChars }) => textResult(listItemsForUser(userId, {
         feedId, folderId, search, bookmarkedOnly, limit, unreadOnly: true, publishedSince: since, publishedUntil: until,
-    }).map(({ content_html, full_content_html, ...item }) => ({
+    }).map(({ content_html, full_content_html, content_lead: _lead, lead_version: _leadVersion, ...item }) => ({
         ...item,
         ...(includeContent === false ? {} : { content_html, full_content_html }),
         content_snippet: snippetChars ? truncate(item.content_snippet, snippetChars) : item.content_snippet,
@@ -500,19 +500,23 @@ export function createMcpServerForUser(userId, scope) {
     if (scope === "write")
         server.registerTool("publish_edition", {
             title: "Publish curated edition",
-            description: "Publishes an ordered list of 1 to 24 accessible article IDs. Curated editions override automatic ones until expiry (default 24 hours, maximum seven days). App-only hidden sections and muted keywords are respected. Supply the revision from get_edition as expectedRevision and a UUID requestId for conflict detection and safe retries. Optional title and summary are editorial text; original articles stay unchanged. Optional layout (version 1) says how the issue is set: a character (dominant, calm or briefing) and blocks (lead, dossier, feature, standard, briefs, photo, quote) that reference the issue's article IDs at most once each; it never changes the articles.",
+            description: "Publishes an ordered list of 1 to 24 accessible article IDs. The latest curated edition stays readable until another is published; expiry (default 24 hours, maximum seven days) controls when a new publication is due. App-only hidden sections and muted keywords are respected. Supply the revision from get_edition as expectedRevision and a UUID requestId for conflict detection and safe retries. Optional title and summary are editorial text; original articles stay unchanged. Optional layout (version 1) says how the issue is set: a character (dominant, calm or briefing) and blocks (lead, dossier, feature, standard, briefs, photo, quote) that reference the issue's article IDs at most once each; it never changes the articles.",
             inputSchema: publishEditionFields,
         }, async (input) => capabilityAction("editions", () => publishCuratedEdition(userId, input)));
     if (scope === "write")
         server.registerTool("generate_edition", {
-            title: "Generate an automatic edition",
-            description: "Ensures an automatic edition exists without disturbing a valid current issue. force=true requests a new automatic selection and requires expectedRevision. A valid curated edition must be explicitly dismissed first. Use a stable UUID requestId for retries.",
-            inputSchema: generateEditionSchema.shape,
+            title: "Get the latest curated edition",
+            description: "Compatibility alias for reading the latest curated publication. It never selects articles or publishes an edition. force is accepted for older clients but does not create a replacement; use publish_edition to publish a new curated selection.",
+            inputSchema: ensureEditionSchema.shape,
         }, async (input) => capabilityAction("editions", () => {
-            const parsed = generateEditionSchema.safeParse(input);
+            const parsed = ensureEditionSchema.safeParse(input);
             if (!parsed.success)
                 throw new NativeError("invalid_input");
-            return { edition: generateEdition(userId, parsed.data), current: getEditionState(userId) };
+            const current = getEditionState(userId);
+            if (parsed.data.expectedRevision !== undefined && parsed.data.expectedRevision !== current.revision) {
+                throw new NativeError("revision_conflict", 409, current);
+            }
+            return { edition: getEdition(userId), current };
         }));
     if (scope === "write")
         server.registerTool("dismiss_edition", {

package/server/dist/native/editions.js

@@ -1,11 +1,9 @@
-import { hasUserCapability } from "../auth/capabilities.js";
 import { randomUUID } from "node:crypto";
-import cron from "node-cron";
 import { z } from "zod";
 import { db } from "../db/index.js";
 import { canAccessItem } from "../feeds/repository.js";
 import { decodeEntities } from "../feeds/text.js";
-import { editionPeriod, nextEditionBoundary, selectEdition } from "./editionSelection.js";
+import { editionPeriod, selectEdition } from "./editionSelection.js";
 import { getNativePreferences } from "./preferences.js";
 import { checkEditionLayout, editionLayoutSchema } from "./editionLayout.js";
 import { NativeError, nativeRequest } from "./requests.js";
@@ -22,9 +20,9 @@ function storedEdition(userId) {
 function serialize(row) {
     const prefs = getNativePreferences(row.user_id);
     return {
-        id: row.edition_id, revision: row.revision, source: row.source,
+        id: row.edition_id, revision: row.revision, source: "curated",
         createdAt: row.created_at, updatedAt: row.updated_at, expiresAt: row.expires_at,
-        period: row.period?.split(":")[1] ?? editionPeriod(new Date(row.created_at), prefs.timeZone).period,
+        period: row.period?.split(":").pop() ?? editionPeriod(new Date(row.created_at), prefs.timeZone).period,
         title: row.title, summary: row.summary, overview: row.overview ? JSON.parse(row.overview) : null, layout: row.layout ? JSON.parse(row.layout) : null,
         preferencesRevision: row.preferences_revision,
         itemIds: JSON.parse(row.item_ids),
@@ -34,11 +32,15 @@ export function getEditionState(userId, now = new Date()) {
     const row = storedEdition(userId);
     if (!row)
         return { revision: 0, status: "none", edition: null };
+    if (row.source !== "curated")
+        return { revision: row.revision, status: "expired", edition: null };
     const status = row.status === "active" && Date.parse(row.expires_at) <= now.getTime() ? "expired" : row.status;
     return { revision: row.revision, status, edition: status === "active" ? serialize(row) : null };
 }
-export function getEdition(userId, now = new Date()) {
-    return getEditionState(userId, now).edition;
+/** Expiry schedules the next publication; it does not take the last issue away from readers. */
+export function getEdition(userId, _now = new Date()) {
+    const row = storedEdition(userId);
+    return row?.source === "curated" && row.status !== "dismissed" ? serialize(row) : null;
 }
 function checkRevision(userId, expectedRevision, now) {
     const current = getEditionState(userId, now);
@@ -77,7 +79,13 @@ export const publishEditionFields = {
     layout: editionLayoutSchema.optional(),
 };
 const publishSchema = z.object(publishEditionFields).strict().refine((input) => !(input.expiresAt && input.durationHours), "Use expiresAt or durationHours, not both");
-/** Both automatic and MCP editions obey the app's hard exclusions. Read/saved articles
+/** Legacy ensure requests read the latest publication; they never select or publish articles. */
+export const ensureEditionSchema = z.object({
+    requestId: z.uuid(),
+    expectedRevision: z.number().int().min(0).optional(),
+    force: z.boolean().default(false),
+}).strict().refine((input) => !input.force || input.expectedRevision !== undefined, "force requires expectedRevision");
+/** Curated publications obey the app's hard exclusions. Read/saved articles
  * may be selected deliberately by an agent, but foreign and muted items cannot be published. */
 export function publishCuratedEdition(userId, input, now = new Date()) {
     if (input.layout !== undefined && !editionLayoutSchema.safeParse(input.layout).success)
@@ -135,7 +143,7 @@ export function deleteEdition(userId) {
     dismissEdition(userId);
     return existed;
 }
-/** The automatic edition looks at every candidate; smaller limits are shortlists for an agent or the curator. */
+/** Bound candidate gathering before building a shortlist for an agent or the curator. */
 const AUTOMATIC_EDITION_LIMIT = 4000;
 /** How often each feed appeared in the editions of the last 72 hours, so a feed that was shown a lot waits its turn. */
 function recentSourceUse(userId, itemIds) {
@@ -237,58 +245,4 @@ export function editionCandidates(userId, now = new Date(), limit = 200) {
     }
     return scored.sort((a, b) => b.score - a.score || b.id - a.id);
 }
-export const generateEditionSchema = z.object({
-    requestId: z.uuid(),
-    expectedRevision: z.number().int().min(0).optional(),
-    force: z.boolean().default(false),
-}).strict().refine((input) => !input.force || input.expectedRevision !== undefined, "force requires expectedRevision");
-export function generateEdition(userId, input = {}, now = new Date()) {
-    return nativeRequest(userId, input.requestId, "edition.generate", { expectedRevision: input.expectedRevision, force: input.force ?? false }, () => {
-        checkRevision(userId, input.expectedRevision, now);
-        const prefs = getNativePreferences(userId);
-        const period = editionPeriod(now, prefs.timeZone).key;
-        const state = getEditionState(userId, now);
-        if (state.edition) {
-            if (state.edition.source === "curated") {
-                if (input.force)
-                    throw new NativeError("curated_edition_active", 409, state);
-                return state.edition;
-            }
-            if (!input.force)
-                return state.edition;
-        }
-        const row = storedEdition(userId);
-        if (!input.force && row?.status === "dismissed" && row.period === period)
-            return null;
-        if (row?.status === "active" && !state.edition) {
-            db.prepare("UPDATE editions SET status = 'expired', revision = revision + 1, updated_at = ? WHERE user_id = ?").run(now.toISOString(), userId);
-        }
-        const chosen = selectEdition(editionCandidates(userId, now, AUTOMATIC_EDITION_LIMIT), prefs);
-        if (!chosen.length)
-            return state.edition;
-        if (state.edition && JSON.stringify(state.edition.itemIds) === JSON.stringify(chosen.map((item) => item.id)))
-            return state.edition;
-        return writeEdition(userId, chosen.map((item) => item.id), "automatic", nextEditionBoundary(now, prefs.timeZone), now);
-    }, now);
-}
-/** Only users who opted in by storing preferences or creating an edition are scheduled. */
-export function maintainEditions(now = new Date()) {
-    const users = db.prepare("SELECT user_id FROM native_preferences UNION SELECT user_id FROM editions").all();
-    for (const user of users) {
-        if (!hasUserCapability(user.user_id, "editions"))
-            continue;
-        try {
-            generateEdition(user.user_id, {}, now);
-        }
-        catch (error) {
-            console.error("[edition] could not refresh an edition", error instanceof Error ? error.message : "unknown error");
-        }
-    }
-    // Receipts remain valid for at least 35 days, beyond the native offline window.
-    db.prepare("DELETE FROM native_requests WHERE created_at < ?").run(new Date(now.getTime() - 35 * 86_400_000).toISOString());
-}
-export function startEditionScheduler() {
-    maintainEditions();
-    cron.schedule("* * * * *", () => maintainEditions());
-}
 //# sourceMappingURL=editions.js.map

package/server/dist/native/requests.js

@@ -1,3 +1,4 @@
+import cron from "node-cron";
 import { createHash } from "node:crypto";
 import { db } from "../db/index.js";
 export class NativeError extends Error {
@@ -32,4 +33,11 @@ export function nativeRequest(userId, requestId, operation, input, action, now =
         return result;
     })();
 }
+/** Receipts outlive the native offline window; publication metadata stays in the edition slot. */
+export function startNativeRequestCleanup() {
+    cron.schedule("0 * * * *", () => {
+        db.prepare("DELETE FROM native_requests WHERE created_at < ?")
+            .run(new Date(Date.now() - 35 * 86_400_000).toISOString());
+    });
+}
 //# sourceMappingURL=requests.js.map

package/server/dist/server.js

@@ -18,13 +18,15 @@ import { installPollingShutdown, startPollingScheduler } from "./feeds/poller.js
 import { startCleanupScheduler } from "./feeds/cleanup.js";
 import { repairEncodedText } from "./feeds/repository.js";
 import { pruneArchive, scheduleMissingArchives } from "./feeds/archive.js";
-import { startEditionScheduler } from "./native/editions.js";
+import { startNativeRequestCleanup } from "./native/requests.js";
+import { startLeadBackfill } from "./feeds/lead.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const WEB_DIST = join(__dirname, "../../web/dist");
 runMigrations();
 const repairedItems = repairEncodedText();
 if (repairedItems > 0)
     console.log(`[db] decoded HTML entities in ${repairedItems} stored items`);
+startLeadBackfill();
 pruneArchive();
 const pendingArchives = scheduleMissingArchives();
 if (pendingArchives > 0)
@@ -32,7 +34,7 @@ if (pendingArchives > 0)
 startPollingScheduler();
 startFullTextScheduler();
 startCleanupScheduler();
-startEditionScheduler();
+startNativeRequestCleanup();
 const app = express();
 // Express validates the addresses here, so a mistyped TRUST_PROXY list stops the server at startup.
 if (config.trustProxy !== false) {

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.26.0",
+  "version": "0.26.2",
   "private": true,
   "type": "module"
 }