feedkeeper 0.26.5 → 0.27.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.26.5",
+  "version": "0.27.0",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",
@@ -47,7 +47,7 @@
     "express-rate-limit": "^8.7.1",
     "fast-xml-parser": "^5.11.2",
     "helmet": "^8.3.0",
-    "ipaddr.js": "^1.9.1",
+    "ipaddr.js": "^2.5.0",
     "jsdom": "^30.1.2",
     "node-cron": "^4.6.0",
     "otpauth": "^9.5.2",

package/server/dist/api/items.js

@@ -49,8 +49,8 @@ itemsRouter.get("/", (req, res) => {
     res.json(items.map(({ content_lead: _lead, lead_version: _leadVersion, ...item }) => ({
         ...item,
         content_snippet: cleanSnippet(item.content_snippet, item.title),
-        content_html: readerHtml(item.content_html, item.title),
-        full_content_html: readerHtml(item.full_content_html, item.title),
+        content_html: readerHtml(item.content_html, item.title, item.author),
+        full_content_html: readerHtml(item.full_content_html, item.title, item.author),
         read: Boolean(item.read),
         bookmarked: Boolean(item.bookmarked),
     })));
@@ -126,7 +126,7 @@ itemsRouter.post("/:itemId/extract-content", async (req, res) => {
     const [served] = withArchivedImages([{ id: result.id, link: findItemById(result.id)?.link ?? null, full_content_html: result.fullContentHtml }], requestOrigin(req));
     res.json({
         id: result.id,
-        full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title),
+        full_content_html: readerHtml(served.full_content_html, result.title ?? findItemById(result.id)?.title, findItemById(result.id)?.author),
         ...(result.cached ? {} : { title: result.title, byline: result.byline }),
         cached: result.cached,
     });

package/server/dist/api/v1/index.js

@@ -26,6 +26,7 @@ const FEATURES = [
     "content.revisions",
     "content.prepare",
     "item.lead",
+    "item.author",
     "images",
     "muted-keywords",
     "opml",

package/server/dist/api/v1/serialize.js

@@ -29,6 +29,7 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
         subscriptionId: row.feed_id,
         subscriptionTitle: row.feed_title,
         title: row.title,
+        author: row.author ?? null,
         url: row.link,
         publishedAt: row.published_at,
         addedAt: row.created_at,
@@ -37,7 +38,7 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
         imageUrl: row.image_url,
         hasFullText: Boolean(row.full_content_html),
         content: contentMetadata(row),
-        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title), fullTextHtml: readerHtml(row.full_content_html, row.title) } : {}),
+        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title, row.author), fullTextHtml: readerHtml(row.full_content_html, row.title, row.author) } : {}),
         state: {
             read: Boolean(row.read),
             saved: Boolean(row.bookmarked),

package/server/dist/db/migrations/0037_item_author.sql

@@ -0,0 +1,2 @@
+-- The author names a feed gives for an item, cleaned and joined into one line ("Anna Beispiel, Ben Muster").
+ALTER TABLE items ADD COLUMN author TEXT;

package/server/dist/feeds/articleContent.js

@@ -20,14 +20,14 @@ const readerCache = new Map();
  * Article HTML as a reader shows it, without breadcrumbs, repeated headline and similar page chrome. Computed when
  * served and never stored, so the publisher's original stays intact and rule changes apply to old articles too.
  */
-export function readerHtml(html, title) {
+export function readerHtml(html, title, author) {
     if (!html)
         return html ?? null;
-    const key = createHash("sha1").update(title ?? "").update("\0").update(html).digest("base64");
+    const key = createHash("sha1").update(title ?? "").update("\0").update(author ?? "").update("\0").update(html).digest("base64");
     const cached = readerCache.get(key);
     if (cached !== undefined)
         return cached;
-    const cleaned = cleanReaderHtml(html, title);
+    const cleaned = cleanReaderHtml(html, title, {}, author);
     if (readerCache.size >= READER_CACHE_LIMIT)
         readerCache.delete(readerCache.keys().next().value);
     readerCache.set(key, cleaned);

package/server/dist/feeds/author.js

@@ -0,0 +1,52 @@
+import { decodeEntities } from "./text.js";
+const GENERIC_NAMES = /^(admin(istrator)?|redaktion(steam)?|redaktionsteam|editors?|editorial( team| staff)?|staff|team|webmaster|redakteur(in)?|autor|author|unknown|unbekannt|n\/?a|null|undefined|anonymous|anonym|gast|guest|user|news|newsdesk|desk)$/i;
+const MAX_AUTHORS = 6;
+const MAX_LENGTH = 160;
+function key(value) {
+    return value.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
+}
+/** One name of a feed's author field. `staff@site.test (Anna Muster)` is the name in brackets; a bare address is no name. */
+function nameOf(raw) {
+    let text = decodeEntities(raw).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim();
+    const bracketed = text.match(/^\S+@\S+\s*\(([^)]+)\)$/);
+    if (bracketed)
+        text = bracketed[1].trim();
+    else if (/^<?\S+@\S+>?$/.test(text))
+        return null;
+    const withAddress = text.match(/^(.+?)\s*<\S+@\S+>$/);
+    if (withAddress)
+        text = withAddress[1].trim();
+    text = text.replace(/^(von|by|text|autor(in)?|author)\s*:?\s+/i, "").trim();
+    if (!text || /^https?:\/\//i.test(text) || GENERIC_NAMES.test(text))
+        return null;
+    return text;
+}
+/**
+ * The author line of an item: every distinct name the feed gives, in order, joined by commas. Nothing is made up:
+ * addresses without a name, generic names ("admin", "Redaktion") and the publication's own name are no authors.
+ */
+export function authorLine(raw, publication = {}) {
+    const values = (Array.isArray(raw) ? raw : [raw]).filter((value) => typeof value === "string");
+    const own = new Set();
+    if (publication.title)
+        own.add(key(publication.title));
+    try {
+        if (publication.url)
+            own.add(key(new URL(publication.url).hostname.replace(/^www\./, "")));
+    }
+    catch { /* an unparsable feed address is no reason to lose the author */ }
+    // "Engadget" is the publication of "Engadget - Technology News" and of engadget.com alike.
+    const isOwn = (name) => name.length >= 4 && [...own].some(known => known === name || known.startsWith(name));
+    const names = [];
+    for (const value of values) {
+        for (const part of (decodeEntities(value) ?? "").split(/\s*(?:;|\s\/\s|\s&\s|\s+und\s+|\s+and\s)\s*/i)) {
+            const name = nameOf(part);
+            if (!name || isOwn(key(name)) || names.some(known => key(known) === key(name)))
+                continue;
+            names.push(name);
+        }
+    }
+    const line = names.slice(0, MAX_AUTHORS).join(", ");
+    return line && line.length <= MAX_LENGTH ? line : line ? line.slice(0, MAX_LENGTH).replace(/[,\s]+\S*$/, "") || null : null;
+}
+//# sourceMappingURL=author.js.map

package/server/dist/feeds/contentNoise.js

@@ -45,7 +45,42 @@ const CONSENT_COMPANIONS = [
     /\bInhalte? (von \S+ )?erlauben\b/i, /-Inhalte erlauben\b/i, /^Ich bin damit einverstanden, dass mir .+ angezeigt werden/i,
     /Personenbezogene Daten können an Drittplattformen/i, /^Link zum .+-Inhalt$/i, /^Mehr dazu in unserer Datenschutzerklärung/i,
 ];
-function removeReaderNoise(root, title, stats) {
+const PRODUCT_CARD_CHARACTERS = 220;
+/** An amount of money: `88,89 EUR`, `$199`, `19.99 €`. */
+const PRICE = /(?:\d[\d.,]*\s?(?:€|EUR|\$|USD|£|GBP|CHF)|(?:€|\$|£)\s?\d)/;
+const MONTHS = "Januar|Februar|März|Maerz|April|Mai|Juni|Juli|August|September|Oktober|November|Dezember|January|February|March|June|July|October|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec|Jan\\.|Feb\\.";
+const TIME_OF_DAY = "(?:[,\\s•·|-]+\\d{1,2}[:.]\\d{2}(?::\\d{2})?(?:\\s?(?:Uhr|[ap]\\.?m\\.?))?)?";
+/** A whole line that is a date, with or without a time: `15. Oktober 2026, 10:51 Uhr`, `Oct 15, 2026`, `15.10.2026`. */
+const DATE_PART = `(?:\\d{1,2}\\.\\s?\\d{1,2}\\.\\s?\\d{2,4}|\\d{1,2}\\.?\\s(?:${MONTHS})\\.?\\s\\d{4}|(?:${MONTHS})\\.?\\s\\d{1,2},?\\s\\d{4})`;
+const DATE_ONLY = new RegExp(`^${DATE_PART}${TIME_OF_DAY}$`, "iu");
+/** The date at the end of a byline (`von Michael Bartl I 15.10.2026 - 11:41`): the author stays, the date is in the reader's head. */
+const BYLINE = /^(von|by|text|autor(in)?|author|written by)\b/i;
+const BYLINE_DATE_TAIL = new RegExp(`(?:^|\\s)[|•·–—I-]\\s*${DATE_PART}${TIME_OF_DAY}\\s*$`, "iu");
+function stripBylineDate(block) {
+    const walker = block.ownerDocument.createTreeWalker(block, 4);
+    let last = null;
+    for (let node = walker.nextNode(); node; node = walker.nextNode())
+        if (flat(node.textContent))
+            last = node;
+    if (!last || !BYLINE_DATE_TAIL.test(last.data))
+        return false;
+    last.data = last.data.replace(BYLINE_DATE_TAIL, "");
+    return true;
+}
+/** A whole line that is a reading time: `2 min`, `5 Min. Lesezeit`, `Lesezeit: 3 Minuten`, `4 min read`. */
+const READING_TIME_ONLY = /^(?:Lesezeit:?\s*)?\d{1,3}\s*(?:min\.?|minuten?|minutes?)(?:\s*(?:Lesezeit|read|lesen))?$/i;
+/** JSON-LD and similar: an object with a schema context or type, standing as text. */
+function isStructuredData(text) {
+    if (text.length < 20 || !text.startsWith("{") || !/"@(context|type)"/.test(text))
+        return false;
+    try {
+        return typeof JSON.parse(text) === "object";
+    }
+    catch {
+        return false;
+    }
+}
+function removeReaderNoise(root, title, stats, author) {
     let changed = false;
     const drop = (element, rule) => {
         if (!element.parentNode)
@@ -73,8 +108,35 @@ function removeReaderNoise(root, title, stats) {
         if (onlyLinks)
             drop(list, "breadcrumb");
     }
+    // A product card of a shop link: its picture is a CSS background, which no reader draws, and its price is a snapshot
+    // that is out of date when the article is read later. What is left would be a label, a link and a number.
+    for (const shell of [...root.querySelectorAll("[style*='url(' i]")]) {
+        if (!shell.parentNode || !/background(-image)?\s*:[^;]*url\(/i.test(shell.getAttribute("style") ?? "") || flat(shell.textContent) !== "")
+            continue;
+        let card = null;
+        for (let ancestor = shell.parentElement; ancestor; ancestor = ancestor.parentElement) {
+            const text = flat(ancestor.textContent);
+            if (text.length > PRODUCT_CARD_CHARACTERS)
+                break;
+            if (ancestor.querySelector("a[href]") && PRICE.test(text))
+                card = ancestor;
+        }
+        if (card)
+            drop(card, "product-card");
+    }
+    // Structured data is for machines. Frameworks render it as an element of any name (`<component type="application/ld+json">`),
+    // and a stray block may hold nothing else: neither is text for a reader.
+    for (const element of [...root.querySelectorAll("[type]")]) {
+        if (/^application\/([\w.+-]*\+)?json(;.*)?$/i.test(element.getAttribute("type") ?? ""))
+            drop(element, "structured-data");
+    }
+    for (const block of leafBlocks(root)) {
+        if (block.parentNode && isStructuredData(flat(block.textContent)))
+            drop(block, "structured-data");
+    }
     const headline = normalized(title ?? "");
     const seen = new Set();
+    const authorKeys = (author ?? "").split(",").map(name => normalized(name)).filter(Boolean);
     let leading = 0;
     let reachedText = false;
     for (const block of leafBlocks(root)) {
@@ -98,6 +160,26 @@ function removeReaderNoise(root, title, stats) {
                 continue;
             }
         }
+        // Near the start, a line that is only a date or a reading time repeats what the reader shows in its own head.
+        if (text.length <= 60 && charsBefore(root, block) <= 800) {
+            if (DATE_ONLY.test(text)) {
+                drop(block, "dateline");
+                continue;
+            }
+            if (READING_TIME_ONLY.test(text)) {
+                drop(block, "reading-time");
+                continue;
+            }
+            // When the feed names the author, the reader shows the name in its head: a byline that only says that goes.
+            if (BYLINE.test(text) && authorKeys.length > 0 && authorKeys.every(name => normalized(text).includes(name))) {
+                drop(block, "byline");
+                continue;
+            }
+            if (BYLINE.test(text) && stripBylineDate(block)) {
+                stats["byline-date"] = (stats["byline-date"] ?? 0) + 1;
+                changed = true;
+            }
+        }
         if (headline && normalized(text) === headline && (position < 4 || (heading && position < 14))) {
             drop(block, "title-repeat");
             continue;
@@ -143,6 +225,34 @@ function removeReaderNoise(root, title, stats) {
             seen.add(key);
         }
     }
+    // A comment section's house rules at the end of the text: no reader has comments, so heading and what follows go.
+    const total = (root.textContent ?? "").length;
+    for (const block of leafBlocks(root)) {
+        if (!block.parentNode || !COMMENT_RULES_HEADING.test(flat(block.textContent)))
+            continue;
+        if (total - charsBefore(root, block) > TAIL_CHARACTERS)
+            continue;
+        let node = block;
+        while (node.parentNode && node !== root) {
+            while (node.nextSibling)
+                node.nextSibling.remove();
+            node = node.parentNode;
+        }
+        drop(block, "comment-rules");
+        break;
+    }
+    // An ad label whose ad is not there (a script fills it): sponsored content itself stays, the label above nothing goes.
+    const labelled = leafBlocks(root);
+    labelled.forEach((block, index) => {
+        if (!block.parentNode || !AD_LABEL.test(flat(block.textContent)) || block.querySelector("img, iframe, a"))
+            return;
+        const sibling = block.nextElementSibling;
+        if (sibling && (sibling.matches("img, iframe, video, picture, figure, a") || sibling.querySelector("img, iframe, video, picture, a")))
+            return;
+        const next = labelled.slice(index + 1).find(other => other.parentNode && flat(other.textContent));
+        if (!next || /^H[1-6]$/.test(next.tagName) || AD_LABEL.test(flat(next.textContent)))
+            drop(block, "empty-ad-label");
+    });
     // Consent placeholders of embedded third-party content: only the neighbours of a known anchor phrase go.
     const blocks = leafBlocks(root);
     blocks.forEach((block, index) => {
@@ -240,14 +350,135 @@ function serialize(root) {
     return holder.innerHTML;
 }
 /** The article HTML as a reader should show it. Unchanged input is returned as is. */
-export function cleanReaderHtml(html, title, stats = {}) {
+export function cleanReaderHtml(html, title, stats = {}, author) {
     if (!html)
         return html;
     const root = JSDOM.fragment(html);
     const images = resolveCustomImages(root, stats);
-    const noise = removeReaderNoise(root, title, stats);
+    const variants = collapseImageVariants(root, stats);
+    const videos = resolveVideoPlayers(root, stats);
+    const noise = removeReaderNoise(root, title, stats, author);
     const typography = removeTypography(root, stats);
-    return images || noise || typography ? serialize(root) : html;
+    videos.links.forEach(link => { if (root.contains(link))
+        link.setAttribute("data-media", "video"); });
+    return images || variants || videos.changed || noise || typography ? serialize(root) : html;
+}
+const PLAY_LABEL = /^(video starten|video abspielen|abspielen|wiedergabe( starten)?|play( video)?|watch( video)?|start video)$/i;
+const PLAY_SOURCE = /(?:^|[/_.-])play[-_]?(?:icon|button|btn)?\.(?:svg|png|webp|gif)(?:[?#]|$)/i;
+const DURATION = /^\d{1,2}:\d{2}(:\d{2})?$/;
+const PLAYER_BLOCK_LIMIT = 120;
+function isPlayOverlay(image) {
+    return PLAY_LABEL.test(flat(image.getAttribute("alt"))) || PLAY_SOURCE.test((image.getAttribute("src") ?? "").trim());
+}
+/** Takes away what is left empty once a player is gone, up to (not including) the fragment itself. */
+function pruneEmpty(node) {
+    while (node && node.parentNode && node.nodeType === 1) {
+        const element = node;
+        if (flat(element.textContent) !== "" || element.querySelector("img, picture, video, iframe, a[href]"))
+            return;
+        const parent = element.parentNode;
+        element.remove();
+        node = parent;
+    }
+}
+/** Direct video URL of the embeds that are common enough to be worth a link. Anything else is not turned into a link. */
+function embedTarget(source) {
+    let address;
+    try {
+        address = new URL(source.startsWith("//") ? `https:${source}` : source);
+    }
+    catch {
+        return null;
+    }
+    const host = address.hostname.replace(/^www\./, "");
+    const segments = address.pathname.split("/").filter(Boolean);
+    if (/^youtube(-nocookie)?\.com$/.test(host) && segments[0] === "embed" && /^[\w-]{6,}$/.test(segments[1] ?? "")) {
+        return { url: `https://www.youtube.com/watch?v=${segments[1]}`, name: "YouTube" };
+    }
+    if (host === "player.vimeo.com" && segments[0] === "video" && /^\d+$/.test(segments[1] ?? "")) {
+        return { url: `https://vimeo.com/${segments[1]}`, name: "Vimeo" };
+    }
+    if (host === "dailymotion.com" && segments[0] === "embed" && segments[1] === "video" && /^\w+$/.test(segments[2] ?? "")) {
+        return { url: `https://www.dailymotion.com/video/${segments[2]}`, name: "Dailymotion" };
+    }
+    return null;
+}
+function videoLink(document, url, text) {
+    const paragraph = document.createElement("p");
+    const link = document.createElement("a");
+    link.setAttribute("href", url);
+    link.textContent = text;
+    paragraph.append(link);
+    return paragraph;
+}
+/**
+ * Players need the publisher's script: a reader without it shows their play-button picture as a broken image and
+ * leaves a dead block. What can be reached stays as one plain link to the video (`data-media="video"` tells a client
+ * that the target is a video page); what cannot, because the player is a script-driven button without an address,
+ * is left out. The video itself is never embedded here: that is for the client to decide.
+ */
+function resolveVideoPlayers(root, stats) {
+    const links = [];
+    let changed = false;
+    const count = () => { stats["video"] = (stats["video"] ?? 0) + 1; changed = true; };
+    for (const image of [...root.querySelectorAll("img")].filter(isPlayOverlay)) {
+        if (!root.contains(image))
+            continue;
+        const anchor = image.closest("a[href]");
+        if (anchor) {
+            const parent = anchor.parentNode;
+            image.remove();
+            const href = anchor.getAttribute("href");
+            const scope = anchor.closest("p, div, figure, li") ?? anchor.parentElement;
+            const twin = scope ? [...scope.querySelectorAll("a[href]")].find(other => other !== anchor && other.getAttribute("href") === href) : undefined;
+            const label = flat(anchor.textContent);
+            if (twin) {
+                anchor.remove();
+                pruneEmpty(parent);
+                links.push(twin);
+            }
+            else {
+                if (label === "" || DURATION.test(label))
+                    anchor.textContent = flat(anchor.getAttribute("title")) || "Video";
+                links.push(anchor);
+            }
+            count();
+            continue;
+        }
+        // The player block is the widest wrapper that holds nothing but the label, the picture and no link to follow.
+        let block = image;
+        while (block.parentElement) {
+            const parent = block.parentElement;
+            if (flat(parent.textContent).length > PLAYER_BLOCK_LIMIT || parent.querySelector("a[href]") || parent.querySelectorAll("img").length > 1)
+                break;
+            block = parent;
+        }
+        const parent = block.parentNode;
+        block.remove();
+        pruneEmpty(parent);
+        count();
+    }
+    for (const frame of [...root.querySelectorAll("iframe")]) {
+        const target = embedTarget(frame.getAttribute("src") ?? frame.getAttribute("data-src") ?? "");
+        if (!target)
+            continue;
+        const parent = frame.parentNode;
+        const replacement = videoLink(frame.ownerDocument, target.url, flat(frame.getAttribute("title")) || target.name);
+        frame.replaceWith(replacement);
+        pruneEmpty(parent);
+        links.push(replacement.querySelector("a"));
+        count();
+    }
+    for (const video of [...root.querySelectorAll("video")]) {
+        const source = (video.getAttribute("src") ?? video.querySelector("source[src]")?.getAttribute("src") ?? "").trim();
+        if (!/^https?:\/\//i.test(source))
+            continue;
+        const replacement = videoLink(video.ownerDocument, source, flat(video.getAttribute("title")) || "Video");
+        video.replaceWith(replacement);
+        links.push(replacement.querySelector("a"));
+        count();
+    }
+    return { changed, links };
 }
 /**
  * Some publishers wrap a picture in their own element and let a script fill the image: `<a-img src="/imgs/…">`
@@ -276,6 +507,31 @@ function resolveCustomImages(root, stats) {
         stats["custom-image"] = (stats["custom-image"] ?? 0) + 1;
     return changed;
 }
+/**
+ * Publishers put one badge in two colour schemes into a link (`<a><img alt="X"><img alt="X"></a>`) and let their
+ * stylesheet show one of them. A reader has no such stylesheet, so both stand there at full size. Images that carry
+ * the same label inside one link and no other text are variants of one thing: the link keeps its label as text, or,
+ * without a label, its first image. The call to action itself stays.
+ */
+function collapseImageVariants(root, stats) {
+    let changed = false;
+    for (const anchor of [...root.querySelectorAll("a")]) {
+        const images = [...anchor.querySelectorAll("img")];
+        if (images.length < 2 || flat(anchor.textContent) !== "")
+            continue;
+        const labels = new Set(images.map(image => flat(image.getAttribute("alt"))));
+        if (labels.size !== 1)
+            continue;
+        const [label] = labels;
+        if (label)
+            anchor.replaceChildren(anchor.ownerDocument.createTextNode(label));
+        else
+            images.slice(1).forEach(image => image.remove());
+        stats["image-variants"] = (stats["image-variants"] ?? 0) + 1;
+        changed = true;
+    }
+    return changed;
+}
 const CREDIT_PATTERNS = [
     /^(©|\(c\)\s|copyright\b)/i,
     /^[([]?(Bild|Foto|Fotos|Bildquelle|Bildrechte|Bildnachweis|Titelbild|Beitragsbild|Photo|Image|Credits?|Grafik|Illustration)[)\]]?\s*[:/|]/i,
@@ -326,6 +582,8 @@ const COMMENT_PATTERNS = [
     /\b(let us know|tell us) .{0,40}in the comments\b/i,
     /\bNur angemeldete Benutzer können kommentieren\b/i,
 ];
+const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertorial|Sponsored|Gesponsert|Promoted)\s*:?$/i;
+const COMMENT_RULES_HEADING = /^(Regeln für Kommentare|Kommentarregeln|Kommentar-Richtlinien|Comment(ing)? (rules|guidelines|policy)|Community[- ]Richtlinien)\s*:?$/i;
 const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
 const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;
 const SECTION_LABEL = /^(Auch interessant|Das könnte (dich|Sie|euch) (auch )?interessieren|Mehr zum Thema|Lesen Sie auch|Lies auch|Lest auch|Aktuelle News|Weitere (Artikel|Meldungen|News)|Popular Stories|Related (Stories|Articles|Posts|Reading)|More (Stories|from)|Top Stories|Meistgelesen|Topaktuell|Regeln für Kommentare|Kommentare|Comments?)\b\s*:?/i;

package/server/dist/feeds/heroImage.js

@@ -0,0 +1,81 @@
+import { decodeEntities } from "./text.js";
+/** The widest picture a hero needs; a still larger variant is not worth the bytes. */
+const MAX_HERO_WIDTH = 2000;
+/** A larger variant must be clearly larger than the feed's picture, so a small difference does not change anything. */
+const MIN_GAIN = 1.5;
+/**
+ * Splits an image address into what identifies the picture and the width of this size. Publishers name sizes in the
+ * file name (`…__wide__148`, `-300x200.jpg`, `_300x200.jpg`) or in the query (`?w=300`); two addresses with the same
+ * key are the same picture in different sizes. Addresses without a recognised size have no variant.
+ */
+export function imageVariant(raw) {
+    let url;
+    try {
+        url = new URL(raw);
+    }
+    catch {
+        return null;
+    }
+    const path = url.pathname;
+    const named = /^(.*__[a-z]+__)(\d{2,4})$/i.exec(path) ?? /^(.*[-_])(\d{2,4})x\d{2,4}(\.[a-z0-9]{2,5})?$/i.exec(path);
+    if (named) {
+        // The query of such addresses is a cache version (`?v=…`), not part of the picture.
+        return { key: `${url.origin}${named[1]}#${named[3] ?? ""}`, width: Number(named[2]) };
+    }
+    const widthParam = ["w", "width"].find((name) => /^\d{2,4}$/.test(url.searchParams.get(name) ?? ""));
+    if (widthParam) {
+        const rest = new URLSearchParams(url.searchParams);
+        for (const name of ["w", "width", "h", "height", "q", "quality", "fit", "dpr"])
+            rest.delete(name);
+        return { key: `${url.origin}${path}?${rest.toString()}`, width: Number(url.searchParams.get(widthParam)) };
+    }
+    return null;
+}
+/** Every image address in the HTML: `src` of images and each entry of a `srcset`. */
+function imageAddresses(html, baseUrl) {
+    const found = new Set();
+    const add = (raw) => {
+        const value = decodeEntities(raw.trim());
+        if (!value || value.startsWith("data:"))
+            return;
+        try {
+            const url = new URL(value, baseUrl ?? undefined);
+            if (url.protocol === "https:" || url.protocol === "http:")
+                found.add(url.toString());
+        }
+        catch { /* A broken address is not a candidate. */ }
+    };
+    for (const [tag] of html.matchAll(/<(?:img|source)\b[^>]*>/gi)) {
+        const src = /\ssrc\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(tag);
+        if (src)
+            add(src[1] ?? src[2] ?? "");
+        const srcset = /\ssrcset\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(tag);
+        if (srcset)
+            for (const entry of (srcset[1] ?? srcset[2] ?? "").split(/,\s+/))
+                add(entry.trim().split(/\s+/)[0] ?? "");
+    }
+    return [...found];
+}
+/**
+ * A feed often offers a thumbnail (`…__wide__148`) of the picture that its article shows in full size. When the
+ * article carries the same picture in a clearly larger size, that size is the better hero; otherwise there is none.
+ */
+export function largerHeroImage(imageUrl, html, baseUrl) {
+    if (!imageUrl || !html || !html.includes("<"))
+        return null;
+    const current = imageVariant(imageUrl);
+    if (!current)
+        return null;
+    let best = null;
+    for (const address of imageAddresses(html, baseUrl)) {
+        const variant = imageVariant(address);
+        if (!variant || variant.key !== current.key)
+            continue;
+        if (variant.width < current.width * MIN_GAIN || variant.width > MAX_HERO_WIDTH)
+            continue;
+        if (!best || variant.width > best.width)
+            best = { url: address, width: variant.width };
+    }
+    return best?.url ?? null;
+}
+//# sourceMappingURL=heroImage.js.map

package/server/dist/feeds/poller.js

@@ -10,6 +10,7 @@ import { db } from "../db/index.js";
 import { configurePollHostCooldowns, PollQueue } from "./pollQueue.js";
 import { feedRequests, PollDeferredError, sourceHost } from "./requestGate.js";
 import { normalizePublishedAt } from "./dates.js";
+import { authorLine } from "./author.js";
 configurePollHostCooldowns();
 function mediaAttribute(value, key) {
     if (typeof value !== "object" || value === null)
@@ -111,6 +112,7 @@ async function pollAttempt(feed, signal) {
         contentSnippet: decodeEntities(item.contentSnippet ?? item.content) ?? undefined,
         contentHtml: item.contentEncoded ?? item["content:encoded"] ?? item.content ?? null,
         publishedAt: normalizePublishedAt(item.isoDate ?? item.pubDate) ?? undefined, imageUrl: extractImageUrl(item),
+        author: authorLine([item.creator, item.author], { title: current.title, url: current.site_url ?? current.url }),
     }));
     // No await in this transaction. Items, health/validators and job completion become visible together.
     const newItems = db.transaction(() => {

package/server/dist/feeds/repository.js

@@ -1,6 +1,7 @@
 import { db } from "../db/index.js";
 import { articleText } from "./articleContent.js";
 import { LEAD_VERSION } from "./contentNoise.js";
+import { largerHeroImage } from "./heroImage.js";
 import { refreshLead } from "./lead.js";
 import { decodeEntities } from "./text.js";
 export function findFeedByUrl(url) {
@@ -135,8 +136,8 @@ export function updateFeedAfterPoll(feedId, data) {
     }
 }
 export function upsertItems(feedId, items) {
-    const insert = db.prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, published_at, image_url)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    const insert = db.prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, published_at, image_url, author)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
      ON CONFLICT (feed_id, guid) DO NOTHING`);
     const update = db.prepare(`UPDATE items SET
        title = COALESCE(?, title),
@@ -144,7 +145,8 @@ export function upsertItems(feedId, items) {
        content_snippet = COALESCE(?, content_snippet),
        content_html = COALESCE(?, content_html),
        published_at = COALESCE(?, published_at),
-       image_url = COALESCE(?, image_url)
+       image_url = COALESCE(?, image_url),
+       author = COALESCE(?, author)
      WHERE feed_id = ? AND guid = ?`);
     const insertMany = db.transaction((rows) => {
         let inserted = 0;
@@ -157,7 +159,8 @@ export function upsertItems(feedId, items) {
                 row.contentSnippet ?? null,
                 row.contentHtml ?? null,
                 row.publishedAt ?? null,
-                row.imageUrl ?? null,
+                largerHeroImage(row.imageUrl, row.contentHtml, row.link ?? null) ?? row.imageUrl ?? null,
+                row.author ?? null,
             ];
             const result = insert.run(...values);
             if (result.changes > 0) {
@@ -173,7 +176,7 @@ export function upsertItems(feedId, items) {
                 // The summary belongs to the body it was sent with: keeping the longer body keeps its summary too, or
                 // a feed that alternates between two variants would change the content on every poll.
                 const keptPrevious = retained !== null && next !== retained;
-                update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, feedId, row.guid);
+                update.run(row.title ?? null, row.link ?? null, keptPrevious ? null : row.contentSnippet ?? null, retained, row.publishedAt ?? null, row.imageUrl ?? null, row.author ?? null, feedId, row.guid);
                 // Polling rewrites every item; only a change of what the lead is made from needs it derived again.
                 if (previous) {
                     const changed = (retained !== null && retained !== previous.content_html) ||
@@ -425,6 +428,14 @@ export function findItemById(id) {
 export function updateItemFullContent(id, fullContentHtml) {
     db.prepare("UPDATE items SET full_content_html = ? WHERE id = ?").run(fullContentHtml, id);
     refreshLead(id);
+    upgradeHeroImage(id, fullContentHtml);
+}
+/** The feed's picture is often a thumbnail of the one the article shows: use the larger size once the text is known. */
+function upgradeHeroImage(id, html) {
+    const item = db.prepare("SELECT image_url, link FROM items WHERE id = ?").get(id);
+    const larger = item ? largerHeroImage(item.image_url, html, item.link) : null;
+    if (larger)
+        db.prepare("UPDATE items SET image_url = ? WHERE id = ?").run(larger, id);
 }
 /** Apply a per-item change to many items atomically; returns how many rows changed. */
 export function applyToItems(itemIds, change) {

package/server/dist/mcp/items.js

@@ -51,7 +51,7 @@ export function getItemForMcp(userId, itemId, maxChars = MAX_CONTENT_LENGTH, off
         return null;
     if (revision !== undefined && revision !== item.content_revision)
         throw new Error("content_revision_conflict");
-    const content = readerHtml(item.full_content_html ?? item.content_html, item.title);
+    const content = readerHtml(item.full_content_html ?? item.content_html, item.title, item.author);
     const body = format === "text" ? sourceText(item) : content ?? "";
     let end = Math.min(offset + Math.min(maxChars, MAX_CONTENT_LENGTH), body.length);
     if (end < body.length && /[\uD800-\uDBFF]/.test(body[end - 1]))

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.26.5",
+  "version": "0.27.0",
   "private": true,
   "type": "module"
 }