feedkeeper 0.28.0 → 0.28.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.28.0",
+  "version": "0.28.2",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/feeds/contentNoise.js

@@ -121,8 +121,14 @@ function removeReaderNoise(root, title, stats, author) {
             if (ancestor.querySelector("a[href]") && PRICE.test(text))
                 card = ancestor;
         }
-        if (card)
+        if (card) {
+            const parent = card.parentElement;
             drop(card, "product-card");
+            // A long product name can put the enclosing header just beyond the card's size limit.
+            if (parent && PRODUCT_LABEL.test(flat(parent.textContent)) && !parent.querySelector("img, picture, iframe, video, a[href]")) {
+                drop(parent, "empty-product-label");
+            }
+        }
     }
     // Structured data is for machines. Frameworks render it as an element of any name (`<component type="application/ld+json">`),
     // and a stray block may hold nothing else: neither is text for a reader.
@@ -137,6 +143,32 @@ function removeReaderNoise(root, title, stats, author) {
     const headline = normalized(title ?? "");
     const seen = new Set();
     const authorKeys = (author ?? "").split(",").map(name => normalized(name)).filter(Boolean);
+    const cleanByline = (block, text) => {
+        if (!BYLINE.test(text))
+            return;
+        if (authorKeys.length > 0 && authorKeys.every(name => normalized(text).includes(name))) {
+            drop(block, "byline");
+            return;
+        }
+        if (stripBylineDate(block)) {
+            stats["byline-date"] = (stats["byline-date"] ?? 0) + 1;
+            changed = true;
+        }
+    };
+    // Some publishers use an inline wrapper as a standalone byline. Never treat spans inside prose as metadata.
+    for (const span of root.querySelectorAll("span")) {
+        if (!root.contains(span) || span.closest(BLOCKS) || span.querySelector(`${BLOCKS}, div`))
+            continue;
+        const text = flat(span.textContent);
+        if (text.length > 60 || !BYLINE.test(text) || charsBefore(root, span) > 800)
+            continue;
+        if ([...(span.parentNode?.childNodes ?? [])].some(node => node !== span && node.nodeType === 3 && flat(node.textContent)))
+            continue;
+        if ([...(span.parentElement?.children ?? [])].some(sibling => sibling !== span &&
+            !sibling.matches(`${BLOCKS}, div, article, section, header, footer, figure, ul, ol, table, aside`) && flat(sibling.textContent)))
+            continue;
+        cleanByline(span, text);
+    }
     let leading = 0;
     let reachedText = false;
     let pastStart = false;
@@ -177,14 +209,9 @@ function removeReaderNoise(root, title, stats, author) {
                     continue;
                 }
                 // When the feed names the author, the reader shows the name in its head: a byline that only says that goes.
-                if (BYLINE.test(text) && authorKeys.length > 0 && authorKeys.every(name => normalized(text).includes(name))) {
-                    drop(block, "byline");
+                cleanByline(block, text);
+                if (!root.contains(block))
                     continue;
-                }
-                if (BYLINE.test(text) && stripBylineDate(block)) {
-                    stats["byline-date"] = (stats["byline-date"] ?? 0) + 1;
-                    changed = true;
-                }
             }
         }
         if (headline && normalized(text) === headline && (position < 4 || (heading && position < 14))) {
@@ -248,6 +275,27 @@ function removeReaderNoise(root, title, stats, author) {
         drop(block, "comment-rules");
         break;
     }
+    // Empty nested slots are structural evidence: the next story paragraph is outside this ad container.
+    // Do not infer an empty ad from a standalone heading followed by text (which may itself be sponsored).
+    for (const container of [...root.querySelectorAll("div, aside, section, p")]) {
+        if (!root.contains(container))
+            continue;
+        const text = flat(container.textContent);
+        const product = PRODUCT_LABEL.test(text);
+        if (!product && !AD_LABEL.test(text))
+            continue;
+        if (container.querySelector("img, picture, iframe, video, audio, a[href], object, embed"))
+            continue;
+        const emptySlot = container.querySelector("div:empty, ins:empty");
+        // Readability may remove the empty slot, leaving only its enclosing block and label paragraph.
+        const isolated = container.matches("div, aside, section") && container.querySelector("p") &&
+            (product || AD_SLOT_LABEL.test(text));
+        const next = container.nextElementSibling;
+        const nextHasMedia = next && (next.matches("img, picture, iframe, video, audio, a[href]") ||
+            next.querySelector("img, picture, iframe, video, audio, a[href]"));
+        if (emptySlot || (isolated && !nextHasMedia))
+            drop(container, product ? "empty-product-label" : "empty-ad-label");
+    }
     // An ad label whose ad is not there (a script fills it): sponsored content itself stays, the label above nothing goes.
     const labelled = leafBlocks(root);
     labelled.forEach((block, index) => {
@@ -362,13 +410,14 @@ export function cleanReaderHtml(html, title, stats = {}, author) {
         return html;
     const root = JSDOM.fragment(html);
     const images = resolveCustomImages(root, stats);
+    const duplicates = collapseAdjacentImages(root, stats);
     const variants = collapseImageVariants(root, stats);
     const videos = resolveVideoPlayers(root, stats);
     const noise = removeReaderNoise(root, title, stats, author);
     const typography = removeTypography(root, stats);
     videos.links.forEach(link => { if (root.contains(link))
         link.setAttribute("data-media", "video"); });
-    return images || variants || videos.changed || noise || typography ? serialize(root) : html;
+    return images || duplicates || variants || videos.changed || noise || typography ? serialize(root) : html;
 }
 const PLAY_LABEL = /^(video starten|video abspielen|abspielen|wiedergabe( starten)?|play( video)?|watch( video)?|start video)$/i;
 const PLAY_SOURCE = /(?:^|[/_.-])play[-_]?(?:icon|button|btn)?\.(?:svg|png|webp|gif)(?:[?#]|$)/i;
@@ -539,6 +588,46 @@ function collapseImageVariants(root, stats) {
     }
     return changed;
 }
+/** Responsive copies may be separate images whose CSS normally hides one. Only exact adjacent copies go;
+ * text, other media, different links, captions and responsive sources keep images distinct. */
+function collapseAdjacentImages(root, stats) {
+    const walker = root.ownerDocument.createTreeWalker(root, 1 | 4);
+    let previous = null;
+    const duplicates = [];
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+        if (node.nodeType === 3) {
+            if (flat(node.textContent))
+                previous = null;
+            continue;
+        }
+        const element = node;
+        if (element.matches("video, audio, iframe, object, embed, hr"))
+            previous = null;
+        if (element.tagName !== "IMG")
+            continue;
+        const src = flat(element.getAttribute("data-src") || element.getAttribute("data-lazy-src") ||
+            element.getAttribute("data-original") || element.getAttribute("src"));
+        if (!src || /^(data|blob):/i.test(src)) {
+            previous = null;
+            continue;
+        }
+        const picture = element.closest("picture");
+        const key = JSON.stringify([src, element.getAttribute("srcset"), element.getAttribute("data-srcset"),
+            element.getAttribute("alt"), element.getAttribute("title"), element.closest("a")?.getAttribute("href"),
+            [...(picture?.querySelectorAll("source") ?? [])].map(source => [source.getAttribute("srcset"), source.getAttribute("media"), source.getAttribute("type")])]);
+        if (previous === key)
+            duplicates.push(picture?.querySelectorAll("img").length === 1 ? picture : element);
+        else
+            previous = key;
+    }
+    for (const duplicate of duplicates) {
+        const parent = duplicate.parentNode;
+        duplicate.remove();
+        pruneEmpty(parent);
+        stats["duplicate-image"] = (stats["duplicate-image"] ?? 0) + 1;
+    }
+    return duplicates.length > 0;
+}
 const CREDIT_PATTERNS = [
     /^(©|\(c\)\s|copyright\b)/i,
     /^[([]?(Bild|Foto|Fotos|Bildquelle|Bildrechte|Bildnachweis|Titelbild|Beitragsbild|Photo|Image|Credits?|Grafik|Illustration)[)\]]?\s*[:/|]/i,
@@ -589,7 +678,9 @@ const COMMENT_PATTERNS = [
     /\b(let us know|tell us) .{0,40}in the comments\b/i,
     /\bNur angemeldete Benutzer können kommentieren\b/i,
 ];
-const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|Sponsored( content| post)?|Paid (content|post)|Gesponsert|Promoted)\s*:?$/i;
+const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|Sponsored( content| post)?|Paid (content|post)|Gesponsert|Promoted|広告|広告掲載|スポンサー(?:記事|コンテンツ)?|宣伝|PR)\s*[:：]?$/i;
+const AD_SLOT_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Ads?|広告|広告掲載|宣伝)\s*[:：]?$/i;
+const PRODUCT_LABEL = /^(Produkthinweis|Produktempfehlung|Product (notice|recommendation)|商品紹介|商品案内|製品紹介)\s*[:：]?$/i;
 const COMMENT_RULES_HEADING = /^(Regeln für Kommentare|Kommentarregeln|Kommentar-Richtlinien|Community[- ]Richtlinien|(Comment(ing)?|Community|Discussion|House) (rules|guidelines|policy|policies)|Rules for comments|Commenting rules)\s*:?$/i;
 const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
 const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;

package/server/dist/native/editionContent.js

@@ -0,0 +1,32 @@
+import { JSDOM } from "jsdom";
+import { decodeEntities } from "../feeds/text.js";
+const PROMPTS = new Set([
+    "mehr", "weiterlesen", "mehr lesen", "read more", "more", "continue reading",
+    "video", "zum video", "video ansehen", "video abspielen", "watch video", "watch the video", "play video",
+    "audio", "zum audio", "audio anhören", "listen", "listen now", "listen to audio",
+    "続きを読む", "もっと読む", "動画を見る", "動画を再生", "音声を聞く",
+]);
+function normalize(text) {
+    return text.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+}
+/** No length threshold: a short news sentence is useful, a media player or read-more link alone is not. */
+export function hasEditionText(item) {
+    const title = normalize(decodeEntities(item.title) ?? "");
+    return [item.content_snippet, item.content_html, item.full_content_html].some(source => {
+        if (!source)
+            return false;
+        const fragment = JSDOM.fragment(source);
+        fragment.querySelectorAll("script, style, noscript, nav, footer, video, audio, iframe, object, embed").forEach(node => node.remove());
+        fragment.querySelectorAll("a, p, span, h1, h2, h3, h4").forEach(node => {
+            const text = normalize(node.textContent ?? "");
+            if (text === title || PROMPTS.has(text))
+                node.remove();
+        });
+        fragment.querySelectorAll("p, div, br, li, h1, h2, h3, h4, blockquote, figcaption").forEach(node => node.append("\n"));
+        return (fragment.textContent ?? "").split(/\n/).some(line => {
+            const text = normalize(line);
+            return text !== "" && text !== title && !PROMPTS.has(text);
+        });
+    });
+}
+//# sourceMappingURL=editionContent.js.map

package/server/dist/native/editions.js

@@ -7,6 +7,7 @@ import { editionPeriod, selectEdition } from "./editionSelection.js";
 import { getNativePreferences } from "./preferences.js";
 import { checkEditionLayout, editionLayoutSchema } from "./editionLayout.js";
 import { NativeError, nativeRequest } from "./requests.js";
+import { hasEditionText } from "./editionContent.js";
 export const overviewSchema = z.array(z.object({
     heading: z.string().trim().min(1).max(160),
     segments: z.array(z.object({
@@ -109,10 +110,11 @@ export function publishCuratedEdition(userId, input, now = new Date()) {
             if (!canAccessItem(userId, id))
                 throw new NativeError("item_not_found", 404);
             const item = db.prepare(`SELECT s.folder_id, s.id IS NOT NULL AS subscribed, i.is_empty_gallery,
+          i.title, i.content_snippet, i.content_html, i.full_content_html,
           EXISTS (SELECT 1 FROM user_muted_keywords m WHERE m.user_id = ? AND
             (instr(lower(coalesce(i.title, '')), lower(m.keyword)) > 0 OR instr(lower(coalesce(i.content_snippet, '')), lower(m.keyword)) > 0)) AS muted
          FROM items i LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ? WHERE i.id = ?`).get(userId, userId, id);
-            if (item.is_empty_gallery || item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
+            if (item.is_empty_gallery || !hasEditionText(item) || item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
                 throw new NativeError("item_excluded");
             }
         }
@@ -215,7 +217,7 @@ export function editionCandidates(userId, now = new Date(), limit = 200) {
     const previous = new Set(previousItemIds);
     // Fetch bounded candidates per source, so a busy feed cannot hide the others.
     const rows = db.prepare(`SELECT r.id, r.feed_id, r.folder_id, r.title, r.link, r.content_snippet, r.image_url,
-        r.source_name, r.folder_name, r.published, r.source_rank,
+        r.source_name, r.folder_name, r.published, r.source_rank, full_item.content_html, full_item.full_content_html,
         length(coalesce(full_item.full_content_html, full_item.content_html, r.content_snippet, '')) AS content_chars
       FROM (
         SELECT i.id, i.feed_id, s.folder_id, i.title, i.link, i.content_snippet, i.image_url,
@@ -233,7 +235,7 @@ export function editionCandidates(userId, now = new Date(), limit = 200) {
       JOIN items full_item ON full_item.id = r.id
       WHERE r.source_rank <= 40 ORDER BY r.source_rank, r.published DESC, r.id DESC LIMIT 4000`)
         .all(userId, userId, userId, new Date(now.getTime() - 7 * 86_400_000).toISOString(), now.toISOString());
-    const scored = rows.filter((row) => row.folder_id === null ? prefs.showUnfiled : !prefs.hiddenFolderIds.includes(row.folder_id))
+    const scored = rows.filter((row) => (row.folder_id === null ? prefs.showUnfiled : !prefs.hiddenFolderIds.includes(row.folder_id)) && hasEditionText(row))
         .map((row) => ({
         id: row.id, sourceName: row.source_name, folderName: row.folder_name, subscriptionId: row.feed_id, folderId: row.folder_id,
         title: row.title, url: row.link, publishedAt: row.published, snippet: decodeEntities(row.content_snippet) ?? null, imageUrl: row.image_url,

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.28.0",
+  "version": "0.28.2",
   "private": true,
   "type": "module"
 }