feedkeeper 0.27.2 → 0.28.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.27.2",
+  "version": "0.28.1",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/v1/index.js

@@ -27,6 +27,7 @@ const FEATURES = [
     "content.prepare",
     "item.lead",
     "item.author",
+    "item.empty-gallery",
     "images",
     "muted-keywords",
     "opml",

package/server/dist/db/migrations/0040_empty_gallery.sql

@@ -0,0 +1,10 @@
+-- Classification is shared publisher metadata, not a deletion or a per-user read state.
+ALTER TABLE items ADD COLUMN is_empty_gallery INTEGER NOT NULL DEFAULT 0 CHECK (is_empty_gallery IN (0, 1));
+-- Reuse the bounded metadata backfill; existing content and reading history stay intact.
+UPDATE items SET lead_version = NULL;
+
+CREATE TRIGGER trg_gallery_revision AFTER UPDATE OF is_empty_gallery ON items
+WHEN OLD.is_empty_gallery IS NOT NEW.is_empty_gallery
+BEGIN
+  UPDATE items SET content_revision = content_revision + 1 WHERE id = NEW.id;
+END;

package/server/dist/feeds/articleContent.js

@@ -36,7 +36,7 @@ export function readerHtml(html, title, author) {
 export function contentMetadata(item) {
     const source = item.full_content_html ? "reader" : item.content_html ? "feed" : "snippet";
     return { revision: item.content_revision, source, status: item.extraction_status,
-        attemptedAt: item.extraction_attempted_at, retryAt: item.extraction_retry_at };
+        attemptedAt: item.extraction_attempted_at, retryAt: item.extraction_retry_at, isEmptyGallery: Boolean(item.is_empty_gallery) };
 }
 export function sourceText(item) {
     return `${decodeEntities(item.title) ?? ""}\n${derivedText(item.full_content_html || item.content_html, item.title) || cleanSnippet(decodeEntities(item.content_snippet), item.title) || ""}`.trim();

package/server/dist/feeds/contentNoise.js

@@ -121,8 +121,14 @@ function removeReaderNoise(root, title, stats, author) {
             if (ancestor.querySelector("a[href]") && PRICE.test(text))
                 card = ancestor;
         }
-        if (card)
+        if (card) {
+            const parent = card.parentElement;
             drop(card, "product-card");
+            // A long product name can put the enclosing header just beyond the card's size limit.
+            if (parent && PRODUCT_LABEL.test(flat(parent.textContent)) && !parent.querySelector("img, picture, iframe, video, a[href]")) {
+                drop(parent, "empty-product-label");
+            }
+        }
     }
     // Structured data is for machines. Frameworks render it as an element of any name (`<component type="application/ld+json">`),
     // and a stray block may hold nothing else: neither is text for a reader.
@@ -248,6 +254,27 @@ function removeReaderNoise(root, title, stats, author) {
         drop(block, "comment-rules");
         break;
     }
+    // Empty nested slots are structural evidence: the next story paragraph is outside this ad container.
+    // Do not infer an empty ad from a standalone heading followed by text (which may itself be sponsored).
+    for (const container of [...root.querySelectorAll("div, aside, section, p")]) {
+        if (!root.contains(container))
+            continue;
+        const text = flat(container.textContent);
+        const product = PRODUCT_LABEL.test(text);
+        if (!product && !AD_LABEL.test(text))
+            continue;
+        if (container.querySelector("img, picture, iframe, video, audio, a[href], object, embed"))
+            continue;
+        const emptySlot = container.querySelector("div:empty, ins:empty");
+        // Readability may remove the empty slot, leaving only its enclosing block and label paragraph.
+        const isolated = container.matches("div, aside, section") && container.querySelector("p") &&
+            (product || AD_SLOT_LABEL.test(text));
+        const next = container.nextElementSibling;
+        const nextHasMedia = next && (next.matches("img, picture, iframe, video, audio, a[href]") ||
+            next.querySelector("img, picture, iframe, video, audio, a[href]"));
+        if (emptySlot || (isolated && !nextHasMedia))
+            drop(container, product ? "empty-product-label" : "empty-ad-label");
+    }
     // An ad label whose ad is not there (a script fills it): sponsored content itself stays, the label above nothing goes.
     const labelled = leafBlocks(root);
     labelled.forEach((block, index) => {
@@ -362,13 +389,14 @@ export function cleanReaderHtml(html, title, stats = {}, author) {
         return html;
     const root = JSDOM.fragment(html);
     const images = resolveCustomImages(root, stats);
+    const duplicates = collapseAdjacentImages(root, stats);
     const variants = collapseImageVariants(root, stats);
     const videos = resolveVideoPlayers(root, stats);
     const noise = removeReaderNoise(root, title, stats, author);
     const typography = removeTypography(root, stats);
     videos.links.forEach(link => { if (root.contains(link))
         link.setAttribute("data-media", "video"); });
-    return images || variants || videos.changed || noise || typography ? serialize(root) : html;
+    return images || duplicates || variants || videos.changed || noise || typography ? serialize(root) : html;
 }
 const PLAY_LABEL = /^(video starten|video abspielen|abspielen|wiedergabe( starten)?|play( video)?|watch( video)?|start video)$/i;
 const PLAY_SOURCE = /(?:^|[/_.-])play[-_]?(?:icon|button|btn)?\.(?:svg|png|webp|gif)(?:[?#]|$)/i;
@@ -539,6 +567,46 @@ function collapseImageVariants(root, stats) {
     }
     return changed;
 }
+/** Responsive copies may be separate images whose CSS normally hides one. Only exact adjacent copies go;
+ * text, other media, different links, captions and responsive sources keep images distinct. */
+function collapseAdjacentImages(root, stats) {
+    const walker = root.ownerDocument.createTreeWalker(root, 1 | 4);
+    let previous = null;
+    const duplicates = [];
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+        if (node.nodeType === 3) {
+            if (flat(node.textContent))
+                previous = null;
+            continue;
+        }
+        const element = node;
+        if (element.matches("video, audio, iframe, object, embed, hr"))
+            previous = null;
+        if (element.tagName !== "IMG")
+            continue;
+        const src = flat(element.getAttribute("data-src") || element.getAttribute("data-lazy-src") ||
+            element.getAttribute("data-original") || element.getAttribute("src"));
+        if (!src || /^(data|blob):/i.test(src)) {
+            previous = null;
+            continue;
+        }
+        const picture = element.closest("picture");
+        const key = JSON.stringify([src, element.getAttribute("srcset"), element.getAttribute("data-srcset"),
+            element.getAttribute("alt"), element.getAttribute("title"), element.closest("a")?.getAttribute("href"),
+            [...(picture?.querySelectorAll("source") ?? [])].map(source => [source.getAttribute("srcset"), source.getAttribute("media"), source.getAttribute("type")])]);
+        if (previous === key)
+            duplicates.push(picture?.querySelectorAll("img").length === 1 ? picture : element);
+        else
+            previous = key;
+    }
+    for (const duplicate of duplicates) {
+        const parent = duplicate.parentNode;
+        duplicate.remove();
+        pruneEmpty(parent);
+        stats["duplicate-image"] = (stats["duplicate-image"] ?? 0) + 1;
+    }
+    return duplicates.length > 0;
+}
 const CREDIT_PATTERNS = [
     /^(©|\(c\)\s|copyright\b)/i,
     /^[([]?(Bild|Foto|Fotos|Bildquelle|Bildrechte|Bildnachweis|Titelbild|Beitragsbild|Photo|Image|Credits?|Grafik|Illustration)[)\]]?\s*[:/|]/i,
@@ -589,7 +657,9 @@ const COMMENT_PATTERNS = [
     /\b(let us know|tell us) .{0,40}in the comments\b/i,
     /\bNur angemeldete Benutzer können kommentieren\b/i,
 ];
-const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|Sponsored( content| post)?|Paid (content|post)|Gesponsert|Promoted)\s*:?$/i;
+const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|Sponsored( content| post)?|Paid (content|post)|Gesponsert|Promoted|広告|広告掲載|スポンサー(?:記事|コンテンツ)?|宣伝|PR)\s*[:：]?$/i;
+const AD_SLOT_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Ads?|広告|広告掲載|宣伝)\s*[:：]?$/i;
+const PRODUCT_LABEL = /^(Produkthinweis|Produktempfehlung|Product (notice|recommendation)|商品紹介|商品案内|製品紹介)\s*[:：]?$/i;
 const COMMENT_RULES_HEADING = /^(Regeln für Kommentare|Kommentarregeln|Kommentar-Richtlinien|Community[- ]Richtlinien|(Comment(ing)?|Community|Discussion|House) (rules|guidelines|policy|policies)|Rules for comments|Commenting rules)\s*:?$/i;
 const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
 const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;

package/server/dist/feeds/fullTextQueue.js

@@ -1,4 +1,5 @@
 import { isCachedConsentSnippet } from "./extractor.js";
+import { refreshLead } from "./lead.js";
 import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
 import { loadFullText, executeFullText } from "./fullText.js";
@@ -344,6 +345,7 @@ export function startFullTextScheduler() {
         for (const item of cached)
             if (isCachedConsentSnippet(item.full_content_html)) {
                 db.prepare("UPDATE items SET full_content_html=NULL,extraction_status='pending',extraction_retry_at=NULL WHERE id=?").run(item.id);
+                refreshLead(item.id);
             }
     })();
     tick();

package/server/dist/feeds/galleryContent.js

@@ -0,0 +1,33 @@
+import { JSDOM } from "jsdom";
+const GALLERY_COMPONENTS = new Set([
+    "bilder", "bildergalerie", "bildergalerien", "fotogalerie", "fotogalerien", "fotostrecke", "fotostrecken",
+    "galerie", "galerien", "gallery", "galleries", "photogallery", "photogalleries", "slideshow", "slideshows",
+    "photos", "pictures", "images",
+]);
+const PLACEHOLDERS = new Set([
+    "", "mehr", "weiterlesen", "mehr lesen", "read more", "more", "continue reading",
+    "zur galerie", "zur bildergalerie", "view gallery", "続きを読む", "もっと読む",
+]);
+function isPlaceholder(source) {
+    const fragment = JSDOM.fragment(source ?? "");
+    fragment.querySelectorAll("script, style, noscript").forEach(node => node.remove());
+    fragment.querySelectorAll("p, div, br, li, h1, h2, h3, blockquote, figcaption").forEach(node => node.append(" "));
+    const normalized = (fragment.textContent ?? "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+    return PLACEHOLDERS.has(normalized);
+}
+/** A gallery path alone is not enough: even one editorial sentence or caption keeps the article visible. */
+export function isEmptyGallery(item) {
+    if (!item.link)
+        return false;
+    let path;
+    try {
+        path = decodeURIComponent(new URL(item.link).pathname).toLowerCase();
+    }
+    catch {
+        return false;
+    }
+    if (!path.split(/[^\p{L}\p{N}]+/u).some(component => GALLERY_COMPONENTS.has(component)))
+        return false;
+    return isPlaceholder(item.content_snippet) && isPlaceholder(item.full_content_html || item.content_html);
+}
+//# sourceMappingURL=galleryContent.js.map

package/server/dist/feeds/lead.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { db } from "../db/index.js";
 import { LEAD_VERSION, leadText } from "./contentNoise.js";
+import { isEmptyGallery } from "./galleryContent.js";
 const BACKFILL_BATCH = 50;
 const BACKFILL_PAUSE_MS = 100;
 const COMPUTED_LIMIT = 300;
@@ -10,10 +11,17 @@ export function computeLead(item) {
 }
 /** Derive and store the lead of one item; call after its title, summary or content was written. */
 export function refreshLead(itemId) {
-    const item = db.prepare("SELECT title, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
+    const item = db.prepare("SELECT title, link, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
     if (!item)
         return;
-    db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?").run(computeLead(item), LEAD_VERSION, itemId);
+    db.prepare("UPDATE items SET content_lead = ?, lead_version = ?, is_empty_gallery = ? WHERE id = ?")
+        .run(computeLead(item), LEAD_VERSION, Number(isEmptyGallery(item)), itemId);
+}
+/** A changed feed summary/link can affect visibility while the lead still belongs to the retained full text. */
+export function refreshGalleryClassification(itemId) {
+    const item = db.prepare("SELECT link, content_snippet, content_html, full_content_html FROM items WHERE id = ?").get(itemId);
+    if (item)
+        db.prepare("UPDATE items SET is_empty_gallery = ? WHERE id = ?").run(Number(isEmptyGallery(item)), itemId);
 }
 /**
  * The lead to serve: the stored one while its rules are current, otherwise derived on the spot, so an article is
@@ -35,10 +43,10 @@ export function leadOf(item) {
 }
 /** Derive leads for items that have none or were derived by older rules; returns how many were handled. */
 export function backfillLeads(limit = BACKFILL_BATCH) {
-    const rows = db.prepare("SELECT id, title, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
-    const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ? WHERE id = ?");
+    const rows = db.prepare("SELECT id, title, link, content_snippet, content_html, full_content_html FROM items WHERE lead_version IS NULL OR lead_version <> ? LIMIT ?").all(LEAD_VERSION, limit);
+    const update = db.prepare("UPDATE items SET content_lead = ?, lead_version = ?, is_empty_gallery = ? WHERE id = ?");
     db.transaction(() => { for (const row of rows)
-        update.run(computeLead(row), LEAD_VERSION, row.id); })();
+        update.run(computeLead(row), LEAD_VERSION, Number(isEmptyGallery(row)), row.id); })();
     return rows.length;
 }
 /** Works through the backlog in small steps so polling and requests are never held up for long. */

package/server/dist/feeds/repository.js

@@ -2,7 +2,7 @@ import { db } from "../db/index.js";
 import { articleText } from "./articleContent.js";
 import { LEAD_VERSION } from "./contentNoise.js";
 import { largerHeroImage } from "./heroImage.js";
-import { refreshLead } from "./lead.js";
+import { refreshGalleryClassification, refreshLead } from "./lead.js";
 import { decodeEntities } from "./text.js";
 export function findFeedByUrl(url) {
     return db.prepare("SELECT * FROM feeds WHERE url = ?").get(url);
@@ -191,10 +191,13 @@ export function upsertItems(feedId, items) {
                     if (changed) {
                         update.run(row.title ?? null, row.link ?? null, nextSnippet, retained, row.publishedAt ?? null, nextImage, row.author ?? null, feedId, row.guid);
                         const leadChanged = (retained !== null && retained !== previous.content_html) ||
+                            (row.link !== undefined && row.link !== previous.link) ||
                             (row.title !== undefined && row.title !== previous.title) ||
                             (!keptPrevious && row.contentSnippet !== undefined && row.contentSnippet !== previous.content_snippet);
                         if (previous.lead_version !== LEAD_VERSION || (leadChanged && !previous.full_content_html))
                             refreshLead(previous.id);
+                        else if (leadChanged)
+                            refreshGalleryClassification(previous.id);
                     }
                     else if (previous.lead_version !== LEAD_VERSION && !previous.full_content_html) {
                         refreshLead(previous.id);
@@ -216,6 +219,10 @@ export function listItemsForUser(userId, opts = {}) {
         conditions.push("(s.id IS NOT NULL OR b.item_id IS NOT NULL)");
     }
     const params = [];
+    // Keep the stored item available to direct reads and sync; deliberate saves/notes override stream filtering.
+    conditions.push(`(i.is_empty_gallery = 0 OR b.item_id IS NOT NULL OR
+    EXISTS (SELECT 1 FROM item_notes gallery_note WHERE gallery_note.item_id = i.id AND gallery_note.user_id = ?))`);
+    params.push(userId);
     if (opts.feedId) {
         conditions.push("i.feed_id = ?");
         params.push(opts.feedId);
@@ -564,6 +571,7 @@ export function repairEncodedText() {
             const snippet = decodeEntities(row.content_snippet);
             if (title !== row.title || snippet !== row.content_snippet) {
                 update.run(title ?? null, snippet ?? null, row.id);
+                refreshLead(row.id);
                 fixed++;
             }
         }
@@ -686,7 +694,8 @@ export function needsSubstringSearch(raw) {
 }
 export function searchItemsForUser(userId, opts) {
     const fts = needsSubstringSearch(opts.query) ? "" : formatFtsQuery(opts.query);
-    const conditions = ["(s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)"];
+    const conditions = ["(s.id IS NOT NULL OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)",
+        "(i.is_empty_gallery = 0 OR b.item_id IS NOT NULL OR n.user_id IS NOT NULL)"];
     const params = [];
     if (opts.feedId) {
         conditions.push("i.feed_id = ?");

package/server/dist/native/editions.js

@@ -108,11 +108,11 @@ export function publishCuratedEdition(userId, input, now = new Date()) {
         for (const id of data.itemIds) {
             if (!canAccessItem(userId, id))
                 throw new NativeError("item_not_found", 404);
-            const item = db.prepare(`SELECT s.folder_id, s.id IS NOT NULL AS subscribed,
+            const item = db.prepare(`SELECT s.folder_id, s.id IS NOT NULL AS subscribed, i.is_empty_gallery,
           EXISTS (SELECT 1 FROM user_muted_keywords m WHERE m.user_id = ? AND
             (instr(lower(coalesce(i.title, '')), lower(m.keyword)) > 0 OR instr(lower(coalesce(i.content_snippet, '')), lower(m.keyword)) > 0)) AS muted
          FROM items i LEFT JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ? WHERE i.id = ?`).get(userId, userId, id);
-            if (item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
+            if (item.is_empty_gallery || item.muted || (item.subscribed && (item.folder_id === null ? !prefs.showUnfiled : prefs.hiddenFolderIds.includes(item.folder_id)))) {
                 throw new NativeError("item_excluded");
             }
         }
@@ -224,7 +224,8 @@ export function editionCandidates(userId, now = new Date(), limit = 200) {
           row_number() OVER (PARTITION BY i.feed_id ORDER BY coalesce(julianday(i.published_at), julianday(i.created_at)) DESC, i.id DESC) AS source_rank
         FROM items i JOIN subscriptions s ON s.feed_id = i.feed_id AND s.user_id = ?
         JOIN feeds f ON f.id = i.feed_id LEFT JOIN folders ON folders.id = s.folder_id
-        WHERE NOT EXISTS (SELECT 1 FROM item_reads r WHERE r.item_id = i.id AND r.user_id = ?)
+        WHERE i.is_empty_gallery = 0
+          AND NOT EXISTS (SELECT 1 FROM item_reads r WHERE r.item_id = i.id AND r.user_id = ?)
           AND NOT EXISTS (SELECT 1 FROM user_muted_keywords m WHERE m.user_id = ? AND
             (instr(lower(coalesce(i.title, '')), lower(m.keyword)) > 0 OR instr(lower(coalesce(i.content_snippet, '')), lower(m.keyword)) > 0))
           AND coalesce(julianday(i.published_at), julianday(i.created_at)) BETWEEN julianday(?) AND julianday(?)

package/server/dist/push.js

@@ -112,6 +112,8 @@ export function newItemsPushPayload(feed, userId, label, latest) {
     const muted = db.prepare("SELECT keyword FROM user_muted_keywords WHERE user_id = ?")
         .all(userId).map(row => row.keyword.toLowerCase());
     const fresh = latest.filter(item => {
+        if (item.is_empty_gallery)
+            return false;
         const text = `${item.title ?? ""} ${item.content_snippet ?? ""}`.toLowerCase();
         return !muted.some(keyword => text.includes(keyword));
     });

package/server/dist/pushOutbox.js

@@ -111,7 +111,7 @@ export class PushOutbox {
         if (!feed)
             return; // Feed deletion cascades the event.
         const latest = db.prepare(`
-      SELECT id,title,content_snippet FROM items WHERE feed_id = ? AND id <= ? ORDER BY id DESC LIMIT ?`)
+      SELECT id,title,content_snippet,is_empty_gallery FROM items WHERE feed_id = ? AND id <= ? ORDER BY id DESC LIMIT ?`)
             .all(event.feed_id, event.max_item_id, event.item_count);
         const payload = newItemsPushPayload(feed, subscription.user_id, subscription.label, latest);
         if (!payload) {

package/server/dist/sync/changeLog.js

@@ -57,6 +57,7 @@ function loadItemContents(userId, itemIds) {
     const rows = db
         .prepare(`SELECT i.id,
               i.content_revision,
+              i.is_empty_gallery,
               CASE WHEN i.full_content_html IS NOT NULL AND i.full_content_html != '' THEN 'reader'
                    WHEN i.content_html IS NOT NULL AND i.content_html != '' THEN 'feed'
                    ELSE 'snippet' END AS source,
@@ -75,6 +76,7 @@ function loadItemContents(userId, itemIds) {
             status: row.extraction_status,
             attemptedAt: row.extraction_attempted_at,
             retryAt: row.extraction_retry_at,
+            isEmptyGallery: Boolean(row.is_empty_gallery),
         });
     }
     return result;

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.27.2",
+  "version": "0.28.1",
   "private": true,
   "type": "module"
 }