feedkeeper 0.31.0 → 0.33.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.31.0",
+  "version": "0.33.0",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/feeds.js

@@ -5,6 +5,7 @@ import { requireCapability } from "../auth/capabilities.js";
 import { listSubscriptionsForUser, findFeedById, isUserSubscribed, reorderSubscriptions } from "../feeds/repository.js";
 import { subscribeToFeed, unsubscribeFromFeed, updateFeedSettings, FeedError, MultipleFeedsFoundError } from "../feeds/service.js";
 import { SsrfBlockedError, normalizeUrlCandidate } from "../feeds/ssrfGuard.js";
+import { blockedUrlBody } from "../feeds/blockedHosts.js";
 import { generateOpml, importOpmlFeeds } from "../feeds/opml.js";
 import { pollFeed, pollFeeds } from "../feeds/poller.js";
 import { discoverFeeds } from "../feeds/discovery.js";
@@ -94,7 +95,7 @@ feedsRouter.post("/discover", requireSync, async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError) {
-            res.status(400).json({ error: "blocked_url", message: error.message });
+            res.status(400).json(blockedUrlBody(error));
             return;
         }
         res.status(400).json({ error: "discovery_failed", message: error instanceof Error ? error.message : String(error) });
@@ -117,7 +118,7 @@ feedsRouter.post("/", requireSync, async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError) {
-            res.status(400).json({ error: "blocked_url", message: error.message });
+            res.status(400).json(blockedUrlBody(error));
             return;
         }
         if (error instanceof MultipleFeedsFoundError) {

package/server/dist/api/v1/resources.js

@@ -15,6 +15,7 @@ import { addMutedKeyword, canAccessItem, createFolder, deleteFolder, deleteItemN
 import { saveToInbox, deleteInboxItem } from "../../feeds/inbox.js";
 import { changeFeedUrl, FeedError, MultipleFeedsFoundError, subscribeToFeed, unsubscribeFromFeed, updateFeedSettings } from "../../feeds/service.js";
 import { SsrfBlockedError, normalizeUrlCandidate } from "../../feeds/ssrfGuard.js";
+import { blockedUrlBody } from "../../feeds/blockedHosts.js";
 import { buildOverview } from "../../mcp/digest.js";
 import { decodeCursor, encodeCursor } from "../../mcp/items.js";
 import { readingPositions, serializeItem, serializeSubscriptionDetail } from "./serialize.js";
@@ -55,7 +56,7 @@ resourcesRouter.post("/subscriptions", async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         if (error instanceof MultipleFeedsFoundError)
             return void res.status(409).json({ error: "multiple_feeds_found", feeds: error.feeds });
         if (error instanceof FeedError)
@@ -100,7 +101,7 @@ resourcesRouter.patch("/subscriptions/:id", async (req, res) => {
         if (error instanceof MultipleFeedsFoundError)
             return void res.status(409).json({ error: "multiple_feeds_found", feeds: error.feeds });
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         if (error instanceof FeedError)
             return void res.status(error.message === "not_subscribed" ? 404 : 400).json({ error: error.message });
         res.status(400).json({ error: "invalid_feed", message: error instanceof Error ? error.message : String(error) });
@@ -167,7 +168,7 @@ resourcesRouter.post("/discover", async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         res.status(400).json({ error: "discovery_failed", message: error instanceof Error ? error.message : String(error) });
     }
 });

package/server/dist/feeds/blockedFeeds.js

@@ -0,0 +1,43 @@
+import { db } from "../db/index.js";
+import { FEED_BLOCKED_ERROR, listBlockedHosts, matchingBlockedHost } from "./blockedHosts.js";
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Applies the block list to what is already stored: feeds on blocked hosts get the error state and lose their articles,
+ * except saved articles, articles with notes and articles in a published edition (the same exceptions as retention).
+ * The subscription stays, so people see why nothing arrives and can remove it themselves.
+ */
+export function purgeBlockedFeeds() {
+    const blocked = listBlockedHosts();
+    if (blocked.length === 0)
+        return { feeds: 0, items: 0 };
+    const feeds = db.prepare("SELECT id, url FROM feeds WHERE is_system_inbox = 0").all()
+        .filter((feed) => {
+        const host = hostOf(feed.url);
+        return host !== null && matchingBlockedHost(host, blocked) !== null;
+    });
+    if (feeds.length === 0)
+        return { feeds: 0, items: 0 };
+    const markFeed = db.prepare("UPDATE feeds SET last_error = ?, etag = NULL, last_modified = NULL WHERE id = ?");
+    const deleteItems = db.prepare(`DELETE FROM items
+     WHERE feed_id = ?
+       AND id NOT IN (SELECT item_id FROM item_bookmarks)
+       AND id NOT IN (SELECT item_id FROM item_notes)
+       AND id NOT IN (SELECT CAST(selected.value AS INTEGER) FROM editions e, json_each(e.item_ids) selected
+         WHERE e.source = 'curated' AND e.status <> 'dismissed')`);
+    let items = 0;
+    db.transaction(() => {
+        for (const feed of feeds) {
+            markFeed.run(FEED_BLOCKED_ERROR, feed.id);
+            items += deleteItems.run(feed.id).changes;
+        }
+    })();
+    return { feeds: feeds.length, items };
+}
+//# sourceMappingURL=blockedFeeds.js.map

package/server/dist/feeds/blockedHosts.js

@@ -0,0 +1,90 @@
+import { assertPublicHttpUrl, normalizeUrlCandidate, SsrfBlockedError } from "./ssrfGuard.js";
+export const FEED_BLOCKED_ERROR = "feed_blocked";
+export const FEED_BLOCKED_MESSAGE = "This source is not available on this server.";
+/** The address is on the operator's block list; subscribing, polling, article text and images are all refused. */
+export class FeedBlockedError extends SsrfBlockedError {
+    constructor(url) {
+        super(url);
+        this.name = "FeedBlockedError";
+        this.message = FEED_BLOCKED_MESSAGE;
+    }
+}
+const LABEL = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
+/**
+ * One block list entry as a lower-case ASCII domain. A single label ("com") is refused because it would block a whole
+ * top-level domain by accident; `*.example.com` and `.example.com` mean the same as `example.com`.
+ */
+export function normalizeBlockedHost(raw) {
+    const entry = raw.trim().replace(/^\*?\./, "");
+    let hostname;
+    try {
+        hostname = new URL(entry.includes("://") ? entry : `https://${entry}`).hostname;
+    }
+    catch {
+        throw new Error(`Invalid blocked host "${raw}"`);
+    }
+    hostname = hostname.replace(/\.$/, "");
+    const labels = hostname.split(".");
+    if (labels.length < 2 || !labels.every((label) => LABEL.test(label))) {
+        throw new Error(`Invalid blocked host "${raw}": expected a domain such as example.com`);
+    }
+    return hostname;
+}
+/** `BLOCKED_FEED_HOSTS`: domains separated by commas, spaces or line breaks. Fails on a bad entry instead of silently allowing it. */
+export function parseBlockedHosts(raw) {
+    if (!raw)
+        return [];
+    const entries = raw.split(/[\s,]+/).filter(Boolean);
+    return [...new Set(entries.map(normalizeBlockedHost))];
+}
+let provider = null;
+let cachedEnv = null;
+/** A host application adds its own list (for example one kept in a database) to the environment list. */
+export function setBlockedHostsProvider(next) {
+    provider = next;
+}
+function environmentHosts() {
+    const raw = process.env.BLOCKED_FEED_HOSTS;
+    if (cachedEnv && cachedEnv.raw === raw)
+        return cachedEnv.hosts;
+    const hosts = parseBlockedHosts(raw);
+    cachedEnv = { raw, hosts };
+    return hosts;
+}
+export function listBlockedHosts() {
+    return [...new Set([...environmentHosts(), ...(provider?.() ?? [])])];
+}
+export function matchingBlockedHost(hostname, blocked = listBlockedHosts()) {
+    if (blocked.length === 0)
+        return null;
+    const set = new Set(blocked);
+    const labels = hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "").split(".");
+    for (let start = 0; start < labels.length; start++) {
+        const candidate = labels.slice(start).join(".");
+        if (set.has(candidate))
+            return candidate;
+    }
+    return null;
+}
+export function isHostBlocked(hostname) {
+    return matchingBlockedHost(hostname) !== null;
+}
+/** The check for every address that is fetched for a feed, an article or an image: public, and not blocked by the operator. */
+export async function assertPublicFeedUrl(rawUrl) {
+    // Before DNS: a blocked source costs no lookup. An unparseable address falls through to its usual error.
+    let hostname;
+    try {
+        hostname = new URL(normalizeUrlCandidate(rawUrl)).hostname;
+    }
+    catch {
+        hostname = null;
+    }
+    if (hostname !== null && isHostBlocked(hostname))
+        throw new FeedBlockedError(rawUrl);
+    return assertPublicHttpUrl(rawUrl);
+}
+/** The JSON body of a refused address; a blocked source has its own code so clients can tell it from an internal address. */
+export function blockedUrlBody(error) {
+    return { error: error instanceof FeedBlockedError ? FEED_BLOCKED_ERROR : "blocked_url", message: error.message };
+}
+//# sourceMappingURL=blockedHosts.js.map

package/server/dist/feeds/content/embeds/headings.js

@@ -0,0 +1,38 @@
+import { count, flat, pruneEmpty } from "../shared.js";
+const HEADINGS = "h2, h3, h4, h5";
+const SCOPE_LEVELS = 4;
+/** The wrapper of a card and its heading is small; a larger block holds the story, and its headings stay. */
+const SCOPE_TEXT_LIMIT = 400;
+const FOLLOWING = 4;
+/**
+ * Some sites put the title of a video or post above its player as a heading that links to a page of their own. Once the player is a
+ * card with the same title, the title stands twice. The heading goes when it holds nothing but that title, is at most one link, and sits
+ * in the small wrapper the card sits in.
+ */
+export const embedHeadings = {
+    name: "embed-headings",
+    description: "A heading that only repeats the title of the video or post card right after it is removed.",
+    apply(root, stats) {
+        let changed = false;
+        for (const card of [...root.querySelectorAll("a[data-media]")]) {
+            const title = flat(card.textContent).toLowerCase();
+            if (title === "")
+                continue;
+            let scope = card.parentElement;
+            for (let level = 0; scope && level < SCOPE_LEVELS && flat(scope.textContent).length <= SCOPE_TEXT_LIMIT; level++, scope = scope.parentElement) {
+                const heading = [...scope.querySelectorAll(HEADINGS)].find(candidate => root.contains(candidate) && !candidate.contains(card) && flat(candidate.textContent).toLowerCase() === title &&
+                    candidate.querySelectorAll("a").length <= 1 && (candidate.compareDocumentPosition(card) & FOLLOWING) !== 0);
+                if (!heading)
+                    continue;
+                const parent = heading.parentNode;
+                heading.remove();
+                pruneEmpty(parent);
+                count(stats, "embed-headings");
+                changed = true;
+                break;
+            }
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=headings.js.map

package/server/dist/feeds/content/embeds/index.js

@@ -1,11 +1,12 @@
 import { consentEmbeds } from "./consent.js";
+import { mediaElements } from "./media.js";
 import { socialEmbeds } from "./social.js";
 import { videoPlayers } from "./video.js";
 /**
  * Embeds, in the order they must run: consent plugins first, because the address of the embed sits inside their
- * notice, then the video players and iframes, then the social posts.
+ * notice, then the video players and iframes, then native audio and video elements, then the social posts.
  */
-export const EMBED_RULES = [consentEmbeds, videoPlayers, socialEmbeds];
+export const EMBED_RULES = [consentEmbeds, videoPlayers, mediaElements, socialEmbeds];
 /**
  * Turns every embed of a page into the marked link a reader shows as a card. A fetched page must go through this
  * before its scripts, frames and `noscript` blocks are removed: that is where consent plugins and players keep the address.

package/server/dist/feeds/content/embeds/media.js

@@ -0,0 +1,45 @@
+import { count, flat, linkParagraph, pruneEmpty } from "../shared.js";
+/** The address a native `<audio>` or `<video>` plays: its own `src`, or the first `<source>` inside it. Anything but http(s) is no address. */
+function mediaAddress(element) {
+    const source = element.getAttribute("src") ?? element.querySelector("source[src]")?.getAttribute("src") ?? "";
+    const address = source.trim();
+    if (address === "")
+        return null;
+    if (/^[a-z][a-z0-9+.-]*:/i.test(address) && !/^https?:/i.test(address))
+        return null;
+    return address;
+}
+/**
+ * A native `<audio>` or `<video>` element needs the player of the reader's engine and the publisher's script to do anything useful: without
+ * them it shows a bar with an error. What names an address becomes one marked link to the file (`data-media="audio" | "video"`), what does
+ * not, because the file is set by a script, is left out.
+ */
+export const mediaElements = {
+    name: "media-elements",
+    description: "Native audio and video elements become one marked link to their file, or are removed when they name no address.",
+    apply(root, stats) {
+        const document = root.ownerDocument ?? root;
+        let changed = false;
+        for (const element of [...root.querySelectorAll("audio, video")]) {
+            if (!root.contains(element))
+                continue;
+            const kind = element.tagName.toLowerCase() === "audio" ? "audio" : "video";
+            const address = mediaAddress(element);
+            const parent = element.parentNode;
+            if (address) {
+                const label = flat(element.getAttribute("title")) || flat(element.getAttribute("aria-label")) || (kind === "audio" ? "Audio" : "Video");
+                const replacement = linkParagraph(document, address, label);
+                replacement.querySelector("a")?.setAttribute("data-media", kind);
+                element.replaceWith(replacement);
+            }
+            else {
+                element.remove();
+                pruneEmpty(parent);
+            }
+            count(stats, "media-elements");
+            changed = true;
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=media.js.map

package/server/dist/feeds/content/index.js

@@ -1,11 +1,15 @@
 import { EMBED_RULES, resolveEmbeds } from "./embeds/index.js";
+import { embedHeadings } from "./embeds/headings.js";
+import { boilerplate } from "./noise/boilerplate.js";
+import { loadingPlaceholders } from "./noise/loading.js";
+import { trackingPixels } from "./noise/pixels.js";
 import { sidebars } from "./noise/sidebars.js";
 /**
  * Every rule of the content pipeline, in the order they run on a fetched page: embeds first (their address sits in
  * tags that are removed next), then the clutter around the story. The tests check each rule against its fixtures in
  * `test/fixtures/content/<rule name>` and that running it twice changes nothing. See `docs/content-rules.md`.
  */
-export const PAGE_RULES = [...EMBED_RULES, sidebars];
+export const PAGE_RULES = [...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate];
 export { resolveEmbeds };
 /** What a fetched page goes through before its scripts, frames and `noscript` blocks are removed. */
 export function cleanPage(root, stats = {}) {

package/server/dist/feeds/content/noise/boilerplate.js

@@ -0,0 +1,106 @@
+import { BOILERPLATE_ANYWHERE, BOILERPLATE_TRAILING } from "../patterns/boilerplate/index.js";
+import { count, flat, pruneEmpty } from "../shared.js";
+const BLOCKS = "p, div, li, h2, h3, h4, h5, h6, span";
+const NESTED = "p, div, li, ul, ol, h2, h3, h4, h5, h6";
+/** Footers can only sit in the second half of the text: before it, the same words are part of the story. */
+const AFTER_SHARE = 0.5;
+/** A list after a removed box ("Follow topics and authors …") that holds names, or nothing at all, belongs to the box. */
+const LIST_ITEM_LIMIT = 60;
+function textBefore(root, node) {
+    const range = root.ownerDocument?.createRange() ?? root.createRange();
+    range.setStart(root, 0);
+    range.setEndBefore(node);
+    return flat(range.toString()).length;
+}
+/** The list that directly follows a block, when it only holds short names or empty items. */
+function followingNameList(block) {
+    const next = block.nextElementSibling;
+    if (!next || !/^(UL|OL)$/.test(next.tagName))
+        return null;
+    const items = [...next.querySelectorAll("li")];
+    return items.length > 0 && items.every(item => flat(item.textContent).length <= LIST_ITEM_LIMIT) ? next : null;
+}
+/** Elements that start a line of their own; a run of text between two `<br>` that holds one of them is not a plain line. */
+const LINE_BREAKING = `${NESTED}, img, picture, video, audio, iframe, table, blockquote, figure`;
+/** The text nodes and inline elements of a container, cut at every `<br>`. */
+function linesOf(container) {
+    const lines = [[]];
+    container.childNodes.forEach(node => {
+        if (node.nodeName === "BR")
+            lines.push([]);
+        else
+            lines[lines.length - 1].push(node);
+    });
+    return lines;
+}
+function isPlainLine(nodes) {
+    return nodes.every(node => node.nodeType === 3 || (node.nodeType === 1 && !node.matches(LINE_BREAKING) && !node.querySelector(LINE_BREAKING)));
+}
+/**
+ * Lines that sites add to their own text: "Follow us on Google", the words of a download button, the file name of a picture shown as text,
+ * "This article first appeared on …", the related links under a MacRumors story. A block goes when its whole text is one of these lines
+ * (footers only in the second half), together with the list of names that follows a "follow topics" box.
+ */
+export const boilerplate = {
+    name: "boilerplate",
+    description: "Blocks that only hold a follow-us call, a download button text, an image file name or an \"appeared first on\" footer are removed.",
+    apply(root, stats) {
+        let changed = false;
+        const total = flat(root.textContent).length;
+        for (const block of [...root.querySelectorAll(BLOCKS)]) {
+            if (!root.contains(block) || block.querySelector("img, picture, video, audio, iframe") || block.querySelector(NESTED))
+                continue;
+            const text = flat(block.textContent);
+            if (text === "" || text.length > 400)
+                continue;
+            const anywhere = BOILERPLATE_ANYWHERE.some(pattern => pattern.test(text));
+            const trailing = !anywhere && total > 0 && textBefore(root, block) / total >= AFTER_SHARE && BOILERPLATE_TRAILING.some(pattern => pattern.test(text));
+            if (!anywhere && !trailing)
+                continue;
+            const parent = block.parentNode;
+            followingNameList(block)?.remove();
+            block.remove();
+            pruneEmpty(parent);
+            count(stats, "boilerplate");
+            changed = true;
+        }
+        return lineFooters(root, stats, total) || changed;
+    },
+};
+/**
+ * Footers that stand as bare text between `<br>` ("This article … first appeared on …", "Discuss this article in our forums"), not in a block of
+ * their own. Each such line is checked like a block; the breaks left at the end of the container go with them.
+ */
+function lineFooters(root, stats, total) {
+    let changed = false;
+    const containers = [root, ...root.querySelectorAll("div, p, li, td, section, article, main, blockquote")];
+    for (const container of containers) {
+        if (![...container.childNodes].some(node => node.nodeName === "BR"))
+            continue;
+        let removed = false;
+        for (const line of linesOf(container)) {
+            if (line.length === 0 || !isPlainLine(line))
+                continue;
+            const text = flat(line.map(node => node.textContent ?? "").join(""));
+            if (text === "" || text.length > 400)
+                continue;
+            const anywhere = BOILERPLATE_ANYWHERE.some(pattern => pattern.test(text));
+            const trailing = !anywhere && total > 0 && textBefore(root, line[0]) / total >= AFTER_SHARE && BOILERPLATE_TRAILING.some(pattern => pattern.test(text));
+            if (!anywhere && !trailing)
+                continue;
+            const after = line[line.length - 1].nextSibling;
+            line.forEach(node => node.parentNode?.removeChild(node));
+            if (after?.nodeName === "BR")
+                after.parentNode?.removeChild(after);
+            count(stats, "boilerplate");
+            removed = true;
+        }
+        if (!removed)
+            continue;
+        while (container.lastChild && (container.lastChild.nodeName === "BR" || (container.lastChild.nodeType === 3 && flat(container.lastChild.textContent) === "")))
+            container.removeChild(container.lastChild);
+        changed = true;
+    }
+    return changed;
+}
+//# sourceMappingURL=boilerplate.js.map

package/server/dist/feeds/content/noise/loading.js

@@ -0,0 +1,29 @@
+import { LOADING_LABELS } from "../patterns/loading/index.js";
+import { count, flat, pruneEmpty } from "../shared.js";
+/**
+ * A script fills these blocks with content ("wird geladen …", "Loading…"). A reader does not run it, so the placeholder would stay in the
+ * story as a line of text. Only a block that holds nothing but the label goes: no link, no picture, no other words.
+ */
+export const loadingPlaceholders = {
+    name: "loading-placeholders",
+    description: "Blocks that hold only a loading label such as \"wird geladen …\" or \"Loading…\" are removed.",
+    apply(root, stats) {
+        let changed = false;
+        for (const block of [...root.querySelectorAll("p, div, span, li")]) {
+            if (!root.contains(block) || block.querySelector("a[href], img, picture, video, audio, iframe"))
+                continue;
+            if (!LOADING_LABELS.some(pattern => pattern.test(flat(block.textContent))))
+                continue;
+            // The innermost block that holds the label: a wrapper with more blocks inside is looked at on its own.
+            if (block.querySelector("p, div, li"))
+                continue;
+            const parent = block.parentNode;
+            block.remove();
+            pruneEmpty(parent);
+            count(stats, "loading-placeholders");
+            changed = true;
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=loading.js.map

package/server/dist/feeds/content/noise/pixels.js

@@ -0,0 +1,27 @@
+import { count, pruneEmpty } from "../shared.js";
+const LIMIT = 2;
+function isPixel(image) {
+    const width = Number(image.getAttribute("width"));
+    const height = Number(image.getAttribute("height"));
+    return width > 0 && height > 0 && width <= LIMIT && height <= LIMIT;
+}
+/**
+ * Counting pixels (VG Wort, Chartbeat, publisher analytics) arrive as pictures of one by one pixel inside the text. They show nothing, only
+ * a gap in some readers, and every client would load them: they go.
+ */
+export const trackingPixels = {
+    name: "tracking-pixels",
+    description: "Pictures of one or two pixels in each direction, which only count a view, are removed.",
+    apply(root, stats) {
+        let changed = false;
+        for (const image of [...root.querySelectorAll("img")].filter(isPixel)) {
+            const parent = image.parentNode;
+            image.remove();
+            pruneEmpty(parent);
+            count(stats, "tracking-pixels");
+            changed = true;
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=pixels.js.map

package/server/dist/feeds/content/patterns/boilerplate/de.js

@@ -0,0 +1,22 @@
+/**
+ * Blocks, in German, that publishers put into their text without being part of the story. A block counts only when its whole text is one
+ * of these lines (see `noise/boilerplate.ts`), so a sentence of the story that contains the words stays.
+ */
+export const ANYWHERE = [
+    // Aufrufe, der Seite bei Google oder in anderen Diensten zu folgen.
+    /^(?:Uns|Folge uns|Folgen Sie uns) (?:auf|bei) Google(?: News| Discover)? folgen$/i,
+    /^(?:Wähle|Wählen Sie|Füge|Fügen Sie) [^.!?]{2,50} (?:als )?(?:bevorzugte|bevorzugten|bevorzugter)\b[^.!?]{0,30}$/i,
+    // „★ ifun.de bei Google bevorzugen ★“ und ähnliche Bitten, die Seite als bevorzugte Quelle zu wählen.
+    /^★?\s*[^★!?]{2,40} bei Google bevorzugen\s*★?$/i,
+    // Texte von Download-Schaltflächen, deren Grafik nicht mehr da ist.
+    /^(?:Laden im (?:App|Mac App|Google Play) Store|Im (?:App|Mac App) Store laden|Jetzt bei Google Play|Bei Google Play herunterladen|Laden bei Google Play)$/i,
+    // Der Dateiname eines Bildes, den ein Link oder eine Beschriftung als Text zeigt.
+    /^[\w\-. ]{3,120}\.(?:jpe?g|png|webp|gif|avif)$/i,
+];
+/** Fußzeilen am Ende des Textes; sie zählen nur in der zweiten Hälfte. */
+export const TRAILING = [
+    /^(?:Der Beitrag|Dieser Beitrag|Der Artikel|Dieser Artikel) .{3,200} (?:erschien|erschien zuerst|erschien erstmals) (?:zuerst )?(?:auf|bei) .{2,60}$/i,
+    /^Der Beitrag .{3,200} wurde (?:zuerst|erstmals) (?:auf|bei) .{2,60} veröffentlicht\.?$/i,
+    /^Diskutiere (?:diesen|den) (?:Artikel|Beitrag) .{0,40}$/i,
+];
+//# sourceMappingURL=de.js.map

package/server/dist/feeds/content/patterns/boilerplate/en.js

@@ -0,0 +1,21 @@
+/**
+ * Blocks, in English, that publishers put into their text without being part of the story. A block counts only when its whole text is one
+ * of these lines (see `noise/boilerplate.ts`), so a sentence of the story that contains the words stays.
+ */
+export const ANYWHERE = [
+    // Calls to follow the site on Google or elsewhere.
+    /^(?:Follow us on Google(?: News| Discover)?|Add [^.!?]{2,50} as a preferred (?:source|publisher)(?: on Google)?)$/i,
+    // The text of download buttons whose picture is gone.
+    /^(?:Download on the (?:App|Mac App) Store|Get it on Google Play|Available on Google Play)$/i,
+];
+/** Footers at the end of the text; they count only in the second half. */
+export const TRAILING = [
+    /^(?:This (?:article|post)|The post),? .{3,200} (?:first )?appeared (?:first )?on .{2,60}$/i,
+    /^Discuss this (?:article|story|post)\b.{0,40}$/i,
+    // MacRumors: the related links under every article.
+    /^Related Roundups?:.{0,300}$/i,
+    /^(?:Tag|Tags|Buyer's Guides?|Related Forums?):.{0,200}$/i,
+    // The Verge: the box that offers to follow the story's topics and authors.
+    /^Follow topics and authors from this story\b.{0,200}$/i,
+];
+//# sourceMappingURL=en.js.map

package/server/dist/feeds/content/patterns/boilerplate/index.js

@@ -0,0 +1,6 @@
+import * as de from "./de.js";
+import * as en from "./en.js";
+/** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
+export const BOILERPLATE_ANYWHERE = [de.ANYWHERE, en.ANYWHERE].flat();
+export const BOILERPLATE_TRAILING = [de.TRAILING, en.TRAILING].flat();
+//# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/loading/de.js

@@ -0,0 +1,5 @@
+/** Placeholders, in German, that a script replaces with content. Without the script they stay in the text. See `en.ts`. */
+export default [
+    /^(?:wird geladen|lädt|bitte warten)(?:\s*(?:\.{3}|…))?$/i,
+];
+//# sourceMappingURL=de.js.map

package/server/dist/feeds/content/patterns/loading/en.js

@@ -0,0 +1,8 @@
+/**
+ * Whole-line placeholders that a script replaces with content (a comment box, a gallery, a player). A reader that does not run the
+ * script keeps them as text. Only a block that holds nothing else counts, so a sentence that contains the word stays.
+ */
+export default [
+    /^(?:loading|please wait)(?:\s*(?:\.{3}|…))?$/i,
+];
+//# sourceMappingURL=en.js.map

package/server/dist/feeds/content/patterns/loading/index.js

@@ -0,0 +1,5 @@
+import de from "./de.js";
+import en from "./en.js";
+/** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
+export const LOADING_LABELS = [de, en].flat();
+//# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/sidebars/de.js

@@ -1,5 +1,7 @@
 /** Headings, in German, that start a list of other articles. See `en.ts` for how they are used. */
 export default [
     /^(?:Mehr zum Thema|Weitere Artikel|Weitere Meldungen|Ähnliche Artikel|Verwandte Artikel|Auch interessant|Das könnte (?:dich|Sie|euch) (?:auch )?interessieren|Meistgelesen(?:e Artikel)?|Beliebte Artikel|Lesen Sie auch|Lies auch|Empfehlungen)$/i,
+    // Das Willkommen einer Seite („Willkommen bei GamePro!“) mit Logo und Links am Ende des Textes.
+    /^Willkommen bei [^.!?]{2,40}!?$/i,
 ];
 //# sourceMappingURL=de.js.map

package/server/dist/feeds/content/patterns/sidebars/en.js

@@ -4,5 +4,7 @@
  */
 export default [
     /^(?:Popular (?:Stories|Articles|Posts)|Most (?:Read|Popular)|Trending(?: Now| Stories)?|Related (?:Articles|Stories|Posts|Reading|Content)|You (?:might|may) also like|More (?:from|stories|articles)\b.*|Read (?:more|next)|Further reading|Top Stories|Recommended(?: for you)?|Latest (?:Stories|News))$/i,
+    // A site's welcome ("Welcome to GamePro!") with logo and links at the end of the text.
+    /^Welcome to [^.!?]{2,40}!?$/i,
 ];
 //# sourceMappingURL=en.js.map

package/server/dist/feeds/discovery.js

@@ -1,6 +1,6 @@
 import Parser from "rss-parser";
 import { fetchFeed } from "./fetcher.js";
-import { assertPublicHttpUrl } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { decodeEntities, plainTitle } from "./text.js";
 const rssParser = new Parser();
 // Common alternate feed types found in HTML <head>
@@ -74,7 +74,7 @@ async function keepRealFeeds(candidates, pageUrl) {
  * or if it is an HTML page that links to one or more feeds (auto-discovery).
  */
 export async function discoverFeeds(inputUrl) {
-    const validated = await assertPublicHttpUrl(inputUrl);
+    const validated = await assertPublicFeedUrl(inputUrl);
     const targetUrl = validated.toString();
     // 1. The URL is already a feed.
     const page = await parseFeedAt(targetUrl);

package/server/dist/feeds/extractor.js

@@ -2,7 +2,8 @@ import { Readability } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import { fetch as undiciFetch } from "undici";
 import { cleanPage } from "./content/index.js";
-import { assertPublicHttpUrl, createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { articleWorkPool, ArticleHostBusyError, ArticleAccessDeniedError } from "./articleWorkPool.js";
 import { parseRetryAfter } from "./fetcher.js";
 import { sourceHost } from "./requestGate.js";
@@ -130,7 +131,7 @@ async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl, ownerSign
     let currentUrl = rawUrl;
     for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
         articleWorkPool().move(ownerSignal, sourceHost(currentUrl));
-        const safeUrl = await assertPublicHttpUrl(currentUrl);
+        const safeUrl = await assertPublicFeedUrl(currentUrl);
         signal.throwIfAborted();
         const response = await fetchImpl(safeUrl, {
             signal,

package/server/dist/feeds/feedIcon.js

@@ -31,9 +31,13 @@ export function canAccessFeed(userId, feedId) {
            OR EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = f.id AND b.user_id = ?)
            OR EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = f.id AND n.user_id = ?))`).get(feedId, userId, userId, userId));
 }
-function candidates(feed, declaredUrls) {
+/**
+ * Declared icons first, then the stored one, then the well-known files of the site, and last the feed's own logo (`<image>` of an RSS channel):
+ * a channel logo is often a wide or transparent banner and no icon, so it only serves when the site offers nothing better.
+ */
+function candidates(feed, declaredUrls, logoUrls) {
     const urls = new Set(declaredUrls);
-    if (feed.icon_url)
+    if (feed.icon_url && !logoUrls.includes(feed.icon_url))
         urls.add(feed.icon_url);
     for (const source of [feed.site_url, feed.url]) {
         try {
@@ -47,6 +51,8 @@ function candidates(feed, declaredUrls) {
             // Try the next source.
         }
     }
+    for (const url of logoUrls)
+        urls.add(url);
     return [...urls];
 }
 /** Rejects 8-bit grayscale+alpha or RGBA PNGs whose alpha channel is 0 across every pixel. */
@@ -160,7 +166,7 @@ export function storeFeedIcon(feedId, buffer, mime) {
 }
 /** Re-check remote candidates during the weekly feed poll, even if a stored icon exists. */
 const refreshing = new Map();
-export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
+export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal, logoUrls = []) {
     const current = refreshing.get(feedId);
     if (current)
         return current;
@@ -169,16 +175,16 @@ export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(new DOMException("Icon deadline exceeded", "TimeoutError")), readPollSettings().attemptTimeoutMs);
     const deadline = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal;
-    const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline).finally(() => {
+    const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline, logoUrls).finally(() => {
         clearTimeout(timeout);
         refreshing.delete(feedId);
     });
     refreshing.set(feedId, promise);
     return promise;
 }
-async function refreshIcon(feedId, siteUrl, declaredUrls, signal) {
+async function refreshIcon(feedId, siteUrl, declaredUrls, signal, logoUrls) {
     const feed = db.prepare("SELECT icon_url, site_url, url FROM feeds WHERE id = ?").get(feedId);
-    for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls) : []) {
+    for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls, logoUrls) : []) {
         if (signal.aborted)
             break;
         try {

package/server/dist/feeds/fetcher.js

@@ -1,5 +1,6 @@
 import { fetch as undiciFetch } from "undici";
-import { assertPublicHttpUrl, createPublicDispatcher, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { createPublicDispatcher, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { APP_VERSION } from "../version.js";
 import { feedRequests, sourceHost } from "./requestGate.js";
 import { readPollSettings } from "./pollSettings.js";
@@ -72,7 +73,7 @@ async function validateUrl(url, signal) {
     return new Promise((resolve, reject) => {
         const abort = () => reject(signal.reason);
         signal.addEventListener("abort", abort, { once: true });
-        assertPublicHttpUrl(url).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
+        assertPublicFeedUrl(url).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
     });
 }
 // Re-validate every redirect hop and bound headers, body, DNS and permit waiting together.

package/server/dist/feeds/inbox.js

@@ -4,7 +4,8 @@ import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
 import { configureFullTextPool, deferFullTextHost } from "./fullTextQueue.js";
 import { extractArticle } from "./extractor.js";
-import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { normalizeUrlCandidate } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { articleImageUrls, pruneArchive, scheduleArchive } from "./archive.js";
 import { findFeedById, findFeedByUrl, isUserSubscribed, findSubscriptionForUser, subscribe, bookmarkItem, findItemById, findItemForUser, setItemNote, } from "./repository.js";
 function escapeHtml(str) {
@@ -47,7 +48,7 @@ export async function saveToInbox(userId, input) {
     let imageUrl = null;
     if (input.url?.trim()) {
         const normalized = normalizeUrlCandidate(input.url.trim());
-        await assertPublicHttpUrl(normalized);
+        await assertPublicFeedUrl(normalized);
         targetUrl = normalized;
         if (!contentHtml && hasUserCapability(userId, "fulltext")) {
             try {

package/server/dist/feeds/opml.js

@@ -2,7 +2,8 @@ import { XMLParser } from "fast-xml-parser";
 import { z } from "zod";
 import { config } from "../config.js";
 import { listSubscriptionsForUser, findFeedByUrl, createFeed, isUserSubscribed, subscribe, createFolder, } from "./repository.js";
-import { assertPublicHttpUrl, SsrfBlockedError } from "./ssrfGuard.js";
+import { SsrfBlockedError } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { requestFeedPoll } from "./poller.js";
 import { decodeEntities } from "./text.js";
 function escapeXml(unsafe) {
@@ -146,7 +147,7 @@ export async function importOpmlFeeds(userId, xmlContent) {
         }
         let normalizedUrl;
         try {
-            const validated = await assertPublicHttpUrl(item.url);
+            const validated = await assertPublicFeedUrl(item.url);
             normalizedUrl = validated.toString();
         }
         catch (error) {

package/server/dist/feeds/pollQueue.js

@@ -3,6 +3,7 @@ import { setImmediate as yieldTurn } from "node:timers/promises";
 import { db } from "../db/index.js";
 import { findFeedById, updateFeedAfterPoll } from "./repository.js";
 import { FeedHttpError } from "./fetcher.js";
+import { FEED_BLOCKED_ERROR, FeedBlockedError } from "./blockedHosts.js";
 import { feedRequests, PollDeferredError, PollShutdownError, RequestQueueFullError, sourceHost } from "./requestGate.js";
 import { readPollSettings } from "./pollSettings.js";
 import { pushOutboxMetrics } from "../pushOutbox.js";
@@ -17,6 +18,8 @@ function isTransient(error) {
         : error instanceof TypeError || error instanceof RequestQueueFullError || (error instanceof Error && ["TimeoutError", "AbortError"].includes(error.name));
 }
 export function pollErrorCode(error) {
+    if (error instanceof FeedBlockedError)
+        return FEED_BLOCKED_ERROR;
     if (error instanceof FeedHttpError)
         return `feed_http_${error.status}`;
     if (error instanceof Error && ["TimeoutError", "AbortError"].includes(error.name))

package/server/dist/feeds/poller.js

@@ -34,24 +34,26 @@ const parser = new Parser({
         ],
     },
 });
+/** The feed's own addresses: `icon` is meant as an icon, the `image` of an RSS channel is its logo and ranks below the site's icon files. */
 function extractFeedIconUrls(parsed, baseUrl) {
+    const result = { icons: [], logos: [] };
     if (!parsed)
-        return [];
-    const urls = [];
-    for (const raw of [parsed.icon, parsed.image?.url]) {
+        return result;
+    const read = (raw, into) => {
         if (typeof raw !== "string" || !raw.trim())
-            continue;
+            return;
         try {
             const resolved = new URL(raw.trim(), baseUrl);
-            if (resolved.protocol === "https:" || resolved.protocol === "http:") {
-                urls.push(resolved.toString());
-            }
+            if (resolved.protocol === "https:" || resolved.protocol === "http:")
+                into.push(resolved.toString());
         }
         catch {
             // Ignore malformed feed icon URLs.
         }
-    }
-    return urls;
+    };
+    read(parsed.icon, result.icons);
+    read(parsed.image?.url, result.logos);
+    return result;
 }
 function cleanImageUrl(url) {
     return url.trim().replace(/&/g, "&").replace(/&/g, "&");
@@ -157,7 +159,7 @@ async function pollAttempt(feed, signal) {
             const htmlUrls = siteUrl ? await discoverIconUrls(siteUrl, signal) : [];
             const feedUrls = extractFeedIconUrls(parsed, siteUrl || feed.url);
             signal.throwIfAborted();
-            await refreshFeedIcon(feed.id, siteUrl, [...htmlUrls, ...feedUrls], signal);
+            await refreshFeedIcon(feed.id, siteUrl, [...htmlUrls, ...feedUrls.icons], signal, feedUrls.logos);
         }
     }
     catch { /* Keep the successful poll; icon failures are best effort. */ }

package/server/dist/feeds/service.js

@@ -1,7 +1,7 @@
 import { config } from "../config.js";
 import { createFeed, findFeedByUrl, isUserSubscribed, subscribe, unsubscribe as unsubscribeRepo, findSubscriptionForUser, updateSubscriptionLabel, updateSubscriptionFolder, updateFeedPollInterval, updateSubscriptionFullTextMode, updateSubscriptionNotify, updateSubscriptionBadge, findFolderById, findFeedById, countFeedSubscribers, replaceFeedUrl, moveSubscription, } from "./repository.js";
 import { pollFeed, probeFeed } from "./poller.js";
-import { assertPublicHttpUrl } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { discoverFeeds } from "./discovery.js";
 export class FeedError extends Error {
 }
@@ -21,7 +21,7 @@ export class MultipleFeedsFoundError extends FeedError {
 }
 /** Turn a feed or website URL into a feed URL, using auto-discovery for website addresses. */
 async function resolveFeedUrl(inputUrl) {
-    const validated = await assertPublicHttpUrl(inputUrl.trim());
+    const validated = await assertPublicFeedUrl(inputUrl.trim());
     const targetUrl = validated.toString();
     // A known shared feed needs no second discovery download beside its queued poll.
     if (findFeedByUrl(targetUrl))

package/server/dist/server.js

@@ -20,12 +20,16 @@ import { repairEncodedText } from "./feeds/repository.js";
 import { pruneArchive, scheduleMissingArchives } from "./feeds/archive.js";
 import { startNativeRequestCleanup } from "./native/requests.js";
 import { startLeadBackfill } from "./feeds/lead.js";
+import { purgeBlockedFeeds } from "./feeds/blockedFeeds.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const WEB_DIST = join(__dirname, "../../web/dist");
 runMigrations();
 const repairedItems = repairEncodedText();
 if (repairedItems > 0)
     console.log(`[db] decoded HTML entities in ${repairedItems} stored items`);
+const blockedFeeds = purgeBlockedFeeds();
+if (blockedFeeds.feeds > 0)
+    console.log(`[feeds] ${blockedFeeds.feeds} feeds are on the block list; removed ${blockedFeeds.items} of their articles`);
 startLeadBackfill();
 pruneArchive();
 const pendingArchives = scheduleMissingArchives();

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.31.0",
+  "version": "0.33.0",
   "private": true,
   "type": "module"
 }