feedkeeper 0.31.0 → 0.32.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/feeds.js

@@ -5,6 +5,7 @@ import { requireCapability } from "../auth/capabilities.js";
 import { listSubscriptionsForUser, findFeedById, isUserSubscribed, reorderSubscriptions } from "../feeds/repository.js";
 import { subscribeToFeed, unsubscribeFromFeed, updateFeedSettings, FeedError, MultipleFeedsFoundError } from "../feeds/service.js";
 import { SsrfBlockedError, normalizeUrlCandidate } from "../feeds/ssrfGuard.js";
+import { blockedUrlBody } from "../feeds/blockedHosts.js";
 import { generateOpml, importOpmlFeeds } from "../feeds/opml.js";
 import { pollFeed, pollFeeds } from "../feeds/poller.js";
 import { discoverFeeds } from "../feeds/discovery.js";
@@ -94,7 +95,7 @@ feedsRouter.post("/discover", requireSync, async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError) {
-            res.status(400).json({ error: "blocked_url", message: error.message });
+            res.status(400).json(blockedUrlBody(error));
             return;
         }
         res.status(400).json({ error: "discovery_failed", message: error instanceof Error ? error.message : String(error) });
@@ -117,7 +118,7 @@ feedsRouter.post("/", requireSync, async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError) {
-            res.status(400).json({ error: "blocked_url", message: error.message });
+            res.status(400).json(blockedUrlBody(error));
             return;
         }
         if (error instanceof MultipleFeedsFoundError) {

package/server/dist/api/v1/resources.js

@@ -15,6 +15,7 @@ import { addMutedKeyword, canAccessItem, createFolder, deleteFolder, deleteItemN
 import { saveToInbox, deleteInboxItem } from "../../feeds/inbox.js";
 import { changeFeedUrl, FeedError, MultipleFeedsFoundError, subscribeToFeed, unsubscribeFromFeed, updateFeedSettings } from "../../feeds/service.js";
 import { SsrfBlockedError, normalizeUrlCandidate } from "../../feeds/ssrfGuard.js";
+import { blockedUrlBody } from "../../feeds/blockedHosts.js";
 import { buildOverview } from "../../mcp/digest.js";
 import { decodeCursor, encodeCursor } from "../../mcp/items.js";
 import { readingPositions, serializeItem, serializeSubscriptionDetail } from "./serialize.js";
@@ -55,7 +56,7 @@ resourcesRouter.post("/subscriptions", async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         if (error instanceof MultipleFeedsFoundError)
             return void res.status(409).json({ error: "multiple_feeds_found", feeds: error.feeds });
         if (error instanceof FeedError)
@@ -100,7 +101,7 @@ resourcesRouter.patch("/subscriptions/:id", async (req, res) => {
         if (error instanceof MultipleFeedsFoundError)
             return void res.status(409).json({ error: "multiple_feeds_found", feeds: error.feeds });
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         if (error instanceof FeedError)
             return void res.status(error.message === "not_subscribed" ? 404 : 400).json({ error: error.message });
         res.status(400).json({ error: "invalid_feed", message: error instanceof Error ? error.message : String(error) });
@@ -167,7 +168,7 @@ resourcesRouter.post("/discover", async (req, res) => {
     }
     catch (error) {
         if (error instanceof SsrfBlockedError)
-            return void res.status(400).json({ error: "blocked_url", message: error.message });
+            return void res.status(400).json(blockedUrlBody(error));
         res.status(400).json({ error: "discovery_failed", message: error instanceof Error ? error.message : String(error) });
     }
 });

package/server/dist/feeds/blockedFeeds.js

@@ -0,0 +1,43 @@
+import { db } from "../db/index.js";
+import { FEED_BLOCKED_ERROR, listBlockedHosts, matchingBlockedHost } from "./blockedHosts.js";
+function hostOf(url) {
+    try {
+        return new URL(url).hostname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Applies the block list to what is already stored: feeds on blocked hosts get the error state and lose their articles,
+ * except saved articles, articles with notes and articles in a published edition (the same exceptions as retention).
+ * The subscription stays, so people see why nothing arrives and can remove it themselves.
+ */
+export function purgeBlockedFeeds() {
+    const blocked = listBlockedHosts();
+    if (blocked.length === 0)
+        return { feeds: 0, items: 0 };
+    const feeds = db.prepare("SELECT id, url FROM feeds WHERE is_system_inbox = 0").all()
+        .filter((feed) => {
+        const host = hostOf(feed.url);
+        return host !== null && matchingBlockedHost(host, blocked) !== null;
+    });
+    if (feeds.length === 0)
+        return { feeds: 0, items: 0 };
+    const markFeed = db.prepare("UPDATE feeds SET last_error = ?, etag = NULL, last_modified = NULL WHERE id = ?");
+    const deleteItems = db.prepare(`DELETE FROM items
+     WHERE feed_id = ?
+       AND id NOT IN (SELECT item_id FROM item_bookmarks)
+       AND id NOT IN (SELECT item_id FROM item_notes)
+       AND id NOT IN (SELECT CAST(selected.value AS INTEGER) FROM editions e, json_each(e.item_ids) selected
+         WHERE e.source = 'curated' AND e.status <> 'dismissed')`);
+    let items = 0;
+    db.transaction(() => {
+        for (const feed of feeds) {
+            markFeed.run(FEED_BLOCKED_ERROR, feed.id);
+            items += deleteItems.run(feed.id).changes;
+        }
+    })();
+    return { feeds: feeds.length, items };
+}
+//# sourceMappingURL=blockedFeeds.js.map

package/server/dist/feeds/blockedHosts.js

@@ -0,0 +1,90 @@
+import { assertPublicHttpUrl, normalizeUrlCandidate, SsrfBlockedError } from "./ssrfGuard.js";
+export const FEED_BLOCKED_ERROR = "feed_blocked";
+export const FEED_BLOCKED_MESSAGE = "This source is not available on this server.";
+/** The address is on the operator's block list; subscribing, polling, article text and images are all refused. */
+export class FeedBlockedError extends SsrfBlockedError {
+    constructor(url) {
+        super(url);
+        this.name = "FeedBlockedError";
+        this.message = FEED_BLOCKED_MESSAGE;
+    }
+}
+const LABEL = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
+/**
+ * One block list entry as a lower-case ASCII domain. A single label ("com") is refused because it would block a whole
+ * top-level domain by accident; `*.example.com` and `.example.com` mean the same as `example.com`.
+ */
+export function normalizeBlockedHost(raw) {
+    const entry = raw.trim().replace(/^\*?\./, "");
+    let hostname;
+    try {
+        hostname = new URL(entry.includes("://") ? entry : `https://${entry}`).hostname;
+    }
+    catch {
+        throw new Error(`Invalid blocked host "${raw}"`);
+    }
+    hostname = hostname.replace(/\.$/, "");
+    const labels = hostname.split(".");
+    if (labels.length < 2 || !labels.every((label) => LABEL.test(label))) {
+        throw new Error(`Invalid blocked host "${raw}": expected a domain such as example.com`);
+    }
+    return hostname;
+}
+/** `BLOCKED_FEED_HOSTS`: domains separated by commas, spaces or line breaks. Fails on a bad entry instead of silently allowing it. */
+export function parseBlockedHosts(raw) {
+    if (!raw)
+        return [];
+    const entries = raw.split(/[\s,]+/).filter(Boolean);
+    return [...new Set(entries.map(normalizeBlockedHost))];
+}
+let provider = null;
+let cachedEnv = null;
+/** A host application adds its own list (for example one kept in a database) to the environment list. */
+export function setBlockedHostsProvider(next) {
+    provider = next;
+}
+function environmentHosts() {
+    const raw = process.env.BLOCKED_FEED_HOSTS;
+    if (cachedEnv && cachedEnv.raw === raw)
+        return cachedEnv.hosts;
+    const hosts = parseBlockedHosts(raw);
+    cachedEnv = { raw, hosts };
+    return hosts;
+}
+export function listBlockedHosts() {
+    return [...new Set([...environmentHosts(), ...(provider?.() ?? [])])];
+}
+export function matchingBlockedHost(hostname, blocked = listBlockedHosts()) {
+    if (blocked.length === 0)
+        return null;
+    const set = new Set(blocked);
+    const labels = hostname.toLowerCase().replace(/^\[|\]$/g, "").replace(/\.$/, "").split(".");
+    for (let start = 0; start < labels.length; start++) {
+        const candidate = labels.slice(start).join(".");
+        if (set.has(candidate))
+            return candidate;
+    }
+    return null;
+}
+export function isHostBlocked(hostname) {
+    return matchingBlockedHost(hostname) !== null;
+}
+/** The check for every address that is fetched for a feed, an article or an image: public, and not blocked by the operator. */
+export async function assertPublicFeedUrl(rawUrl) {
+    // Before DNS: a blocked source costs no lookup. An unparseable address falls through to its usual error.
+    let hostname;
+    try {
+        hostname = new URL(normalizeUrlCandidate(rawUrl)).hostname;
+    }
+    catch {
+        hostname = null;
+    }
+    if (hostname !== null && isHostBlocked(hostname))
+        throw new FeedBlockedError(rawUrl);
+    return assertPublicHttpUrl(rawUrl);
+}
+/** The JSON body of a refused address; a blocked source has its own code so clients can tell it from an internal address. */
+export function blockedUrlBody(error) {
+    return { error: error instanceof FeedBlockedError ? FEED_BLOCKED_ERROR : "blocked_url", message: error.message };
+}
+//# sourceMappingURL=blockedHosts.js.map

package/server/dist/feeds/discovery.js

@@ -1,6 +1,6 @@
 import Parser from "rss-parser";
 import { fetchFeed } from "./fetcher.js";
-import { assertPublicHttpUrl } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { decodeEntities, plainTitle } from "./text.js";
 const rssParser = new Parser();
 // Common alternate feed types found in HTML <head>
@@ -74,7 +74,7 @@ async function keepRealFeeds(candidates, pageUrl) {
  * or if it is an HTML page that links to one or more feeds (auto-discovery).
  */
 export async function discoverFeeds(inputUrl) {
-    const validated = await assertPublicHttpUrl(inputUrl);
+    const validated = await assertPublicFeedUrl(inputUrl);
     const targetUrl = validated.toString();
     // 1. The URL is already a feed.
     const page = await parseFeedAt(targetUrl);

package/server/dist/feeds/extractor.js

@@ -2,7 +2,8 @@ import { Readability } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import { fetch as undiciFetch } from "undici";
 import { cleanPage } from "./content/index.js";
-import { assertPublicHttpUrl, createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { articleWorkPool, ArticleHostBusyError, ArticleAccessDeniedError } from "./articleWorkPool.js";
 import { parseRetryAfter } from "./fetcher.js";
 import { sourceHost } from "./requestGate.js";
@@ -130,7 +131,7 @@ async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl, ownerSign
     let currentUrl = rawUrl;
     for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
         articleWorkPool().move(ownerSignal, sourceHost(currentUrl));
-        const safeUrl = await assertPublicHttpUrl(currentUrl);
+        const safeUrl = await assertPublicFeedUrl(currentUrl);
         signal.throwIfAborted();
         const response = await fetchImpl(safeUrl, {
             signal,

package/server/dist/feeds/fetcher.js

@@ -1,5 +1,6 @@
 import { fetch as undiciFetch } from "undici";
-import { assertPublicHttpUrl, createPublicDispatcher, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { createPublicDispatcher, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { APP_VERSION } from "../version.js";
 import { feedRequests, sourceHost } from "./requestGate.js";
 import { readPollSettings } from "./pollSettings.js";
@@ -72,7 +73,7 @@ async function validateUrl(url, signal) {
     return new Promise((resolve, reject) => {
         const abort = () => reject(signal.reason);
         signal.addEventListener("abort", abort, { once: true });
-        assertPublicHttpUrl(url).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
+        assertPublicFeedUrl(url).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
     });
 }
 // Re-validate every redirect hop and bound headers, body, DNS and permit waiting together.

package/server/dist/feeds/inbox.js

@@ -4,7 +4,8 @@ import { db } from "../db/index.js";
 import { hasUserCapability } from "../auth/capabilities.js";
 import { configureFullTextPool, deferFullTextHost } from "./fullTextQueue.js";
 import { extractArticle } from "./extractor.js";
-import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { normalizeUrlCandidate } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { articleImageUrls, pruneArchive, scheduleArchive } from "./archive.js";
 import { findFeedById, findFeedByUrl, isUserSubscribed, findSubscriptionForUser, subscribe, bookmarkItem, findItemById, findItemForUser, setItemNote, } from "./repository.js";
 function escapeHtml(str) {
@@ -47,7 +48,7 @@ export async function saveToInbox(userId, input) {
     let imageUrl = null;
     if (input.url?.trim()) {
         const normalized = normalizeUrlCandidate(input.url.trim());
-        await assertPublicHttpUrl(normalized);
+        await assertPublicFeedUrl(normalized);
         targetUrl = normalized;
         if (!contentHtml && hasUserCapability(userId, "fulltext")) {
             try {

package/server/dist/feeds/opml.js

@@ -2,7 +2,8 @@ import { XMLParser } from "fast-xml-parser";
 import { z } from "zod";
 import { config } from "../config.js";
 import { listSubscriptionsForUser, findFeedByUrl, createFeed, isUserSubscribed, subscribe, createFolder, } from "./repository.js";
-import { assertPublicHttpUrl, SsrfBlockedError } from "./ssrfGuard.js";
+import { SsrfBlockedError } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { requestFeedPoll } from "./poller.js";
 import { decodeEntities } from "./text.js";
 function escapeXml(unsafe) {
@@ -146,7 +147,7 @@ export async function importOpmlFeeds(userId, xmlContent) {
         }
         let normalizedUrl;
         try {
-            const validated = await assertPublicHttpUrl(item.url);
+            const validated = await assertPublicFeedUrl(item.url);
             normalizedUrl = validated.toString();
         }
         catch (error) {

package/server/dist/feeds/pollQueue.js

@@ -3,6 +3,7 @@ import { setImmediate as yieldTurn } from "node:timers/promises";
 import { db } from "../db/index.js";
 import { findFeedById, updateFeedAfterPoll } from "./repository.js";
 import { FeedHttpError } from "./fetcher.js";
+import { FEED_BLOCKED_ERROR, FeedBlockedError } from "./blockedHosts.js";
 import { feedRequests, PollDeferredError, PollShutdownError, RequestQueueFullError, sourceHost } from "./requestGate.js";
 import { readPollSettings } from "./pollSettings.js";
 import { pushOutboxMetrics } from "../pushOutbox.js";
@@ -17,6 +18,8 @@ function isTransient(error) {
         : error instanceof TypeError || error instanceof RequestQueueFullError || (error instanceof Error && ["TimeoutError", "AbortError"].includes(error.name));
 }
 export function pollErrorCode(error) {
+    if (error instanceof FeedBlockedError)
+        return FEED_BLOCKED_ERROR;
     if (error instanceof FeedHttpError)
         return `feed_http_${error.status}`;
     if (error instanceof Error && ["TimeoutError", "AbortError"].includes(error.name))

package/server/dist/feeds/service.js

@@ -1,7 +1,7 @@
 import { config } from "../config.js";
 import { createFeed, findFeedByUrl, isUserSubscribed, subscribe, unsubscribe as unsubscribeRepo, findSubscriptionForUser, updateSubscriptionLabel, updateSubscriptionFolder, updateFeedPollInterval, updateSubscriptionFullTextMode, updateSubscriptionNotify, updateSubscriptionBadge, findFolderById, findFeedById, countFeedSubscribers, replaceFeedUrl, moveSubscription, } from "./repository.js";
 import { pollFeed, probeFeed } from "./poller.js";
-import { assertPublicHttpUrl } from "./ssrfGuard.js";
+import { assertPublicFeedUrl } from "./blockedHosts.js";
 import { discoverFeeds } from "./discovery.js";
 export class FeedError extends Error {
 }
@@ -21,7 +21,7 @@ export class MultipleFeedsFoundError extends FeedError {
 }
 /** Turn a feed or website URL into a feed URL, using auto-discovery for website addresses. */
 async function resolveFeedUrl(inputUrl) {
-    const validated = await assertPublicHttpUrl(inputUrl.trim());
+    const validated = await assertPublicFeedUrl(inputUrl.trim());
     const targetUrl = validated.toString();
     // A known shared feed needs no second discovery download beside its queued poll.
     if (findFeedByUrl(targetUrl))

package/server/dist/server.js

@@ -20,12 +20,16 @@ import { repairEncodedText } from "./feeds/repository.js";
 import { pruneArchive, scheduleMissingArchives } from "./feeds/archive.js";
 import { startNativeRequestCleanup } from "./native/requests.js";
 import { startLeadBackfill } from "./feeds/lead.js";
+import { purgeBlockedFeeds } from "./feeds/blockedFeeds.js";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const WEB_DIST = join(__dirname, "../../web/dist");
 runMigrations();
 const repairedItems = repairEncodedText();
 if (repairedItems > 0)
     console.log(`[db] decoded HTML entities in ${repairedItems} stored items`);
+const blockedFeeds = purgeBlockedFeeds();
+if (blockedFeeds.feeds > 0)
+    console.log(`[feeds] ${blockedFeeds.feeds} feeds are on the block list; removed ${blockedFeeds.items} of their articles`);
 startLeadBackfill();
 pruneArchive();
 const pendingArchives = scheduleMissingArchives();

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.31.0",
+  "version": "0.32.0",
   "private": true,
   "type": "module"
 }