feedkeeper 0.23.1 → 0.25.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.23.1",
+  "version": "0.25.0",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/imageResponse.js

@@ -0,0 +1,15 @@
+import { createHash } from "node:crypto";
+/** Mutable proxy images revalidate; archive routes keep their separate immutable policy. */
+export function sendArticleImage(req, res, image) {
+    const age = Math.max(0, Math.min(21600, Math.floor((image.cacheTtlMs ?? 21_600_000) / 1000)));
+    const etag = `"${createHash("sha256").update(image.buffer).digest("hex")}"`;
+    res.setHeader("Cache-Control", age ? `private, max-age=${age}` : "private, no-store");
+    res.setHeader("ETag", etag);
+    const matches = req.headers["if-none-match"]?.split(",").some(value => value.trim().replace(/^W\//, "") === etag || value.trim() === "*");
+    if (matches) {
+        res.status(304).end();
+        return;
+    }
+    res.type(image.mime).send(image.buffer);
+}
+//# sourceMappingURL=imageResponse.js.map

package/server/dist/api/items.js

@@ -1,3 +1,4 @@
+import { sendArticleImage } from "./imageResponse.js";
 import { Router } from "express";
 import rateLimit from "express-rate-limit";
 import { z } from "zod";
@@ -13,7 +14,7 @@ export const FULL_TEXT_STATUS = {
     full_text_disabled: 409,
     consent_wall: 422,
     could_not_extract_content: 422,
-    extraction_failed: 502,
+    extraction_failed: 502, queue_full: 503, source_busy: 503,
 };
 export const itemsRouter = Router();
 // The origin the browser used (honours TRUST_PROXY), so archive URLs work behind proxies and in development.
@@ -62,8 +63,7 @@ itemsRouter.get("/:itemId/image", imageLimiter, async (req, res) => {
             res.status(415).json({ error: "unsupported_image" });
         }
         else {
-            res.setHeader("Cache-Control", "private, max-age=31536000, immutable");
-            res.type(image.mime).send(image.buffer);
+            sendArticleImage(req, res, image);
         }
     }
     catch (error) {

package/server/dist/api/v1/resources.js

@@ -1,3 +1,4 @@
+import { sendArticleImage } from "../imageResponse.js";
 import { enqueueArticleContents } from "../../feeds/fullTextQueue.js";
 import { Router } from "express";
 import rateLimit from "express-rate-limit";
@@ -628,8 +629,7 @@ resourcesRouter.get("/items/:id/image", imageLimiter, async (req, res) => {
             return void res.status(404).json({ error: "not_found" });
         if (image === "unsupported")
             return void res.status(415).json({ error: "unsupported_image" });
-        res.setHeader("Cache-Control", "private, max-age=31536000, immutable");
-        res.type(image.mime).send(image.buffer);
+        sendArticleImage(req, res, image);
     }
     catch (error) {
         if (error instanceof ImageCacheBusyError) {

package/server/dist/db/migrations/0033_full_text_jobs.sql

@@ -0,0 +1,5 @@
+ALTER TABLE full_text_jobs ADD COLUMN created_at TEXT NOT NULL DEFAULT '';
+UPDATE full_text_jobs SET created_at = available_at;
+ALTER TABLE full_text_jobs ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE full_text_jobs ADD COLUMN force INTEGER NOT NULL DEFAULT 0;
+CREATE TABLE full_text_hosts (host TEXT PRIMARY KEY, retry_at TEXT NOT NULL);

package/server/dist/feeds/archive.js

@@ -297,7 +297,7 @@ export function itemImageUrls(item) {
     return [...new Set([...(hero ? [hero] : []), ...articleImageUrls(item.full_content_html ?? "", item.link), ...articleImageUrls(item.content_html ?? "", item.link)])];
 }
 function proxiedImageUrl(origin, itemId, imageUrl, apiBase = "/api") {
-    return `${origin.replace(/\/$/, "")}${apiBase}/items/${itemId}/image?src=${encodeURIComponent(imageUrl).replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)}`;
+    return `${origin.replace(/\/$/, "")}${apiBase}/items/${itemId}/image?src=${encodeURIComponent(imageUrl).replace(/[!'()*]/g, (char) => `%${char.charCodeAt(0).toString(16).toUpperCase()}`)}&v=2`;
 }
 /**
  * For offline copies: point the first images of articles that are not archived at an endpoint

package/server/dist/feeds/articleWorkPool.js

@@ -0,0 +1,147 @@
+export class ArticleWorkBusyError extends Error {
+    constructor() { super("article_work_busy"); }
+}
+export class ArticleAccessDeniedError extends Error {
+}
+export class ArticleHostBusyError extends Error {
+    host;
+    retryAt;
+    constructor(host, retryAt) {
+        super("article_host_busy");
+        this.host = host;
+        this.retryAt = retryAt;
+    }
+}
+export function readArticleWorkLimits(env = process.env) {
+    const integer = (name, fallback, min, max) => {
+        const value = Number(env[name] ?? fallback);
+        if (!Number.isSafeInteger(value) || value < min || value > max)
+            throw new Error(`${name} must be between ${min} and ${max}`);
+        return value;
+    };
+    return {
+        concurrency: integer("FULLTEXT_CONCURRENCY", 2, 1, 8),
+        hostConcurrency: integer("FULLTEXT_HOST_CONCURRENCY", 1, 1, 4),
+        maxWaiting: integer("FULLTEXT_MAX_WAITING", 32, 0, 128),
+        timeoutMs: integer("FULLTEXT_TIMEOUT_MS", 20_000, 50, 60_000),
+        agingMs: integer("FULLTEXT_AGING_MS", 60_000, 50, 600_000),
+    };
+}
+/** A bounded admission pool shared by saved links and stored-article extraction. */
+export class ArticleWorkPool {
+    limits;
+    now;
+    waiting = [];
+    active = new Map();
+    deadlines = new Map();
+    closing = false;
+    completed = 0;
+    failed = 0;
+    durationMs = 0;
+    drained = [];
+    constructor(limits, now = Date.now) {
+        this.limits = limits;
+        this.now = now;
+    }
+    cooldown = () => 0;
+    get isClosing() { return this.closing; }
+    metrics() { return { running: this.active.size, waiting: this.waiting.length, completed: this.completed, failed: this.failed, durationMs: this.durationMs }; }
+    canStart(host) {
+        return !this.closing && this.cooldown(host) <= this.now() && this.active.size < this.limits.concurrency && [...this.active.values()].filter(value => value === host).length < this.limits.hostConcurrency;
+    }
+    move(signal, host) {
+        const until = this.cooldown(host);
+        if (until > this.now())
+            throw new ArticleHostBusyError(host, until);
+        const controller = [...this.active.keys()].find(entry => entry.signal === signal);
+        if (!controller || this.active.get(controller) === host)
+            return;
+        const occupied = [...this.active.entries()].filter(([, current]) => current === host);
+        if (occupied.length >= this.limits.hostConcurrency) {
+            throw new ArticleHostBusyError(host, Math.min(this.now() + 1000, ...occupied.map(([entry]) => this.deadlines.get(entry))));
+        }
+        this.active.set(controller, host);
+    }
+    run(host, work, priority = 5) {
+        const until = this.cooldown(host);
+        if (until > this.now())
+            return Promise.reject(new ArticleHostBusyError(host, until));
+        if (this.closing || (!this.canStart(host) && this.waiting.length >= this.limits.maxWaiting))
+            return Promise.reject(new ArticleWorkBusyError());
+        const controller = new AbortController();
+        const added = this.now();
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                controller.abort(new DOMException("Article deadline exceeded", "TimeoutError"));
+                const index = this.waiting.indexOf(job);
+                if (index >= 0) {
+                    this.waiting.splice(index, 1);
+                    reject(controller.signal.reason);
+                }
+            }, this.limits.timeoutMs);
+            const job = { host, priority, added, timer, controller, reject, start: () => {
+                    this.active.set(controller, host);
+                    this.deadlines.set(controller, added + this.limits.timeoutMs);
+                    const finish = () => {
+                        clearTimeout(timer);
+                        this.active.delete(controller);
+                        this.deadlines.delete(controller);
+                        this.completed++;
+                        this.durationMs += this.now() - added;
+                        this.pump();
+                        if (!this.active.size)
+                            this.drained.splice(0).forEach(done => done());
+                    };
+                    void Promise.resolve().then(() => work(controller.signal)).then(value => {
+                        if (this.now() - added >= this.limits.timeoutMs) {
+                            this.failed++;
+                            finish();
+                            reject(new DOMException("Article deadline exceeded", "TimeoutError"));
+                        }
+                        else {
+                            finish();
+                            resolve(value);
+                        }
+                    }, error => { this.failed++; finish(); reject(error); });
+                } };
+            this.waiting.push(job);
+            this.pump();
+        });
+    }
+    pump() {
+        this.waiting.sort((a, b) => (b.priority + Math.floor((this.now() - b.added) / this.limits.agingMs)) -
+            (a.priority + Math.floor((this.now() - a.added) / this.limits.agingMs)) || a.added - b.added);
+        for (let index = 0; index < this.waiting.length;) {
+            const job = this.waiting[index];
+            const until = this.cooldown(job.host);
+            if (until > this.now()) {
+                this.waiting.splice(index, 1);
+                clearTimeout(job.timer);
+                job.reject(new ArticleHostBusyError(job.host, until));
+                continue;
+            }
+            if (!this.canStart(job.host)) {
+                index++;
+                continue;
+            }
+            this.waiting.splice(index, 1);
+            job.start();
+        }
+    }
+    async close() {
+        this.closing = true;
+        for (const job of this.waiting.splice(0)) {
+            clearTimeout(job.timer);
+            job.reject(new ArticleWorkBusyError());
+        }
+        for (const controller of this.active.keys())
+            controller.abort();
+        if (this.active.size)
+            await new Promise(resolve => this.drained.push(resolve));
+    }
+}
+let shared;
+export function articleWorkPool() { return shared ??= new ArticleWorkPool(readArticleWorkLimits()); }
+export function restartArticleWorkPool() { if (shared?.isClosing)
+    shared = undefined; }
+//# sourceMappingURL=articleWorkPool.js.map

package/server/dist/feeds/extractor.js

@@ -2,6 +2,9 @@ import { Readability } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import { fetch as undiciFetch } from "undici";
 import { assertPublicHttpUrl, createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { articleWorkPool, ArticleHostBusyError, ArticleAccessDeniedError } from "./articleWorkPool.js";
+import { parseRetryAfter } from "./fetcher.js";
+import { sourceHost } from "./requestGate.js";
 import { APP_VERSION } from "../version.js";
 const defaultFetch = undiciFetch;
 const CONSENT_SELECTORS = [
@@ -79,9 +82,13 @@ const FETCH_TIMEOUT_MS = 12_000;
 const MAX_REDIRECTS = 5;
 class ArticleHttpError extends Error {
     status;
-    constructor(status) {
+    retryAt;
+    host;
+    constructor(status, retryAt, host) {
         super("article_http_error");
         this.status = status;
+        this.retryAt = retryAt;
+        this.host = host;
     }
 }
 const MAX_HTML_BYTES = 2 * 1024 * 1024;
@@ -93,10 +100,12 @@ function stripHeavyTags(html) {
         .replace(/<iframe\b[^<]*(?:(?!<\/iframe>)<[^<]*)*<\/iframe>/gi, "")
         .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, "");
 }
-async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl) {
+async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl, ownerSignal) {
     let currentUrl = rawUrl;
     for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
+        articleWorkPool().move(ownerSignal, sourceHost(currentUrl));
         const safeUrl = await assertPublicHttpUrl(currentUrl);
+        signal.throwIfAborted();
         const response = await fetchImpl(safeUrl, {
             signal,
             headers: {
@@ -111,19 +120,21 @@ async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl) {
         });
         if (response.status >= 300 && response.status < 400) {
             const location = response.headers.get("location");
+            await response.body?.cancel();
             if (!location)
                 return null;
-            await response.body?.cancel();
             currentUrl = new URL(location, safeUrl).toString();
             continue;
         }
         if (!response.ok) {
             await response.body?.cancel();
-            throw new ArticleHttpError(response.status);
+            throw new ArticleHttpError(response.status, parseRetryAfter(response.headers.get("retry-after")), sourceHost(safeUrl.toString()));
         }
         const contentLength = Number(response.headers.get("content-length"));
-        if (contentLength > MAX_HTML_BYTES)
+        if (contentLength > MAX_HTML_BYTES) {
+            await response.body?.cancel();
             return null;
+        }
         const reader = response.body?.getReader();
         if (!reader)
             return null;
@@ -150,13 +161,22 @@ export async function extractArticleFromUrl(rawUrl, fetchImpl = defaultFetch) {
     return result.status === "ok" ? result.article : null;
 }
 /** Like extractArticleFromUrl, but reports whether the site answered with a consent wall. */
-export async function extractArticle(rawUrl, fetchImpl = defaultFetch) {
+export async function extractArticle(rawUrl, fetchImpl = defaultFetch, priority = 5, authorized = () => true) {
+    return articleWorkPool().run(sourceHost(rawUrl), signal => {
+        if (!authorized())
+            throw new ArticleAccessDeniedError();
+        return extractArticleContent(rawUrl, fetchImpl, signal);
+    }, priority);
+}
+async function extractArticleContent(rawUrl, fetchImpl, signal) {
+    signal.throwIfAborted();
     const safeUrl = await assertPublicHttpUrl(rawUrl);
+    signal.throwIfAborted();
     const dispatcher = createPublicDispatcher();
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
     try {
-        const fetched = await fetchArticleHtml(safeUrl.toString(), controller.signal, dispatcher, fetchImpl);
+        const fetched = await fetchArticleHtml(safeUrl.toString(), AbortSignal.any([signal, controller.signal]), dispatcher, fetchImpl, signal);
         if (!fetched)
             return { status: "failed" };
         const cleanedHtml = stripHeavyTags(fetched.html);
@@ -190,13 +210,17 @@ export async function extractArticle(rawUrl, fetchImpl = defaultFetch) {
         };
     }
     catch (error) {
+        if (error instanceof ArticleHostBusyError)
+            return { status: "throttled", host: error.host, retryAt: error.retryAt, deferred: true };
         if (error instanceof SsrfBlockedError)
             throw error;
-        if (controller.signal.aborted)
+        if (signal.aborted || controller.signal.aborted)
             return { status: "timeout" };
         if (error instanceof ArticleHttpError && [401, 402].includes(error.status))
             return { status: "paywall" };
-        if (error instanceof ArticleHttpError && [403, 429, 503].includes(error.status))
+        if (error instanceof ArticleHttpError && [429, 503].includes(error.status))
+            return { status: "throttled", retryAt: error.retryAt, host: error.host };
+        if (error instanceof ArticleHttpError && error.status === 403)
             return { status: "bot_blocked" };
         return { status: "failed" };
     }

package/server/dist/feeds/fullText.js

@@ -1,21 +1,13 @@
 import { extractArticle, isCachedConsentSnippet } from "./extractor.js";
 import { hasUserCapability } from "../auth/capabilities.js";
 import { db } from "../db/index.js";
+import { articleWorkPool, ArticleWorkBusyError, ArticleAccessDeniedError, ArticleHostBusyError } from "./articleWorkPool.js";
+import { requestFullText, authorizedFullTextRequester } from "./fullTextQueue.js";
+import { SsrfBlockedError } from "./ssrfGuard.js";
 import { improvesContent } from "./articleContent.js";
 import { canAccessItem, findItemById, findSubscriptionFullTextMode, updateItemFullContent } from "./repository.js";
 // Kept for integrations importing the old constant; retries now belong to individual articles.
 export const FULL_TEXT_BLOCK_THRESHOLD = 2;
-const inFlight = new Map();
-const hosts = new Set();
-let active = 0;
-const waiters = [];
-async function acquire(host) {
-    while (active >= 2 || hosts.has(host))
-        await new Promise(resolve => waiters.push(resolve));
-    active++;
-    hosts.add(host);
-    return () => { active--; hosts.delete(host); waiters.splice(0).forEach(resolve => resolve()); };
-}
 function recordAttempt(id, status) {
     const blocked = ["paywall", "consent_wall"].includes(status);
     const retry = status === "ready" || status === "feed_only" ? null :
@@ -24,7 +16,7 @@ function recordAttempt(id, status) {
         .run(status, new Date().toISOString(), retry, id);
 }
 /** All callers share bounded host-aware extraction, cache quality and per-article retry policy. */
-export async function loadFullText(userId, itemId, options = {}) {
+export function cachedFullText(userId, itemId, force = false) {
     const item = findItemById(itemId);
     const mode = item ? findSubscriptionFullTextMode(userId, item.feed_id) ?? (canAccessItem(userId, itemId) ? "auto" : undefined) : undefined;
     if (!item || !mode)
@@ -34,58 +26,73 @@ export async function loadFullText(userId, itemId, options = {}) {
         db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(itemId);
         item.full_content_html = null;
     }
-    if (!options.force && item.full_content_html && !isCachedConsentSnippet(item.full_content_html)) {
+    if (!force && item.full_content_html && !isCachedConsentSnippet(item.full_content_html)) {
         return { ok: true, id: item.id, fullContentHtml: item.full_content_html, cached: true };
     }
     if (!item.link)
         return { ok: false, error: "item_has_no_link" };
     if (mode === "never")
         return { ok: false, error: "full_text_disabled" };
-    if (!options.force && item.extraction_status === "feed_only")
+    if (!force && item.extraction_status === "feed_only")
         return { ok: true, id: itemId, fullContentHtml: item.content_html ?? "", cached: true };
-    if (!options.force && item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now()) {
+    if (!force && item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now() && !db.prepare("SELECT 1 FROM full_text_jobs WHERE item_id=?").get(itemId)) {
         const status = item.extraction_status;
         return { ok: false, error: ["consent_wall", "paywall", "bot_blocked", "timeout", "partial"].includes(status) ? status : "could_not_extract_content" };
     }
     if (!hasUserCapability(userId, "fulltext"))
         return { ok: false, error: "capability_not_available" };
-    if (inFlight.has(itemId))
-        return inFlight.get(itemId);
-    const task = (async () => {
-        let release;
-        try {
-            release = await acquire(new URL(item.link).host);
-            if (!hasUserCapability(userId, "fulltext") || !canAccessItem(userId, itemId))
-                return { ok: false, error: "capability_not_available" };
-            const result = await extractArticle(item.link);
-            if (result.status !== "ok") {
-                recordAttempt(itemId, result.status);
-                return { ok: false, error: result.status === "failed" ? "could_not_extract_content" : result.status };
-            }
-            // Content may have improved while this request was running.
-            const current = findItemById(itemId);
-            if (!current)
-                return { ok: false, error: "item_not_found" };
-            if (current.link !== item.link)
-                return { ok: false, error: "extraction_failed" };
-            if (improvesContent(current, result.article.contentHtml)) {
-                db.transaction(() => { updateItemFullContent(itemId, result.article.contentHtml); recordAttempt(itemId, "ready"); })();
-            }
-            else
-                recordAttempt(itemId, current.full_content_html ? "ready" : "feed_only");
-            return { ok: true, id: itemId, fullContentHtml: findItemById(itemId).full_content_html || current.content_html || "",
-                title: result.article.title, byline: result.article.byline, cached: false };
+    return undefined;
+}
+export async function loadFullText(userId, itemId, options = {}) {
+    return cachedFullText(userId, itemId, options.force) ?? requestFullText(userId, itemId, options.force ?? false);
+}
+export async function executeFullText(userId, itemId, force, priority) {
+    const requester = authorizedFullTextRequester(itemId, userId);
+    if (requester === undefined)
+        return { ok: false, error: "capability_not_available" };
+    const cached = cachedFullText(requester, itemId, force);
+    if (cached)
+        return cached;
+    const item = findItemById(itemId);
+    try {
+        const result = await extractArticle(item.link, undefined, priority, () => authorizedFullTextRequester(itemId, userId) !== undefined);
+        if (articleWorkPool().isClosing)
+            return { ok: false, error: "queue_full" };
+        if (authorizedFullTextRequester(itemId, userId) === undefined)
+            return { ok: false, error: "capability_not_available" };
+        if (result.status === "throttled")
+            return { ok: false, error: "source_busy", retryable: true, retryAt: result.retryAt, host: result.host, deferred: result.deferred };
+        if (result.status !== "ok") {
+            recordAttempt(itemId, result.status);
+            return { ok: false, error: result.status === "failed" ? "could_not_extract_content" : result.status,
+                ...(result.status === "failed" || result.status === "timeout" ? { retryable: true } : {}) };
         }
-        catch {
-            recordAttempt(itemId, "failed");
-            return { ok: false, error: "extraction_failed" };
+        const current = findItemById(itemId);
+        if (!current || current.link !== item.link)
+            return { ok: false, error: "item_not_found" };
+        if (improvesContent(current, result.article.contentHtml)) {
+            db.transaction(() => { updateItemFullContent(itemId, result.article.contentHtml); recordAttempt(itemId, "ready"); })();
         }
-        finally {
-            release?.();
-            inFlight.delete(itemId);
+        else
+            recordAttempt(itemId, current.full_content_html ? "ready" : "feed_only");
+        return { ok: true, id: itemId, fullContentHtml: findItemById(itemId).full_content_html || current.content_html || "",
+            title: result.article.title, byline: result.article.byline, cached: false };
+    }
+    catch (error) {
+        if (error instanceof SsrfBlockedError)
+            return { ok: false, error: "extraction_failed" };
+        if (error instanceof ArticleHostBusyError)
+            return { ok: false, error: "source_busy", retryable: true, deferred: true, host: error.host, retryAt: error.retryAt };
+        if (error instanceof ArticleAccessDeniedError)
+            return { ok: false, error: "capability_not_available" };
+        if (articleWorkPool().isClosing || error instanceof ArticleWorkBusyError)
+            return { ok: false, error: "queue_full", retryable: true };
+        if (error instanceof Error && ["TimeoutError", "AbortError"].includes(error.name)) {
+            recordAttempt(itemId, "timeout");
+            return { ok: false, error: "timeout", retryable: true };
         }
-    })();
-    inFlight.set(itemId, task);
-    return task;
+        recordAttempt(itemId, "failed");
+        return { ok: false, error: "extraction_failed", retryable: true };
+    }
 }
 //# sourceMappingURL=fullText.js.map