feedkeeper 0.0.0-stage → 0.23.0

package/server/dist/feeds/feedIcon.js

@@ -0,0 +1,171 @@
+import crypto from "node:crypto";
+import { Resvg } from "@resvg/resvg-js";
+import { extractLargestImageAsPng } from "@humanwhocodes/ico-to-png";
+import { db } from "../db/index.js";
+import { fetchImage } from "./fetcher.js";
+import { readPollSettings } from "./pollSettings.js";
+import { detectImageType } from "./archive.js";
+const MAX_ICON_BYTES = 1024 * 1024;
+const TTL_MS = 24 * 60 * 60 * 1000;
+const MAX_CACHED = 500;
+const cache = new Map();
+/** Compute stable content hash for versioning and caching. */
+export function hashIconBuffer(buffer) {
+    return crypto.createHash("sha256").update(buffer).digest("hex").slice(0, 16);
+}
+/** Icons are small, so besides raster images the formats typical for favicons are accepted. */
+export function detectIconType(buffer) {
+    const raster = detectImageType(buffer);
+    if (raster)
+        return raster.mime;
+    if (buffer.length >= 4 && buffer[0] === 0 && buffer[1] === 0 && buffer[2] === 1 && buffer[3] === 0)
+        return "image/x-icon";
+    const head = Buffer.from(buffer.subarray(0, 512)).toString("utf8").trimStart();
+    return /^(<\?xml[^>]*>\s*)?(<!--[\s\S]*?-->\s*)?<svg[\s>]/i.test(head) ? "image/svg+xml" : null;
+}
+/** Whether the user subscribes to the feed or has saved or annotated one of its articles. */
+export function canAccessFeed(userId, feedId) {
+    return Boolean(db.prepare(`SELECT 1 FROM feeds f WHERE f.id = ?
+         AND (EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id AND s.user_id = ?)
+           OR EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = f.id AND b.user_id = ?)
+           OR EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = f.id AND n.user_id = ?))`).get(feedId, userId, userId, userId));
+}
+function candidates(feed, declaredUrls) {
+    const urls = new Set(declaredUrls);
+    if (feed.icon_url)
+        urls.add(feed.icon_url);
+    for (const source of [feed.site_url, feed.url]) {
+        try {
+            if (source)
+                urls.add(`${new URL(source).origin}/favicon.ico`);
+        }
+        catch {
+            // Try the next source.
+        }
+    }
+    return [...urls];
+}
+/** Keep the same raster icon in every client, including clients without SVG or ICO support. */
+function rasterIcon(buffer, mime) {
+    if (mime === "image/svg+xml") {
+        const svg = new Resvg(buffer, { fitTo: { mode: "width", value: 512 }, font: { loadSystemFonts: false } });
+        if (svg.width > 4096 || svg.height > 4096 || svg.width < 1 || svg.height < 1 ||
+            svg.width / svg.height > 8 || svg.height / svg.width > 8)
+            throw new Error("Invalid SVG icon size");
+        return { buffer: svg.render().asPng(), mime: "image/png" };
+    }
+    if (mime === "image/x-icon") {
+        const image = extractLargestImageAsPng(buffer);
+        if (!image)
+            throw new Error("Invalid ICO icon");
+        return { buffer: Buffer.from(image.data), mime: "image/png" };
+    }
+    return { buffer, mime };
+}
+/** Store or update feed icon in database, only notifying subscribers when its content changes. */
+export function storeFeedIcon(feedId, buffer, mime) {
+    ({ buffer, mime } = rasterIcon(buffer, mime));
+    const hash = hashIconBuffer(buffer);
+    db.prepare(`INSERT INTO feed_icons (feed_id, data, mime, hash, updated_at)
+     VALUES (?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+     ON CONFLICT (feed_id) DO UPDATE SET data = excluded.data, mime = excluded.mime, hash = excluded.hash, updated_at = excluded.updated_at
+     WHERE feed_icons.hash != excluded.hash OR feed_icons.mime != excluded.mime`).run(feedId, buffer, mime, hash);
+    const icon = { buffer, mime, hash };
+    if (cache.size >= MAX_CACHED)
+        cache.delete(cache.keys().next().value);
+    cache.set(feedId, { icon, fetchedAt: Date.now() });
+    return icon;
+}
+/** Re-check remote candidates during the weekly feed poll, even if a stored icon exists. */
+const refreshing = new Map();
+export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
+    const current = refreshing.get(feedId);
+    if (current)
+        return current;
+    if (refreshing.size >= readPollSettings().maxWaiters)
+        return Promise.resolve(null);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new DOMException("Icon deadline exceeded", "TimeoutError")), readPollSettings().attemptTimeoutMs);
+    const deadline = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal;
+    const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline).finally(() => {
+        clearTimeout(timeout);
+        refreshing.delete(feedId);
+    });
+    refreshing.set(feedId, promise);
+    return promise;
+}
+async function refreshIcon(feedId, siteUrl, declaredUrls, signal) {
+    const feed = db.prepare("SELECT icon_url, site_url, url FROM feeds WHERE id = ?").get(feedId);
+    for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls) : []) {
+        if (signal.aborted)
+            break;
+        try {
+            const { buffer } = await fetchImage(url, MAX_ICON_BYTES, signal);
+            const mime = detectIconType(buffer);
+            if (mime) {
+                const icon = storeFeedIcon(feedId, buffer, mime);
+                db.prepare("UPDATE feeds SET icon_url = ? WHERE id = ?").run(url, feedId);
+                return icon;
+            }
+        }
+        catch {
+            // Keep the last good icon and try the next candidate.
+        }
+    }
+    return null;
+}
+/** Retrieve icon hash from database without reading full blob. */
+export function getFeedIconHash(feedId) {
+    const row = db.prepare("SELECT hash FROM feed_icons WHERE feed_id = ?").get(feedId);
+    return row?.hash ?? null;
+}
+/**
+ * The feed's icon, fetched by this server so the browser and native apps can keep it for offline use
+ * without asking third-party sites. Stored persistently in the database and cached in memory.
+ */
+export async function loadFeedIcon(feedId) {
+    const known = cache.get(feedId);
+    if (known && Date.now() - known.fetchedAt < TTL_MS)
+        return known.icon;
+    // Check persistent storage
+    const row = db.prepare("SELECT data, mime, hash FROM feed_icons WHERE feed_id = ?").get(feedId);
+    if (row) {
+        // Icons stored by older versions may still be SVG or ICO.
+        let icon;
+        try {
+            icon = row.mime === "image/svg+xml" || row.mime === "image/x-icon"
+                ? storeFeedIcon(feedId, row.data, row.mime)
+                : { buffer: row.data, mime: row.mime, hash: row.hash };
+        }
+        catch {
+            return refreshFeedIcon(feedId);
+        }
+        if (cache.size >= MAX_CACHED)
+            cache.delete(cache.keys().next().value);
+        cache.set(feedId, { icon, fetchedAt: Date.now() });
+        return icon;
+    }
+    // Capacity pressure is temporary, not evidence that the publisher has no icon.
+    // Do not poison the 24-hour negative cache when another refresh can try shortly.
+    if (!refreshing.has(feedId) && refreshing.size >= readPollSettings().maxWaiters)
+        return null;
+    const icon = await refreshFeedIcon(feedId);
+    if (!icon) {
+        if (cache.size >= MAX_CACHED)
+            cache.delete(cache.keys().next().value);
+        cache.set(feedId, { icon: null, fetchedAt: Date.now() });
+    }
+    return icon;
+}
+/** Forget remembered icons (tests). */
+export function clearIconCache(feedId) {
+    if (feedId !== undefined) {
+        cache.delete(feedId);
+        db.prepare("DELETE FROM feed_icons WHERE feed_id = ?").run(feedId);
+    }
+    else {
+        cache.clear();
+        db.prepare("DELETE FROM feed_icons").run();
+    }
+}
+//# sourceMappingURL=feedIcon.js.map

package/server/dist/feeds/fetcher.js

@@ -0,0 +1,217 @@
+import { fetch as undiciFetch } from "undici";
+import { assertPublicHttpUrl, createPublicDispatcher, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { APP_VERSION } from "../version.js";
+import { feedRequests, sourceHost } from "./requestGate.js";
+import { readPollSettings } from "./pollSettings.js";
+const MAX_RESPONSE_BYTES = 10 * 1024 * 1024; // 10 MB
+const MAX_REDIRECTS = 5;
+const USER_AGENT = `FeedKeeper/${APP_VERSION} (+https://github.com/visualfusion/feedkeeper)`;
+function cacheLifetime(headers) {
+    const control = headers.get("cache-control") ?? "";
+    if (/(?:^|,)\s*(?:no-store|no-cache|private)\b/i.test(control))
+        return 0;
+    const maxAge = /(?:^|,)\s*max-age\s*=\s*"?(\d+)"?/i.exec(control);
+    const age = Number(headers.get("age") ?? 0);
+    const date = Date.parse(headers.get("date") ?? "");
+    const elapsed = Math.max(0, Number.isFinite(age) ? age * 1_000 : 0, Number.isFinite(date) ? Date.now() - date : 0);
+    if (maxAge)
+        return Math.max(0, Number(maxAge[1]) * 1_000 - elapsed);
+    const expires = Date.parse(headers.get("expires") ?? "");
+    return Number.isFinite(expires) ? Math.max(0, expires - Date.now()) : undefined;
+}
+export class FeedHttpError extends Error {
+    status;
+    retryAt;
+    host;
+    constructor(status, retryAt, host) {
+        super(`Feed responded with HTTP ${status}`);
+        this.status = status;
+        this.retryAt = retryAt;
+        this.host = host;
+    }
+}
+/** RFC 9110: non-negative integer seconds or an HTTP date (including obsolete HTTP dates). */
+export function parseRetryAfter(value, now = Date.now()) {
+    if (value === null)
+        return undefined;
+    const text = value.trim();
+    if (/^\d+$/.test(text)) {
+        const seconds = Number(text);
+        if (!Number.isFinite(seconds))
+            return undefined;
+        return Math.min(8.64e15, now + seconds * 1000);
+    }
+    const day = "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)";
+    const month = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)";
+    const time = "\\d{2}:\\d{2}:\\d{2}";
+    const imf = new RegExp(`^${day}, \\d{2} ${month} \\d{4} ${time} GMT$`);
+    const obsolete = text.match(new RegExp(`^(Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), (\\d{2})-(${month})-(\\d{2}) (${time}) GMT$`));
+    const asctime = text.match(new RegExp(`^(${day}) (${month}) ( [1-9]|\\d{2}) (${time}) (\\d{4})$`));
+    let normalized = text;
+    if (obsolete) {
+        const yearLimit = new Date(now).getUTCFullYear() + 50;
+        let year = Math.floor(yearLimit / 100) * 100 + Number(obsolete[4]);
+        if (year > yearLimit)
+            year -= 100;
+        normalized = `${obsolete[1].slice(0, 3)}, ${obsolete[2]} ${obsolete[3]} ${year} ${obsolete[5]} GMT`;
+    }
+    else if (asctime) {
+        // asctime has no zone suffix; HTTP dates are always UTC, never the server's local time.
+        normalized = `${asctime[1]}, ${asctime[3].trim().padStart(2, "0")} ${asctime[2]} ${asctime[5]} ${asctime[4]} GMT`;
+    }
+    else if (!imf.test(text))
+        return undefined;
+    const date = Date.parse(normalized);
+    if (!Number.isFinite(date) || new Date(date).toUTCString() !== normalized)
+        return undefined;
+    return Math.max(now, date);
+}
+/** DNS preflight cannot be cancelled, but must not hold the caller past its deadline. */
+async function validateUrl(url, signal) {
+    signal.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const abort = () => reject(signal.reason);
+        signal.addEventListener("abort", abort, { once: true });
+        assertPublicHttpUrl(url).then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
+    });
+}
+// Re-validate every redirect hop and bound headers, body, DNS and permit waiting together.
+async function downloadPublic(url, opts) {
+    let currentUrl = url;
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new DOMException("Download deadline exceeded", "TimeoutError")), readPollSettings().requestTimeoutMs);
+    const signal = opts.signal ? AbortSignal.any([controller.signal, opts.signal]) : controller.signal;
+    const dispatcher = createPublicDispatcher();
+    try {
+        for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
+            const candidate = new URL(normalizeUrlCandidate(currentUrl));
+            const host = sourceHost(candidate.href);
+            const release = await feedRequests.acquire(host, signal);
+            try {
+                const validated = await validateUrl(candidate.href, signal);
+                signal.throwIfAborted();
+                const headers = { "User-Agent": USER_AGENT, Accept: opts.accept };
+                if (redirects === 0 && opts.etag)
+                    headers["If-None-Match"] = opts.etag;
+                if (redirects === 0 && opts.lastModified)
+                    headers["If-Modified-Since"] = opts.lastModified;
+                const response = await undiciFetch(validated, { headers, redirect: "manual", signal, dispatcher });
+                if (response.status === 304) {
+                    await response.body?.cancel();
+                    return { notModified: true, buffer: Buffer.alloc(0), finalUrl: currentUrl, contentType: response.headers.get("content-type") ?? undefined };
+                }
+                if (response.status >= 300 && response.status < 400) {
+                    const location = response.headers.get("location");
+                    await response.body?.cancel();
+                    if (!location)
+                        throw new Error("Redirect without a Location header");
+                    currentUrl = new URL(location, validated).toString();
+                    continue;
+                }
+                if (!response.ok) {
+                    let retryAt = parseRetryAfter(response.headers.get("retry-after"));
+                    if (response.status === 429 || response.status === 503) {
+                        retryAt = Math.max(retryAt ?? 0, Date.now() + readPollSettings().retryBaseMs * (0.5 + Math.random() * 0.5));
+                        feedRequests.defer(host, retryAt);
+                    }
+                    await response.body?.cancel();
+                    throw new FeedHttpError(response.status, retryAt, host);
+                }
+                const contentLength = response.headers.get("content-length");
+                if (contentLength && Number(contentLength) > opts.maxBytes) {
+                    await response.body?.cancel();
+                    throw new Error("Feed response exceeds the maximum allowed size");
+                }
+                const reader = response.body?.getReader();
+                if (!reader)
+                    throw new Error("Feed response had no body");
+                const chunks = [];
+                let total = 0;
+                for (;;) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                        break;
+                    total += value.byteLength;
+                    if (total > opts.maxBytes) {
+                        await reader.cancel();
+                        throw new Error("Feed response exceeds the maximum allowed size");
+                    }
+                    chunks.push(value);
+                }
+                return {
+                    cacheTtlMs: cacheLifetime(response.headers),
+                    notModified: false, buffer: Buffer.concat(chunks, total),
+                    contentType: response.headers.get("content-type") ?? undefined,
+                    etag: response.headers.get("etag") ?? undefined,
+                    lastModified: response.headers.get("last-modified") ?? undefined, finalUrl: currentUrl,
+                };
+            }
+            finally {
+                release();
+            }
+        }
+        throw new Error("Too many redirects while fetching feed");
+    }
+    finally {
+        clearTimeout(timeout);
+        await dispatcher.destroy();
+    }
+}
+// Fetches a feed URL with conditional GET headers and decodes it to text.
+export async function fetchFeed(url, opts = {}) {
+    const download = await downloadPublic(url, {
+        accept: "application/rss+xml, application/atom+xml, application/xml, text/xml, text/html, application/xhtml+xml;q=0.9, */*;q=0.5",
+        maxBytes: MAX_RESPONSE_BYTES,
+        etag: opts.etag,
+        lastModified: opts.lastModified,
+        signal: opts.signal,
+    });
+    if (download.notModified) {
+        return { notModified: true, body: "", finalUrl: download.finalUrl, contentType: download.contentType };
+    }
+    return {
+        body: decodeFeedBuffer(download.buffer, download.contentType),
+        notModified: false,
+        etag: download.etag,
+        lastModified: download.lastModified,
+        contentType: download.contentType,
+        finalUrl: download.finalUrl,
+    };
+}
+/** Downloads an image for the archive; the caller checks the file signature. */
+export async function fetchImage(url, maxBytes, signal) {
+    const download = await downloadPublic(url, { accept: "image/avif,image/webp,image/png,image/jpeg,image/gif;q=0.9,*/*;q=0.5", maxBytes, signal });
+    return { buffer: download.buffer, finalUrl: download.finalUrl, cacheTtlMs: download.cacheTtlMs };
+}
+/**
+ * Decodes raw response buffer into a string respecting character encoding from:
+ * 1. HTTP Content-Type header (e.g. charset=ISO-8859-1)
+ * 2. XML declaration header (e.g. <?xml version="1.0" encoding="ISO-8859-1"?>)
+ * Defaults to UTF-8.
+ */
+function decodeFeedBuffer(buffer, contentType) {
+    let charset = null;
+    if (contentType) {
+        const match = contentType.match(/charset=([^;]+)/i);
+        if (match) {
+            charset = match[1].trim().replace(/^["']|["']$/g, "");
+        }
+    }
+    if (!charset) {
+        const snippet = buffer.subarray(0, 1024).toString("ascii");
+        const xmlMatch = snippet.match(/<\?xml[^>]+encoding=["']([^"']+)["']/i);
+        if (xmlMatch) {
+            charset = xmlMatch[1].trim();
+        }
+    }
+    if (charset) {
+        try {
+            return new TextDecoder(charset).decode(buffer);
+        }
+        catch {
+            // If TextDecoder does not recognize the charset, fall back to utf-8
+        }
+    }
+    return buffer.toString("utf8");
+}
+//# sourceMappingURL=fetcher.js.map

package/server/dist/feeds/fullText.js

@@ -0,0 +1,91 @@
+import { extractArticle, isCachedConsentSnippet } from "./extractor.js";
+import { hasUserCapability } from "../auth/capabilities.js";
+import { db } from "../db/index.js";
+import { improvesContent } from "./articleContent.js";
+import { canAccessItem, findItemById, findSubscriptionFullTextMode, updateItemFullContent } from "./repository.js";
+// Kept for integrations importing the old constant; retries now belong to individual articles.
+export const FULL_TEXT_BLOCK_THRESHOLD = 2;
+const inFlight = new Map();
+const hosts = new Set();
+let active = 0;
+const waiters = [];
+async function acquire(host) {
+    while (active >= 2 || hosts.has(host))
+        await new Promise(resolve => waiters.push(resolve));
+    active++;
+    hosts.add(host);
+    return () => { active--; hosts.delete(host); waiters.splice(0).forEach(resolve => resolve()); };
+}
+function recordAttempt(id, status) {
+    const blocked = ["paywall", "consent_wall"].includes(status);
+    const retry = status === "ready" || status === "feed_only" ? null :
+        new Date(Date.now() + (blocked ? 7 * 86400_000 : status === "partial" ? 86400_000 : 3600_000)).toISOString();
+    db.prepare("UPDATE items SET extraction_status = ?, extraction_attempted_at = ?, extraction_retry_at = ? WHERE id = ?")
+        .run(status, new Date().toISOString(), retry, id);
+}
+/** All callers share bounded host-aware extraction, cache quality and per-article retry policy. */
+export async function loadFullText(userId, itemId, options = {}) {
+    const item = findItemById(itemId);
+    const mode = item ? findSubscriptionFullTextMode(userId, item.feed_id) ?? (canAccessItem(userId, itemId) ? "auto" : undefined) : undefined;
+    if (!item || !mode)
+        return { ok: false, error: "item_not_found" };
+    if (item.full_content_html && isCachedConsentSnippet(item.full_content_html)) {
+        // Repair cached walls before any caller can mistake them for reader content again.
+        db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(itemId);
+        item.full_content_html = null;
+    }
+    if (!options.force && item.full_content_html && !isCachedConsentSnippet(item.full_content_html)) {
+        return { ok: true, id: item.id, fullContentHtml: item.full_content_html, cached: true };
+    }
+    if (!item.link)
+        return { ok: false, error: "item_has_no_link" };
+    if (mode === "never")
+        return { ok: false, error: "full_text_disabled" };
+    if (!options.force && item.extraction_status === "feed_only")
+        return { ok: true, id: itemId, fullContentHtml: item.content_html ?? "", cached: true };
+    if (!options.force && item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now()) {
+        const status = item.extraction_status;
+        return { ok: false, error: ["consent_wall", "paywall", "bot_blocked", "timeout", "partial"].includes(status) ? status : "could_not_extract_content" };
+    }
+    if (!hasUserCapability(userId, "fulltext"))
+        return { ok: false, error: "capability_not_available" };
+    if (inFlight.has(itemId))
+        return inFlight.get(itemId);
+    const task = (async () => {
+        let release;
+        try {
+            release = await acquire(new URL(item.link).host);
+            if (!hasUserCapability(userId, "fulltext") || !canAccessItem(userId, itemId))
+                return { ok: false, error: "capability_not_available" };
+            const result = await extractArticle(item.link);
+            if (result.status !== "ok") {
+                recordAttempt(itemId, result.status);
+                return { ok: false, error: result.status === "failed" ? "could_not_extract_content" : result.status };
+            }
+            // Content may have improved while this request was running.
+            const current = findItemById(itemId);
+            if (!current)
+                return { ok: false, error: "item_not_found" };
+            if (current.link !== item.link)
+                return { ok: false, error: "extraction_failed" };
+            if (improvesContent(current, result.article.contentHtml)) {
+                db.transaction(() => { updateItemFullContent(itemId, result.article.contentHtml); recordAttempt(itemId, "ready"); })();
+            }
+            else
+                recordAttempt(itemId, current.full_content_html ? "ready" : "feed_only");
+            return { ok: true, id: itemId, fullContentHtml: findItemById(itemId).full_content_html || current.content_html || "",
+                title: result.article.title, byline: result.article.byline, cached: false };
+        }
+        catch {
+            recordAttempt(itemId, "failed");
+            return { ok: false, error: "extraction_failed" };
+        }
+        finally {
+            release?.();
+            inFlight.delete(itemId);
+        }
+    })();
+    inFlight.set(itemId, task);
+    return task;
+}
+//# sourceMappingURL=fullText.js.map

package/server/dist/feeds/fullTextQueue.js

@@ -0,0 +1,159 @@
+import { isCachedConsentSnippet } from "./extractor.js";
+import { db } from "../db/index.js";
+import { hasUserCapability } from "../auth/capabilities.js";
+import { loadFullText } from "./fullText.js";
+import { canAccessItem, findSubscriptionFullTextMode, findItemById } from "./repository.js";
+/** Refillable durable queue, bounded independently of library size. */
+export function scheduleFullTexts() {
+    const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
+    const room = Math.max(0, 200 - queued.n);
+    if (!room)
+        return 0;
+    const eligible = db.prepare("SELECT id FROM users").all().filter(user => hasUserCapability(user.id, "fulltext")).map(user => user.id);
+    if (!eligible.length)
+        return 0;
+    const candidates = db.prepare(`
+    WITH candidate_articles AS (
+      SELECT ids.value AS item_id, e.user_id, 3 AS priority, i.created_at
+      FROM editions e, json_each(e.item_ids) ids
+      JOIN items i ON i.id = ids.value
+      WHERE e.user_id IN (SELECT value FROM json_each(?))
+        AND e.status = 'active' AND e.expires_at > strftime('%Y-%m-%dT%H:%M:%fZ','now')
+        AND i.link IS NOT NULL AND i.full_content_html IS NULL
+        AND i.extraction_status != 'feed_only'
+        AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+        AND NOT EXISTS (SELECT 1 FROM full_text_jobs j WHERE j.item_id = i.id)
+
+      UNION ALL
+
+      SELECT b.item_id, b.user_id, 2 AS priority, i.created_at
+      FROM item_bookmarks b
+      JOIN items i ON i.id = b.item_id
+      WHERE b.user_id IN (SELECT value FROM json_each(?))
+        AND i.link IS NOT NULL AND i.full_content_html IS NULL
+        AND i.extraction_status != 'feed_only'
+        AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+        AND NOT EXISTS (SELECT 1 FROM full_text_jobs j WHERE j.item_id = i.id)
+
+      UNION ALL
+
+      SELECT n.item_id, n.user_id, 2 AS priority, i.created_at
+      FROM item_notes n
+      JOIN items i ON i.id = n.item_id
+      WHERE n.user_id IN (SELECT value FROM json_each(?))
+        AND i.link IS NOT NULL AND i.full_content_html IS NULL
+        AND i.extraction_status != 'feed_only'
+        AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+        AND NOT EXISTS (SELECT 1 FROM full_text_jobs j WHERE j.item_id = i.id)
+
+      UNION ALL
+
+      SELECT i.id AS item_id, s.user_id, 1 AS priority, i.created_at
+      FROM subscriptions s
+      JOIN items i ON i.feed_id = s.feed_id
+      WHERE s.user_id IN (SELECT value FROM json_each(?))
+        AND COALESCE(s.full_text_mode, 'auto') = 'auto'
+        AND i.link IS NOT NULL AND i.full_content_html IS NULL
+        AND i.extraction_status != 'feed_only'
+        AND (i.extraction_retry_at IS NULL OR i.extraction_retry_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now'))
+        AND NOT EXISTS (SELECT 1 FROM full_text_jobs j WHERE j.item_id = i.id)
+    )
+    SELECT item_id AS id, user_id, MAX(priority) AS priority
+    FROM candidate_articles
+    GROUP BY item_id
+    ORDER BY priority DESC, created_at DESC LIMIT ?`).all(JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), JSON.stringify(eligible), room);
+    const add = db.prepare("INSERT OR IGNORE INTO full_text_jobs(item_id, user_id, priority) VALUES (?, ?, ?)");
+    let count = 0;
+    db.transaction(() => {
+        for (const row of candidates)
+            if (hasUserCapability(row.user_id, "fulltext"))
+                count += add.run(row.id, row.user_id, row.priority).changes;
+    })();
+    return count;
+}
+/** Explicit priority requests never run publisher I/O inside an HTTP batch request. */
+export function enqueueArticleContents(userId, ids) {
+    if (ids.length > 50 || ids.some(id => !Number.isSafeInteger(id) || id <= 0))
+        throw new Error("invalid_source_selection");
+    if (!hasUserCapability(userId, "fulltext"))
+        throw new Error("capability_not_available");
+    const unique = [...new Set(ids)];
+    if (unique.some(id => !canAccessItem(userId, id)))
+        throw new Error("item_not_found");
+    return db.transaction(() => {
+        const count = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
+        if (count.n + unique.length > 1000)
+            throw new Error("queue_full");
+        const add = db.prepare("INSERT INTO full_text_jobs(item_id,user_id,priority) VALUES (?,?,4) ON CONFLICT(item_id) DO UPDATE SET priority = MAX(priority,4), user_id = excluded.user_id");
+        const queued = [];
+        for (const id of unique) {
+            const item = findItemById(id);
+            if (item.full_content_html || item.extraction_status === "feed_only" || findSubscriptionFullTextMode(userId, item.feed_id) === "never")
+                continue;
+            if (item.extraction_retry_at && Date.parse(item.extraction_retry_at) > Date.now())
+                continue;
+            add.run(id, userId);
+            queued.push(id);
+        }
+        return queued;
+    })();
+}
+/** Synchronous consumers can prepare selected articles without waiting for the worker. */
+export async function prepareArticleContents(userId, ids) {
+    if (ids.length > 50 || ids.some(id => !Number.isSafeInteger(id) || id <= 0))
+        throw new Error("invalid_source_selection");
+    const unique = [...new Set(ids)];
+    if (unique.some(id => !canAccessItem(userId, id)))
+        throw new Error("item_not_found");
+    if (!hasUserCapability(userId, "fulltext"))
+        throw new Error("capability_not_available");
+    const results = [];
+    // Avoid queueing an unbounded number of suspended host slots.
+    for (let start = 0; start < unique.length; start += 2) {
+        results.push(...await Promise.all(unique.slice(start, start + 2).map(async (id) => {
+            if (!hasUserCapability(userId, "fulltext"))
+                throw new Error("capability_not_available");
+            const result = await loadFullText(userId, id);
+            return { id, ok: result.ok, ...(!result.ok ? { error: result.error } : {}) };
+        })));
+    }
+    return results;
+}
+let running = false;
+export async function runFullTextQueue() {
+    if (running)
+        return;
+    running = true;
+    try {
+        const queued = db.prepare("SELECT COUNT(*) n FROM full_text_jobs").get();
+        if (queued.n < 200)
+            scheduleFullTexts();
+        const jobs = db.prepare("SELECT * FROM full_text_jobs WHERE available_at <= ? ORDER BY priority DESC, available_at LIMIT 1").all(new Date().toISOString());
+        await Promise.all(jobs.map(async (job) => {
+            const item = findItemById(job.item_id);
+            if (item && hasUserCapability(job.user_id, "fulltext") && canAccessItem(job.user_id, job.item_id) && findSubscriptionFullTextMode(job.user_id, item.feed_id) !== "never") {
+                await loadFullText(job.user_id, job.item_id);
+            }
+            db.prepare("DELETE FROM full_text_jobs WHERE item_id = ?").run(job.item_id);
+        }));
+    }
+    finally {
+        running = false;
+    }
+}
+export function startFullTextScheduler() {
+    const tick = () => { void runFullTextQueue().catch(() => console.error("[fulltext] queue failed")); };
+    // One-time startup repair of old consent-only caches, without contacting publishers.
+    const cached = db.prepare("SELECT id, full_content_html FROM items WHERE full_content_html IS NOT NULL AND (lower(full_content_html) LIKE '%cookie%' OR lower(full_content_html) LIKE '%consent%') LIMIT 100").all();
+    db.transaction(() => {
+        for (const item of cached)
+            if (isCachedConsentSnippet(item.full_content_html)) {
+                db.prepare("UPDATE items SET full_content_html = NULL, extraction_status = 'pending', extraction_retry_at = NULL WHERE id = ?").run(item.id);
+            }
+    })();
+    tick();
+    const timer = setInterval(tick, 15000);
+    timer.unref();
+    return timer;
+}
+//# sourceMappingURL=fullTextQueue.js.map