feedkeeper 0.0.0-stage → 0.23.1

package/server/dist/feeds/discovery.js

@@ -0,0 +1,153 @@
+import Parser from "rss-parser";
+import { fetchFeed } from "./fetcher.js";
+import { assertPublicHttpUrl } from "./ssrfGuard.js";
+import { decodeEntities, plainTitle } from "./text.js";
+const rssParser = new Parser();
+// Common alternate feed types found in HTML <head>
+const FEED_MIME_TYPES = [
+    "application/rss+xml",
+    "application/atom+xml",
+    "text/xml",
+    "application/xml",
+];
+const MAX_PROBES = 15;
+const COMMON_PATHS = ["/feed", "/rss.xml", "/atom.xml", "/feed.xml", "/index.xml", "/rss", "/rss.php?feed=RSS2.0", "/rss.php?feed=ATOM1.0"];
+// Feed-looking URLs anywhere in a page, including JSON embedded in attributes.
+const FEED_URL_IN_TEXT = /https?:\/\/[^\s"'<>\\]+?(?:\.rss|\.atom|\/feed\/?|\/(?:rss|atom|feed)\.xml)(?=["'\s<>\\]|$)/gi;
+function sameUrl(a, b) {
+    const normalize = (value) => {
+        try {
+            const url = new URL(value);
+            url.hash = "";
+            return url.toString().replace(/\/$/, "");
+        }
+        catch {
+            return value;
+        }
+    };
+    return normalize(a) === normalize(b);
+}
+async function parseFeedAt(url) {
+    const fetched = await fetchFeed(url);
+    const finalUrl = fetched.finalUrl || url;
+    try {
+        const parsed = await rssParser.parseString(fetched.body);
+        if (parsed.items && (parsed.title || parsed.description || parsed.items.length > 0)) {
+            return { feed: { url: finalUrl, title: plainTitle(parsed.title) ?? null, type: "application/rss+xml" }, html: null, finalUrl };
+        }
+    }
+    catch {
+        // Not a feed; the caller may look for feed links in the HTML instead.
+    }
+    return { feed: null, html: fetched.body, finalUrl };
+}
+/**
+ * Keep only candidates that really are feeds. Sites sometimes announce their own
+ * homepage as RSS, so self-references are dropped and every candidate is fetched.
+ */
+async function keepRealFeeds(candidates, pageUrl) {
+    const toProbe = candidates.filter((candidate) => !sameUrl(candidate.url, pageUrl)).slice(0, MAX_PROBES);
+    const probed = await Promise.all(toProbe.map(async (candidate) => {
+        try {
+            const { feed } = await parseFeedAt(candidate.url);
+            return feed ? { ...feed, title: candidate.title ?? feed.title } : null;
+        }
+        catch {
+            return null;
+        }
+    }));
+    const seen = new Set();
+    return probed.filter((feed) => {
+        if (!feed || seen.has(feed.url))
+            return false;
+        seen.add(feed.url);
+        return true;
+    });
+}
+/**
+ * Given a URL, determines if it is already a direct RSS/Atom feed
+ * or if it is an HTML page that links to one or more feeds (auto-discovery).
+ */
+export async function discoverFeeds(inputUrl) {
+    const validated = await assertPublicHttpUrl(inputUrl);
+    const targetUrl = validated.toString();
+    // 1. The URL is already a feed.
+    const page = await parseFeedAt(targetUrl);
+    if (page.feed)
+        return [page.feed];
+    if (!page.html)
+        return [];
+    // 2. Feeds announced or linked on the page.
+    const fromPage = await keepRealFeeds(extractFeedsFromHtml(page.html, page.finalUrl), page.finalUrl);
+    if (fromPage.length > 0)
+        return fromPage;
+    // 3. Common feed paths; some lead to an overview page that lists the site's feeds.
+    for (const path of COMMON_PATHS) {
+        try {
+            const candidateUrl = new URL(path, page.finalUrl).toString();
+            if (sameUrl(candidateUrl, targetUrl))
+                continue;
+            const candidate = await parseFeedAt(candidateUrl);
+            if (candidate.feed)
+                return [candidate.feed];
+            if (candidate.html && !sameUrl(candidate.finalUrl, page.finalUrl)) {
+                const listed = await keepRealFeeds(extractFeedsFromHtml(candidate.html, candidate.finalUrl), candidate.finalUrl);
+                if (listed.length > 0)
+                    return listed;
+            }
+        }
+        catch {
+            // Try the next path.
+        }
+    }
+    return [];
+}
+/**
+ * Collect feed candidates from an HTML page: `<link rel="alternate">` tags first,
+ * then feed-like links, then feed URLs mentioned in the page text. Candidates are
+ * unverified; discoverFeeds checks them before use.
+ */
+export function extractFeedsFromHtml(html, baseUrl) {
+    const results = [];
+    const seenUrls = new Set();
+    const add = (href, title, type) => {
+        try {
+            const resolved = new URL(href, baseUrl).toString();
+            if (!/^https?:/i.test(resolved) || seenUrls.has(resolved))
+                return;
+            seenUrls.add(resolved);
+            results.push({ url: resolved, title, type });
+        }
+        catch {
+            // Invalid URL, skip.
+        }
+    };
+    for (const match of html.matchAll(/<link\b([^>]*)\/?>/gi)) {
+        const attrs = match[1];
+        const rel = getAttr(attrs, "rel")?.toLowerCase();
+        const type = getAttr(attrs, "type")?.toLowerCase();
+        const href = getAttr(attrs, "href");
+        const title = getAttr(attrs, "title");
+        if (href && type && rel?.split(/\s+/).includes("alternate") && FEED_MIME_TYPES.includes(type)) {
+            add(href, title ? decodeEntities(title.trim()) : null, type);
+        }
+    }
+    for (const match of html.matchAll(/<a\b([^>]*href=["']([^"']+)["'][^>]*)>(.*?)<\/a>/gis)) {
+        const href = match[2];
+        if (href.endsWith("/feed") || href.endsWith(".rss") || href.endsWith(".xml") || href.includes("/rss/")) {
+            add(href, decodeEntities(match[3].replace(/<[^>]+>/g, "").trim()) || null, "application/rss+xml");
+        }
+    }
+    for (const match of decodeEntities(html).matchAll(FEED_URL_IN_TEXT)) {
+        add(match[0], null, "application/rss+xml");
+    }
+    return results;
+}
+function getAttr(attrsStr, attrName) {
+    const regex = new RegExp(`\\b${attrName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i");
+    const match = regex.exec(attrsStr);
+    if (!match)
+        return null;
+    return match[1] ?? match[2] ?? match[3] ?? null;
+}
+//# sourceMappingURL=discovery.js.map

package/server/dist/feeds/extractor.js

@@ -0,0 +1,208 @@
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import { fetch as undiciFetch } from "undici";
+import { assertPublicHttpUrl, createPublicDispatcher, SsrfBlockedError } from "./ssrfGuard.js";
+import { APP_VERSION } from "../version.js";
+const defaultFetch = undiciFetch;
+const CONSENT_SELECTORS = [
+    "#onetrust-consent-sdk",
+    "#cmpbox",
+    "#usercentrics-root",
+    "[id*='sp_message_container']",
+    "[class*='cookie-banner']",
+    "[class*='cookie-consent']",
+    "[class*='consent-banner']",
+    "[id*='cookie-notice']",
+    "[id*='cookie-banner']",
+    ".qc-cmp2-container",
+    "#didomi-host",
+    "dialog[open]",
+    ".cookie-wall",
+    ".consent-wall",
+    "[data-nosnippet]",
+];
+function cleanConsentDom(doc) {
+    for (const sel of CONSENT_SELECTORS) {
+        try {
+            doc.querySelectorAll(sel).forEach((el) => el.remove());
+        }
+        catch {
+            // ignore invalid selector in edge environments
+        }
+    }
+}
+/**
+ * Checks if a given URL or extracted HTML/text is a cookie / consent wall rather than article content.
+ */
+export function isConsentContent(finalUrl, html, title, text) {
+    if (/\/zustimmung[/?]/i.test(finalUrl) ||
+        /\/consent[/?]/i.test(finalUrl) ||
+        /\/cookie-wall/i.test(finalUrl) ||
+        /\/pur-abo[/?]/i.test(finalUrl) ||
+        /\/cmp[/?]/i.test(finalUrl) ||
+        /\/privacy-wall/i.test(finalUrl)) {
+        return true;
+    }
+    const combined = `${title || ""} ${text || ""} ${html}`.toLowerCase();
+    const strongSignatures = [
+        "cookies zustimmen",
+        "nutzung aller cookies",
+        "tracking zustimmen",
+        "einwilligung verwalten",
+        "werbung und tracking, indem sie",
+        "zustimmen & weiter",
+        "bevor sie fortfahren",
+        "accept all cookies",
+        "cookie-einstellungen",
+        "alle cookies akzeptieren",
+        "cookies akzeptieren",
+        "we value your privacy",
+    ];
+    const headText = `${title || ""} ${text || ""}`.toLowerCase().slice(0, 600);
+    const hits = strongSignatures.filter(sig => combined.includes(sig));
+    if (hits.length === 0)
+        return false;
+    const heading = (title || "").toLowerCase();
+    if (strongSignatures.some(sig => heading.includes(sig)))
+        return true;
+    return (text || "").length < 1200 && hits.length >= 2 && hits.some(sig => headText.includes(sig));
+}
+/**
+ * Checks cached HTML to see if it was a saved consent banner instead of real article content.
+ */
+export function isCachedConsentSnippet(html) {
+    return isConsentContent("", html, "", html.replace(/<[^>]+>/g, " "));
+}
+// Browser-like so article pages render normally, but identifies FeedKeeper honestly.
+const FETCH_USER_AGENT = `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 FeedKeeper/${APP_VERSION}`;
+const FETCH_TIMEOUT_MS = 12_000;
+const MAX_REDIRECTS = 5;
+class ArticleHttpError extends Error {
+    status;
+    constructor(status) {
+        super("article_http_error");
+        this.status = status;
+    }
+}
+const MAX_HTML_BYTES = 2 * 1024 * 1024;
+function stripHeavyTags(html) {
+    return html
+        .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "")
+        .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "")
+        .replace(/<svg\b[^<]*(?:(?!<\/svg>)<[^<]*)*<\/svg>/gi, "")
+        .replace(/<iframe\b[^<]*(?:(?!<\/iframe>)<[^<]*)*<\/iframe>/gi, "")
+        .replace(/<noscript\b[^<]*(?:(?!<\/noscript>)<[^<]*)*<\/noscript>/gi, "");
+}
+async function fetchArticleHtml(rawUrl, signal, dispatcher, fetchImpl) {
+    let currentUrl = rawUrl;
+    for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
+        const safeUrl = await assertPublicHttpUrl(currentUrl);
+        const response = await fetchImpl(safeUrl, {
+            signal,
+            headers: {
+                "User-Agent": FETCH_USER_AGENT,
+                Accept: "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+                // Article URLs already point to one language; don't prefer any locale.
+                "Accept-Language": "*",
+                "Cache-Control": "no-cache",
+            },
+            redirect: "manual",
+            dispatcher,
+        });
+        if (response.status >= 300 && response.status < 400) {
+            const location = response.headers.get("location");
+            if (!location)
+                return null;
+            await response.body?.cancel();
+            currentUrl = new URL(location, safeUrl).toString();
+            continue;
+        }
+        if (!response.ok) {
+            await response.body?.cancel();
+            throw new ArticleHttpError(response.status);
+        }
+        const contentLength = Number(response.headers.get("content-length"));
+        if (contentLength > MAX_HTML_BYTES)
+            return null;
+        const reader = response.body?.getReader();
+        if (!reader)
+            return null;
+        const chunks = [];
+        let total = 0;
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            total += value.byteLength;
+            if (total > MAX_HTML_BYTES) {
+                await reader.cancel();
+                return null;
+            }
+            chunks.push(value);
+        }
+        const html = new TextDecoder().decode(Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))));
+        return { html, url: safeUrl.toString() };
+    }
+    return null;
+}
+export async function extractArticleFromUrl(rawUrl, fetchImpl = defaultFetch) {
+    const result = await extractArticle(rawUrl, fetchImpl);
+    return result.status === "ok" ? result.article : null;
+}
+/** Like extractArticleFromUrl, but reports whether the site answered with a consent wall. */
+export async function extractArticle(rawUrl, fetchImpl = defaultFetch) {
+    const safeUrl = await assertPublicHttpUrl(rawUrl);
+    const dispatcher = createPublicDispatcher();
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+    try {
+        const fetched = await fetchArticleHtml(safeUrl.toString(), controller.signal, dispatcher, fetchImpl);
+        if (!fetched)
+            return { status: "failed" };
+        const cleanedHtml = stripHeavyTags(fetched.html);
+        const dom = new JSDOM(cleanedHtml, { url: fetched.url });
+        let parsed;
+        try {
+            const paywall = /"isAccessibleForFree"\s*:\s*(?:false|"false")/i.test(fetched.html);
+            cleanConsentDom(dom.window.document);
+            parsed = new Readability(dom.window.document, { charThreshold: 60 }).parse();
+            if (isConsentContent(fetched.url, parsed?.content ?? fetched.html, parsed?.title, parsed?.textContent)) {
+                return { status: "consent_wall" };
+            }
+            if (paywall || ((parsed?.textContent?.length ?? 0) < 2500 && /(?:subscribe to (?:continue|read)|sign in to (?:continue|read)|abonnieren und weiterlesen|nur für abonnenten|jetzt abonnieren.{0,40}weiterlesen)/i.test(parsed?.textContent ?? "")))
+                return { status: "paywall" };
+            if (!parsed?.content)
+                return { status: "failed" };
+        }
+        finally {
+            dom.window.close();
+        }
+        return {
+            status: (parsed.textContent?.trim().length ?? 0) < 200 ? "partial" : "ok",
+            article: {
+                title: parsed.title || null,
+                byline: parsed.byline || null,
+                contentHtml: parsed.content,
+                textContent: parsed.textContent || "",
+                excerpt: parsed.excerpt || null,
+                siteName: parsed.siteName || null,
+            },
+        };
+    }
+    catch (error) {
+        if (error instanceof SsrfBlockedError)
+            throw error;
+        if (controller.signal.aborted)
+            return { status: "timeout" };
+        if (error instanceof ArticleHttpError && [401, 402].includes(error.status))
+            return { status: "paywall" };
+        if (error instanceof ArticleHttpError && [403, 429, 503].includes(error.status))
+            return { status: "bot_blocked" };
+        return { status: "failed" };
+    }
+    finally {
+        clearTimeout(timeout);
+        dispatcher.destroy();
+    }
+}
+//# sourceMappingURL=extractor.js.map

package/server/dist/feeds/feedIcon.js

@@ -0,0 +1,171 @@
+import crypto from "node:crypto";
+import { Resvg } from "@resvg/resvg-js";
+import { extractLargestImageAsPng } from "@humanwhocodes/ico-to-png";
+import { db } from "../db/index.js";
+import { fetchImage } from "./fetcher.js";
+import { readPollSettings } from "./pollSettings.js";
+import { detectImageType } from "./archive.js";
+const MAX_ICON_BYTES = 1024 * 1024;
+const TTL_MS = 24 * 60 * 60 * 1000;
+const MAX_CACHED = 500;
+const cache = new Map();
+/** Compute stable content hash for versioning and caching. */
+export function hashIconBuffer(buffer) {
+    return crypto.createHash("sha256").update(buffer).digest("hex").slice(0, 16);
+}
+/** Icons are small, so besides raster images the formats typical for favicons are accepted. */
+export function detectIconType(buffer) {
+    const raster = detectImageType(buffer);
+    if (raster)
+        return raster.mime;
+    if (buffer.length >= 4 && buffer[0] === 0 && buffer[1] === 0 && buffer[2] === 1 && buffer[3] === 0)
+        return "image/x-icon";
+    const head = Buffer.from(buffer.subarray(0, 512)).toString("utf8").trimStart();
+    return /^(<\?xml[^>]*>\s*)?(<!--[\s\S]*?-->\s*)?<svg[\s>]/i.test(head) ? "image/svg+xml" : null;
+}
+/** Whether the user subscribes to the feed or has saved or annotated one of its articles. */
+export function canAccessFeed(userId, feedId) {
+    return Boolean(db.prepare(`SELECT 1 FROM feeds f WHERE f.id = ?
+         AND (EXISTS (SELECT 1 FROM subscriptions s WHERE s.feed_id = f.id AND s.user_id = ?)
+           OR EXISTS (SELECT 1 FROM items i JOIN item_bookmarks b ON b.item_id = i.id WHERE i.feed_id = f.id AND b.user_id = ?)
+           OR EXISTS (SELECT 1 FROM items i JOIN item_notes n ON n.item_id = i.id WHERE i.feed_id = f.id AND n.user_id = ?))`).get(feedId, userId, userId, userId));
+}
+function candidates(feed, declaredUrls) {
+    const urls = new Set(declaredUrls);
+    if (feed.icon_url)
+        urls.add(feed.icon_url);
+    for (const source of [feed.site_url, feed.url]) {
+        try {
+            if (source)
+                urls.add(`${new URL(source).origin}/favicon.ico`);
+        }
+        catch {
+            // Try the next source.
+        }
+    }
+    return [...urls];
+}
+/** Keep the same raster icon in every client, including clients without SVG or ICO support. */
+function rasterIcon(buffer, mime) {
+    if (mime === "image/svg+xml") {
+        const svg = new Resvg(buffer, { fitTo: { mode: "width", value: 512 }, font: { loadSystemFonts: false } });
+        if (svg.width > 4096 || svg.height > 4096 || svg.width < 1 || svg.height < 1 ||
+            svg.width / svg.height > 8 || svg.height / svg.width > 8)
+            throw new Error("Invalid SVG icon size");
+        return { buffer: svg.render().asPng(), mime: "image/png" };
+    }
+    if (mime === "image/x-icon") {
+        const image = extractLargestImageAsPng(buffer);
+        if (!image)
+            throw new Error("Invalid ICO icon");
+        return { buffer: Buffer.from(image.data), mime: "image/png" };
+    }
+    return { buffer, mime };
+}
+/** Store or update feed icon in database, only notifying subscribers when its content changes. */
+export function storeFeedIcon(feedId, buffer, mime) {
+    ({ buffer, mime } = rasterIcon(buffer, mime));
+    const hash = hashIconBuffer(buffer);
+    db.prepare(`INSERT INTO feed_icons (feed_id, data, mime, hash, updated_at)
+     VALUES (?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))
+     ON CONFLICT (feed_id) DO UPDATE SET data = excluded.data, mime = excluded.mime, hash = excluded.hash, updated_at = excluded.updated_at
+     WHERE feed_icons.hash != excluded.hash OR feed_icons.mime != excluded.mime`).run(feedId, buffer, mime, hash);
+    const icon = { buffer, mime, hash };
+    if (cache.size >= MAX_CACHED)
+        cache.delete(cache.keys().next().value);
+    cache.set(feedId, { icon, fetchedAt: Date.now() });
+    return icon;
+}
+/** Re-check remote candidates during the weekly feed poll, even if a stored icon exists. */
+const refreshing = new Map();
+export function refreshFeedIcon(feedId, siteUrl, declaredUrls = [], signal) {
+    const current = refreshing.get(feedId);
+    if (current)
+        return current;
+    if (refreshing.size >= readPollSettings().maxWaiters)
+        return Promise.resolve(null);
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(new DOMException("Icon deadline exceeded", "TimeoutError")), readPollSettings().attemptTimeoutMs);
+    const deadline = signal ? AbortSignal.any([controller.signal, signal]) : controller.signal;
+    const promise = refreshIcon(feedId, siteUrl, declaredUrls, deadline).finally(() => {
+        clearTimeout(timeout);
+        refreshing.delete(feedId);
+    });
+    refreshing.set(feedId, promise);
+    return promise;
+}
+async function refreshIcon(feedId, siteUrl, declaredUrls, signal) {
+    const feed = db.prepare("SELECT icon_url, site_url, url FROM feeds WHERE id = ?").get(feedId);
+    for (const url of feed ? candidates({ ...feed, site_url: siteUrl ?? feed.site_url }, declaredUrls) : []) {
+        if (signal.aborted)
+            break;
+        try {
+            const { buffer } = await fetchImage(url, MAX_ICON_BYTES, signal);
+            const mime = detectIconType(buffer);
+            if (mime) {
+                const icon = storeFeedIcon(feedId, buffer, mime);
+                db.prepare("UPDATE feeds SET icon_url = ? WHERE id = ?").run(url, feedId);
+                return icon;
+            }
+        }
+        catch {
+            // Keep the last good icon and try the next candidate.
+        }
+    }
+    return null;
+}
+/** Retrieve icon hash from database without reading full blob. */
+export function getFeedIconHash(feedId) {
+    const row = db.prepare("SELECT hash FROM feed_icons WHERE feed_id = ?").get(feedId);
+    return row?.hash ?? null;
+}
+/**
+ * The feed's icon, fetched by this server so the browser and native apps can keep it for offline use
+ * without asking third-party sites. Stored persistently in the database and cached in memory.
+ */
+export async function loadFeedIcon(feedId) {
+    const known = cache.get(feedId);
+    if (known && Date.now() - known.fetchedAt < TTL_MS)
+        return known.icon;
+    // Check persistent storage
+    const row = db.prepare("SELECT data, mime, hash FROM feed_icons WHERE feed_id = ?").get(feedId);
+    if (row) {
+        // Icons stored by older versions may still be SVG or ICO.
+        let icon;
+        try {
+            icon = row.mime === "image/svg+xml" || row.mime === "image/x-icon"
+                ? storeFeedIcon(feedId, row.data, row.mime)
+                : { buffer: row.data, mime: row.mime, hash: row.hash };
+        }
+        catch {
+            return refreshFeedIcon(feedId);
+        }
+        if (cache.size >= MAX_CACHED)
+            cache.delete(cache.keys().next().value);
+        cache.set(feedId, { icon, fetchedAt: Date.now() });
+        return icon;
+    }
+    // Capacity pressure is temporary, not evidence that the publisher has no icon.
+    // Do not poison the 24-hour negative cache when another refresh can try shortly.
+    if (!refreshing.has(feedId) && refreshing.size >= readPollSettings().maxWaiters)
+        return null;
+    const icon = await refreshFeedIcon(feedId);
+    if (!icon) {
+        if (cache.size >= MAX_CACHED)
+            cache.delete(cache.keys().next().value);
+        cache.set(feedId, { icon: null, fetchedAt: Date.now() });
+    }
+    return icon;
+}
+/** Forget remembered icons (tests). */
+export function clearIconCache(feedId) {
+    if (feedId !== undefined) {
+        cache.delete(feedId);
+        db.prepare("DELETE FROM feed_icons WHERE feed_id = ?").run(feedId);
+    }
+    else {
+        cache.clear();
+        db.prepare("DELETE FROM feed_icons").run();
+    }
+}
+//# sourceMappingURL=feedIcon.js.map