feedkeeper 0.0.0-stage → 0.23.0

package/server/dist/feeds/icon.js

@@ -0,0 +1,97 @@
+import { fetchFeed } from "./fetcher.js";
+import { decodeEntities } from "./text.js";
+const ICON_RECHECK_MS = 7 * 24 * 60 * 60 * 1000;
+function attribute(tag, name) {
+    const match = tag.match(new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i"));
+    return match ? (match[1] ?? match[2] ?? match[3] ?? null) : null;
+}
+function headLinks(html) {
+    const end = html.search(/<\/head>/i);
+    return html.slice(0, end === -1 ? 200_000 : Math.min(end, 200_000)).match(/<link\b[^>]*>/gi) ?? [];
+}
+function httpUrl(href, baseUrl) {
+    try {
+        const url = new URL(decodeEntities(href), baseUrl);
+        return url.protocol === "https:" || url.protocol === "http:" ? url.toString() : null;
+    }
+    catch {
+        return null;
+    }
+}
+function candidate(href, baseUrl, sizes, type, touch = false) {
+    const url = httpUrl(href, baseUrl);
+    if (!url)
+        return null;
+    // Use the shortest edge of every declared size; a wide banner is not a large square icon.
+    const dimensions = [...sizes.matchAll(/(\d+)x(\d+)/gi)].map((match) => Math.min(Number(match[1]), Number(match[2])));
+    return {
+        url,
+        vector: type.toLowerCase() === "image/svg+xml" || /\.svg$/i.test(new URL(url).pathname),
+        size: Math.max(0, ...dimensions) || (touch ? 180 : 0),
+    };
+}
+function rankedUrls(candidates) {
+    candidates.sort((a, b) => Number(b.vector) - Number(a.vector) || b.size - a.size);
+    return [...new Set(candidates.map((icon) => icon.url))].slice(0, 16);
+}
+function declaredCandidates(html, baseUrl) {
+    const candidates = [];
+    for (const tag of headLinks(html)) {
+        const rel = attribute(tag, "rel")?.toLowerCase().split(/\s+/) ?? [];
+        const href = attribute(tag, "href");
+        if (!href || rel.includes("mask-icon"))
+            continue;
+        const touch = rel.includes("apple-touch-icon") || rel.includes("apple-touch-icon-precomposed");
+        if (!touch && !rel.includes("icon"))
+            continue;
+        const icon = candidate(href, baseUrl, attribute(tag, "sizes") ?? "", attribute(tag, "type") ?? "", touch);
+        if (icon)
+            candidates.push(icon);
+    }
+    return candidates;
+}
+/** Prefer scalable artwork, then the largest declared raster icons, including touch icons. */
+export function pickIconUrls(html, baseUrl) {
+    return rankedUrls(declaredCandidates(html, baseUrl));
+}
+export function pickIconUrl(html, baseUrl) {
+    return pickIconUrls(html, baseUrl)[0] ?? null;
+}
+export function iconCheckDue(checkedAt) {
+    return !checkedAt || Date.now() - Date.parse(checkedAt) > ICON_RECHECK_MS;
+}
+/** Include a linked web-app manifest, which often contains the site's highest-resolution icons. */
+export async function discoverIconUrls(siteUrl, signal) {
+    try {
+        const page = await fetchFeed(siteUrl, { signal });
+        const candidates = declaredCandidates(page.body, page.finalUrl);
+        const manifestTag = headLinks(page.body).find((tag) => attribute(tag, "rel")?.toLowerCase().split(/\s+/).includes("manifest"));
+        const href = manifestTag && attribute(manifestTag, "href");
+        const manifestUrl = href && httpUrl(href, page.finalUrl);
+        if (manifestUrl) {
+            try {
+                const manifest = await fetchFeed(manifestUrl, { signal });
+                const data = JSON.parse(manifest.body);
+                const icons = data && typeof data === "object" && "icons" in data ? data.icons : null;
+                for (const entry of Array.isArray(icons) ? icons.slice(0, 32) : []) {
+                    if (!entry || typeof entry !== "object" || typeof entry.src !== "string")
+                        continue;
+                    const purpose = typeof entry.purpose === "string" ? entry.purpose.split(/\s+/) : ["any"];
+                    if (!purpose.includes("any") && !purpose.includes("maskable"))
+                        continue;
+                    const icon = candidate(entry.src, manifest.finalUrl, typeof entry.sizes === "string" ? entry.sizes : "", typeof entry.type === "string" ? entry.type : "");
+                    if (icon)
+                        candidates.push(icon);
+                }
+            }
+            catch {
+                // A missing or invalid manifest must not discard the page's own icons.
+            }
+        }
+        return rankedUrls(candidates);
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=icon.js.map

package/server/dist/feeds/imageCache.js

@@ -0,0 +1,266 @@
+import { createHash } from "node:crypto";
+import { link, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { FeedHttpError } from "./fetcher.js";
+import { detectImageType } from "./imageTypes.js";
+import { readPollSettings } from "./pollSettings.js";
+const MAX_IMAGE_BYTES = 5 * 1024 * 1024;
+const KEY = /^[0-9a-f]{64}$/;
+export class ImageCacheBusyError extends Error {
+    constructor() { super("Image cache is busy"); }
+}
+/** A disposable disk cache, shared only within a feed. One process owns its directory. */
+export class ImageCache {
+    settings;
+    now;
+    directory;
+    entries = new Map();
+    negative = new Map();
+    running = new Map();
+    ready;
+    waiting = [];
+    active = 0;
+    writes = Promise.resolve();
+    stopping = false;
+    diskAvailable = true;
+    bytes = 0;
+    counters = { hits: 0, misses: 0, shared: 0, negativeHits: 0, busy: 0, errors: 0, storageErrors: 0, evictions: 0 };
+    constructor(settings, now = Date.now) {
+        this.settings = settings;
+        this.now = now;
+        this.directory = resolve(settings.path);
+        this.ready = this.restore().catch(() => { this.diskAvailable = false; this.counters.storageErrors++; });
+    }
+    metrics() { return { ...this.counters, entries: this.entries.size, bytes: this.bytes, running: this.active, pending: this.waiting.length }; }
+    async get(scope, url, load) {
+        if (this.stopping)
+            throw new ImageCacheBusyError();
+        const key = createHash("sha256").update(`${scope}\0${url}`).digest("hex");
+        const shared = this.running.get(key);
+        if (shared) {
+            this.counters.shared++;
+            return shared.promise;
+        }
+        if (this.active >= this.settings.concurrency && this.waiting.length >= this.settings.maxWaiters) {
+            this.counters.busy++;
+            throw new ImageCacheBusyError();
+        }
+        const controller = new AbortController();
+        let complete;
+        let fail;
+        let finishWork;
+        const promise = new Promise((resolve, reject) => { complete = resolve; fail = reject; });
+        const finished = new Promise((resolve) => { finishWork = resolve; });
+        let started = false;
+        const deadline = setTimeout(() => controller.abort(new DOMException("Image deadline exceeded", "TimeoutError")), readPollSettings().requestTimeoutMs);
+        deadline.unref();
+        this.running.set(key, { promise, finished, controller });
+        const cancelWaiting = () => {
+            fail(controller.signal.reason);
+            if (started)
+                return; // Keep the slot until any outstanding I/O has actually stopped.
+            const index = this.waiting.indexOf(job);
+            if (index >= 0)
+                this.waiting.splice(index, 1);
+            this.running.delete(key);
+            clearTimeout(deadline);
+            finishWork();
+        };
+        const job = { run: () => {
+                started = true;
+                this.active++;
+                const finish = () => {
+                    clearTimeout(deadline);
+                    controller.signal.removeEventListener("abort", cancelWaiting);
+                    this.active--;
+                    this.running.delete(key);
+                    finishWork();
+                    this.pump();
+                };
+                void this.resolve(key, load, controller.signal).then((image) => { finish(); complete(image); }, (error) => { finish(); fail(error); });
+            } };
+        controller.signal.addEventListener("abort", cancelWaiting, { once: true });
+        this.waiting.push(job);
+        this.pump();
+        return promise;
+    }
+    pump() {
+        while (this.active < this.settings.concurrency && this.waiting.length)
+            this.waiting.shift().run();
+    }
+    async close() {
+        this.stopping = true;
+        const jobs = [...this.running.values()];
+        for (const { controller } of jobs)
+            controller.abort();
+        await this.ready;
+        await Promise.allSettled(jobs.map(({ finished }) => finished));
+        await this.writes;
+        await this.serialize(() => this.saveIndex()).catch(() => undefined);
+    }
+    async prune() {
+        await this.ready;
+        await this.serialize(async () => { await this.trim(); await this.saveIndex(); });
+    }
+    serialize(work) {
+        const result = this.writes.then(work);
+        this.writes = result.catch(() => { this.diskAvailable = false; this.counters.storageErrors++; });
+        return result;
+    }
+    file(key) { return join(this.directory, `${key}.img`); }
+    async restore() {
+        if (!this.settings.maxBytes) {
+            this.diskAvailable = false;
+            return;
+        }
+        await mkdir(this.directory, { recursive: true, mode: 0o700 });
+        const index = join(this.directory, "index.json");
+        let saved = [];
+        try {
+            if ((await stat(index)).size > 8 * 1024 * 1024)
+                throw new Error("Oversized image cache index");
+            saved = JSON.parse(await readFile(index, "utf8"));
+        }
+        catch { /* Missing or interrupted cache metadata is safe to discard. */ }
+        if (Array.isArray(saved)) {
+            for (const row of saved.slice(0, 20_000)) {
+                if (!row || typeof row !== "object" || !KEY.test(row.key) || !Number.isSafeInteger(row.size) || row.size <= 0 || row.size > MAX_IMAGE_BYTES ||
+                    !Number.isFinite(row.expires) || !Number.isFinite(row.used) || row.expires <= this.now() || this.entries.has(row.key))
+                    continue;
+                try {
+                    if ((await stat(this.file(row.key))).size !== row.size)
+                        continue;
+                    this.entries.set(row.key, row);
+                    this.bytes += row.size;
+                }
+                catch { /* Missing files are cache misses. */ }
+            }
+        }
+        await this.trim();
+        for (const file of await readdir(this.directory)) {
+            if ((/^[0-9a-f]{64}\.img$/.test(file) && !this.entries.has(file.slice(0, 64))) || /^[0-9a-f]{64}\.tmp$/.test(file) || file === "index.tmp") {
+                await unlink(join(this.directory, file));
+            }
+        }
+        await this.saveIndex();
+    }
+    async resolve(key, load, signal) {
+        await this.ready;
+        signal.throwIfAborted();
+        const missing = this.negative.get(key);
+        if (missing && missing.expires > this.now()) {
+            this.counters.negativeHits++;
+            if (missing.result === "unsupported")
+                return "unsupported";
+            throw new FeedHttpError(missing.result, undefined, "");
+        }
+        this.negative.delete(key);
+        const entry = this.entries.get(key);
+        if (entry && entry.expires > this.now()) {
+            try {
+                const buffer = await readFile(this.file(key), { signal });
+                const type = detectImageType(buffer);
+                if (buffer.length === entry.size && type) {
+                    entry.used = this.now();
+                    this.counters.hits++;
+                    return { buffer, mime: type.mime, path: this.file(key) };
+                }
+            }
+            catch { /* Re-fetch evicted or damaged files through the guarded downloader. */ }
+        }
+        this.counters.misses++;
+        try {
+            signal.throwIfAborted();
+            const image = await load(signal);
+            signal.throwIfAborted();
+            if (image.buffer.length > MAX_IMAGE_BYTES)
+                throw new Error("Image exceeds the allowed size");
+            const type = detectImageType(image.buffer);
+            if (!type) {
+                this.rememberNegative(key, "unsupported");
+                return "unsupported";
+            }
+            const result = { buffer: image.buffer, mime: type.mime, path: image.path };
+            const ttl = Math.min(this.settings.ttlMs, image.cacheTtlMs ?? this.settings.ttlMs);
+            if (this.diskAvailable && ttl > 0 && image.buffer.length <= this.settings.maxBytes) {
+                try {
+                    await this.serialize(async () => {
+                        if (signal.aborted || !this.diskAvailable)
+                            return;
+                        await this.remove(key);
+                        await this.trim(image.buffer.length, 1);
+                        const temporary = join(this.directory, `${key}.tmp`);
+                        await unlink(temporary).catch((error) => { if (error.code !== "ENOENT")
+                            throw error; });
+                        // Hard links let the cache and permanent archive share the same bytes.
+                        if (image.path) {
+                            await link(image.path, temporary).catch(() => writeFile(temporary, image.buffer, { mode: 0o600 }));
+                        }
+                        else
+                            await writeFile(temporary, image.buffer, { mode: 0o600 });
+                        await rename(temporary, this.file(key));
+                        this.entries.set(key, { key, size: image.buffer.length, expires: this.now() + ttl, used: this.now() });
+                        this.bytes += image.buffer.length;
+                        await this.saveIndex();
+                        result.path = this.file(key);
+                    });
+                }
+                catch {
+                    await unlink(join(this.directory, `${key}.tmp`)).catch(() => undefined);
+                    // A full or read-only cache must not break image delivery.
+                }
+            }
+            return result;
+        }
+        catch (error) {
+            this.counters.errors++;
+            if (error instanceof FeedHttpError && (error.status === 404 || error.status === 410))
+                this.rememberNegative(key, error.status);
+            throw error;
+        }
+    }
+    rememberNegative(key, result) {
+        if (!this.settings.negativeTtlMs)
+            return;
+        for (const [key, row] of this.negative)
+            if (row.expires <= this.now())
+                this.negative.delete(key);
+        while (this.negative.size >= this.settings.maxEntries)
+            this.negative.delete(this.negative.keys().next().value);
+        this.negative.set(key, { result, expires: this.now() + this.settings.negativeTtlMs });
+    }
+    async remove(key) {
+        const entry = this.entries.get(key);
+        if (!entry)
+            return;
+        // Do not forget a file we could not remove: its bytes still count towards the limit.
+        try {
+            await unlink(this.file(key));
+        }
+        catch (error) {
+            if (error.code !== "ENOENT")
+                throw error;
+        }
+        this.entries.delete(key);
+        this.bytes -= entry.size;
+        this.counters.evictions++;
+    }
+    async trim(extraBytes = 0, extraEntries = 0) {
+        const oldest = [...this.entries.values()].sort((a, b) => a.used - b.used);
+        for (const entry of oldest)
+            if (entry.expires <= this.now())
+                await this.remove(entry.key);
+        for (const entry of oldest) {
+            if (this.bytes + extraBytes <= this.settings.maxBytes && this.entries.size + extraEntries <= this.settings.maxEntries)
+                break;
+            await this.remove(entry.key);
+        }
+    }
+    async saveIndex() {
+        if (!this.diskAvailable)
+            return;
+        await writeFile(join(this.directory, "index.tmp"), JSON.stringify([...this.entries.values()]), { mode: 0o600 });
+        await rename(join(this.directory, "index.tmp"), join(this.directory, "index.json"));
+    }
+}
+//# sourceMappingURL=imageCache.js.map

package/server/dist/feeds/imageCacheSettings.js

@@ -0,0 +1,19 @@
+import { dirname, join } from "node:path";
+function integer(env, name, fallback, min, max) {
+    const value = env[name] === undefined ? fallback : Number(env[name]);
+    if (!Number.isSafeInteger(value) || value < min || value > max)
+        throw new Error(`${name} must be an integer between ${min} and ${max}`);
+    return value;
+}
+export function readImageCacheSettings(env = process.env) {
+    return {
+        path: env.IMAGE_CACHE_PATH ?? join(dirname(env.DATABASE_PATH ?? "./data/feedkeeper.sqlite"), "image-cache"),
+        maxBytes: integer(env, "IMAGE_CACHE_MAX_BYTES", 256 * 1024 * 1024, 0, 1024 ** 4),
+        maxEntries: integer(env, "IMAGE_CACHE_MAX_ENTRIES", 2_000, 1, 20_000),
+        ttlMs: integer(env, "IMAGE_CACHE_TTL_SECONDS", 21_600, 1, 604_800) * 1_000,
+        negativeTtlMs: integer(env, "IMAGE_CACHE_NEGATIVE_TTL_SECONDS", 60, 0, 300) * 1_000,
+        concurrency: integer(env, "IMAGE_CACHE_CONCURRENCY", 4, 1, 32),
+        maxWaiters: integer(env, "IMAGE_CACHE_MAX_WAITERS", 32, 0, 256),
+    };
+}
+//# sourceMappingURL=imageCacheSettings.js.map

package/server/dist/feeds/imageTypes.js

@@ -0,0 +1,16 @@
+/** SVG is excluded because it can carry scripts. */
+export function detectImageType(data) {
+    const ascii = (start, end) => String.fromCharCode(...data.subarray(start, end));
+    if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff)
+        return { mime: "image/jpeg", ext: "jpg" };
+    if (data.length >= 8 && ascii(1, 4) === "PNG" && data[0] === 0x89)
+        return { mime: "image/png", ext: "png" };
+    if (data.length >= 12 && ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP")
+        return { mime: "image/webp", ext: "webp" };
+    if (data.length >= 6 && (ascii(0, 6) === "GIF87a" || ascii(0, 6) === "GIF89a"))
+        return { mime: "image/gif", ext: "gif" };
+    if (data.length >= 12 && ascii(4, 8) === "ftyp" && ["avif", "avis"].includes(ascii(8, 12)))
+        return { mime: "image/avif", ext: "avif" };
+    return null;
+}
+//# sourceMappingURL=imageTypes.js.map

package/server/dist/feeds/inbox.js

@@ -0,0 +1,128 @@
+import { randomUUID } from "node:crypto";
+import { db } from "../db/index.js";
+import { extractArticle } from "./extractor.js";
+import { assertPublicHttpUrl, normalizeUrlCandidate } from "./ssrfGuard.js";
+import { articleImageUrls, scheduleArchive } from "./archive.js";
+import { findFeedById, findFeedByUrl, isUserSubscribed, listSubscriptionsForUser, subscribe, bookmarkItem, findItemById, findItemForUser, setItemNote, } from "./repository.js";
+function escapeHtml(str) {
+    return str
+        .replace(/&/g, "&")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&#039;");
+}
+export function ensureUserInbox(userId) {
+    const inboxUrl = `feedkeeper://inbox/${userId}`;
+    let feed = findFeedByUrl(inboxUrl);
+    if (!feed) {
+        db.prepare(`INSERT INTO feeds (url, title, is_system_inbox, poll_interval_minutes)
+       VALUES (?, 'Inbox', 1, 0)
+       ON CONFLICT (url) DO UPDATE SET is_system_inbox = 1`).run(inboxUrl);
+        feed = findFeedByUrl(inboxUrl);
+    }
+    else if (!feed.is_system_inbox) {
+        db.prepare("UPDATE feeds SET is_system_inbox = 1 WHERE id = ?").run(feed.id);
+        feed = findFeedById(feed.id);
+    }
+    if (!isUserSubscribed(userId, feed.id)) {
+        subscribe(userId, feed.id, "Inbox");
+    }
+    const sub = listSubscriptionsForUser(userId).find((s) => s.id === feed.id);
+    if (!sub) {
+        throw new Error("Failed to ensure user inbox subscription");
+    }
+    return sub;
+}
+export async function saveToInbox(userId, input) {
+    let targetUrl = null;
+    let title = input.title?.trim() || null;
+    let contentHtml = input.contentHtml?.trim() || null;
+    let snippet = null;
+    let extractionStatus = contentHtml ? "ready" : "pending";
+    let extractedPartial = false;
+    let imageUrl = null;
+    if (input.url?.trim()) {
+        const normalized = normalizeUrlCandidate(input.url.trim());
+        await assertPublicHttpUrl(normalized);
+        targetUrl = normalized;
+        if (!contentHtml) {
+            try {
+                const extracted = await extractArticle(targetUrl);
+                if (extracted.status === "ok" || extracted.status === "partial") {
+                    extractionStatus = extracted.status === "ok" ? "ready" : "partial";
+                    extractedPartial = extracted.status === "partial";
+                    if (!title && extracted.article.title) {
+                        title = extracted.article.title;
+                    }
+                    contentHtml = extracted.article.contentHtml;
+                    snippet = extracted.article.excerpt ?? (extracted.article.textContent ? extracted.article.textContent.slice(0, 300) : null);
+                    const images = articleImageUrls(contentHtml, targetUrl);
+                    imageUrl = images[0] ?? null;
+                }
+            }
+            catch (err) {
+                console.warn(`[inbox] Article extraction failed for ${targetUrl}:`, err);
+            }
+        }
+    }
+    if (!contentHtml && input.textContent?.trim()) {
+        const cleanText = input.textContent.trim();
+        contentHtml = cleanText
+            .split(/\n\s*\n/)
+            .map((p) => `<p>${escapeHtml(p.trim())}</p>`)
+            .join("\n");
+        if (!snippet) {
+            snippet = cleanText.slice(0, 300);
+        }
+    }
+    if (!title) {
+        title = targetUrl ?? (snippet ? snippet.slice(0, 80) : "Saved Clipping");
+    }
+    const inbox = ensureUserInbox(userId);
+    const now = new Date().toISOString();
+    const guid = targetUrl ? targetUrl : `urn:feedkeeper:inbox:${randomUUID()}`;
+    const row = db
+        .prepare(`INSERT INTO items (feed_id, guid, title, link, content_snippet, content_html, full_content_html, image_url, published_at, created_at, extraction_status, extraction_retry_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT (feed_id, guid) DO UPDATE SET
+         title = COALESCE(excluded.title, items.title),
+         link = COALESCE(excluded.link, items.link),
+         content_snippet = COALESCE(excluded.content_snippet, items.content_snippet),
+         content_html = COALESCE(excluded.content_html, items.content_html),
+         full_content_html = COALESCE(excluded.full_content_html, items.full_content_html),
+         image_url = COALESCE(excluded.image_url, items.image_url),
+         extraction_status = CASE WHEN COALESCE(excluded.full_content_html, items.full_content_html) IS NOT NULL THEN 'ready' ELSE excluded.extraction_status END,
+         extraction_retry_at = excluded.extraction_retry_at
+       RETURNING id`)
+        .get(inbox.id, guid, title, targetUrl, snippet, contentHtml, extractedPartial ? null : contentHtml, imageUrl, now, now, contentHtml && !extractedPartial ? "ready" : extractionStatus, extractedPartial ? new Date(Date.now() + 86400_000).toISOString() : null);
+    const itemId = row.id;
+    bookmarkItem(userId, itemId);
+    let hasNote = false;
+    if (input.note?.trim()) {
+        setItemNote(userId, itemId, input.note.trim());
+        hasNote = true;
+    }
+    scheduleArchive(itemId);
+    const item = findItemForUser(userId, itemId);
+    if (!item) {
+        throw new Error("Item was saved but could not be retrieved");
+    }
+    return { itemId, item, hasNote };
+}
+export function deleteInboxItem(userId, itemId) {
+    const inbox = ensureUserInbox(userId);
+    const item = findItemById(itemId);
+    if (!item || item.feed_id !== inbox.id) {
+        return false;
+    }
+    db.transaction(() => {
+        db.prepare("DELETE FROM item_bookmarks WHERE item_id = ?").run(itemId);
+        db.prepare("DELETE FROM item_reads WHERE item_id = ?").run(itemId);
+        db.prepare("DELETE FROM item_progress WHERE item_id = ?").run(itemId);
+        db.prepare("DELETE FROM item_notes WHERE item_id = ?").run(itemId);
+        db.prepare("DELETE FROM items WHERE id = ?").run(itemId);
+    })();
+    return true;
+}
+//# sourceMappingURL=inbox.js.map