feedkeeper 0.35.3 → 0.35.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.35.3",
+  "version": "0.35.4",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/api/items.js

@@ -49,8 +49,8 @@ itemsRouter.get("/", (req, res) => {
     res.json(items.map(({ content_lead: _lead, lead_version: _leadVersion, ...item }) => ({
         ...item,
         content_snippet: cleanSnippet(item.content_snippet, item.title),
-        content_html: readerHtml(item.content_html, item.title, item.author),
-        full_content_html: readerHtml(item.full_content_html, item.title, item.author),
+        content_html: readerHtml(item.content_html, item.title, item.author, item.link),
+        full_content_html: readerHtml(item.full_content_html, item.title, item.author, item.link),
         read: Boolean(item.read),
         bookmarked: Boolean(item.bookmarked),
     })));
@@ -127,7 +127,7 @@ itemsRouter.post("/:itemId/extract-content", async (req, res) => {
     const [served] = withArchivedImages([{ id: result.id, link: currentItem?.link ?? null, full_content_html: result.fullContentHtml }], requestOrigin(req));
     res.json({
         id: result.id,
-        full_content_html: readerHtml(served.full_content_html, result.title ?? currentItem?.title, currentItem?.author),
+        full_content_html: readerHtml(served.full_content_html, result.title ?? currentItem?.title, currentItem?.author, currentItem?.link),
         ...(result.cached ? {} : { title: result.title, byline: result.byline }),
         cached: result.cached,
     });

package/server/dist/api/v1/serialize.js

@@ -38,7 +38,7 @@ export function serializeItem(row, progress, withContent, hasNote = false) {
         imageUrl: row.image_url,
         hasFullText: Boolean(row.full_content_html),
         content: contentMetadata(row),
-        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title, row.author), fullTextHtml: readerHtml(row.full_content_html, row.title, row.author) } : {}),
+        ...(withContent ? { contentHtml: readerHtml(row.content_html, row.title, row.author, row.link), fullTextHtml: readerHtml(row.full_content_html, row.title, row.author, row.link) } : {}),
         state: {
             read: Boolean(row.read),
             saved: Boolean(row.bookmarked),

package/server/dist/feeds/articleContent.js

@@ -21,14 +21,14 @@ const readerCache = new Map();
  * Article HTML as a reader shows it, without breadcrumbs, repeated headline and similar page chrome. Computed when
  * served and never stored, so the publisher's original stays intact and rule changes apply to old articles too.
  */
-export function readerHtml(html, title, author) {
+export function readerHtml(html, title, author, url) {
     if (!html)
         return html ?? null;
-    const key = createHash("sha1").update(title ?? "").update("\0").update(author ?? "").update("\0").update(html).digest("base64");
+    const key = createHash("sha1").update(title ?? "").update("\0").update(author ?? "").update("\0").update(url ?? "").update("\0").update(html).digest("base64");
     const cached = readerCache.get(key);
     if (cached !== undefined)
         return cached;
-    const cleaned = cleanReaderHtml(html, title, {}, author);
+    const cleaned = cleanReaderHtml(html, title, {}, author, url);
     if (readerCache.size >= READER_CACHE_LIMIT)
         readerCache.delete(readerCache.keys().next().value);
     readerCache.set(key, cleaned);

package/server/dist/feeds/content/index.js

@@ -1,3 +1,4 @@
+import { lazyImages } from "./noise/lazyImages.js";
 import { EMBED_RULES, resolveEmbeds } from "./embeds/index.js";
 import { embedHeadings } from "./embeds/headings.js";
 import { boilerplate } from "./noise/boilerplate.js";
@@ -10,7 +11,7 @@ import { teaserLinks } from "./noise/teaserLinks.js";
  * tags that are removed next), then the clutter around the story. The tests check each rule against its fixtures in
  * `test/fixtures/content/<rule name>` and that running it twice changes nothing. See `docs/content-rules.md`.
  */
-export const PAGE_RULES = [...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate, teaserLinks];
+export const PAGE_RULES = [lazyImages, ...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate, teaserLinks];
 export { resolveEmbeds };
 /** What a fetched page goes through before its scripts, frames and `noscript` blocks are removed. */
 export function cleanPage(root, stats = {}) {

package/server/dist/feeds/content/noise/lazyImages.js

@@ -0,0 +1,22 @@
+/** A script accidentally parsed as a URL is never an image, even after it was made absolute. */
+export const SCRIPT_IMAGE_SOURCE = /\bthis\.src\s*=/i;
+export const lazyImages = {
+    name: "lazy-images",
+    description: "Lazy image addresses replace missing, data placeholders or script fragments parsed as image URLs.",
+    apply(root, stats) {
+        let changed = false;
+        for (const image of root.querySelectorAll("img")) {
+            const current = image.getAttribute("src") ?? "";
+            if (current && !/^data:/i.test(current) && !SCRIPT_IMAGE_SOURCE.test(current))
+                continue;
+            const source = image.getAttribute("data-lazy-src") || image.getAttribute("data-src") || image.getAttribute("data-original");
+            if (!source || !/^(?:https?:\/\/|\/)/i.test(source) || SCRIPT_IMAGE_SOURCE.test(source))
+                continue;
+            image.setAttribute("src", source);
+            stats["lazy-images"] = (stats["lazy-images"] ?? 0) + 1;
+            changed = true;
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=lazyImages.js.map

package/server/dist/feeds/content/noise/videoPages.js

@@ -0,0 +1,67 @@
+import { flat, linkParagraph } from "../shared.js";
+import de from "../patterns/video-pages/de.js";
+import en from "../patterns/video-pages/en.js";
+import ja from "../patterns/video-pages/ja.js";
+const labels = new Set([...de, ...en, ...ja].map(value => value.toLowerCase()));
+const date = /\b\d{1,2}\.\d{1,2}\.\d{4}\b|\b[A-Za-z]{3,9} \d{1,2},? \d{4}\b|\d{4}年\d{1,2}月\d{1,2}日/;
+/** A video landing page with a poster and metadata, but no editorial paragraphs. */
+export function cleanVideoPage(root, url, title, stats) {
+    if (!url)
+        return false;
+    try {
+        const address = new URL(url);
+        if (!/^https?:$/.test(address.protocol) || !/(^|\/)(videos?|mediathek|watch|clips?)(\/|$|[-.])/i.test(address.pathname))
+            return false;
+    }
+    catch {
+        return false;
+    }
+    const mains = root.querySelectorAll("main");
+    if (mains.length !== 1)
+        return false;
+    const main = mains[0];
+    const images = main.querySelectorAll("img");
+    if (images.length !== 1 || main.querySelector("article, blockquote, table, a[data-media]"))
+        return false;
+    for (const link of main.querySelectorAll("a[href]")) {
+        if (link.closest("nav, ul, ol") || link.querySelector("img"))
+            continue;
+        // A home link is navigation; a transcript or another editorial link is content and must survive.
+        try {
+            const target = new URL(link.getAttribute("href"), url);
+            if (target.origin !== new URL(url).origin || target.pathname !== "/" || target.search || target.hash)
+                return false;
+        }
+        catch {
+            return false;
+        }
+    }
+    const copy = main.cloneNode(true);
+    copy.querySelectorAll("nav, ul, ol, form, img, picture, a, h1").forEach(element => element.remove());
+    const blocks = [...copy.querySelectorAll("p")].filter(element => flat(element.textContent));
+    if (!blocks.some(element => labels.has(flat(element.textContent).toLowerCase())))
+        return false;
+    // All remaining words must be short metadata lines containing an explicit date.
+    for (const block of blocks) {
+        const text = flat(block.textContent);
+        if (!labels.has(text.toLowerCase()) && (text.length > 160 || !date.test(text)))
+            return false;
+        block.remove();
+    }
+    if (flat(copy.textContent))
+        return false;
+    const document = main.ownerDocument;
+    const poster = document.createElement("p");
+    poster.append(images[0].cloneNode(true));
+    const card = linkParagraph(document, url, flat(title) || "Video");
+    card.querySelector("a").setAttribute("data-media", "video");
+    // Preserve attribution/broadcast metadata; omit the repeated publication timestamp and kind label.
+    const credits = [...main.querySelectorAll("p")].filter(element => {
+        const text = flat(element.textContent);
+        return date.test(text) && /[,、]/.test(text) && text.length <= 160 && !element.querySelector("a, img");
+    }).map(element => element.cloneNode(true));
+    root.replaceChildren(poster, card, ...credits);
+    stats["video-page"] = (stats["video-page"] ?? 0) + 1;
+    return true;
+}
+//# sourceMappingURL=videoPages.js.map

package/server/dist/feeds/content/patterns/snippet-metadata/de.js

@@ -0,0 +1,3 @@
+/** Whole bylines with an explicit timestamp, only ahead of the first editorial line. */
+export default [/^von\s+[^.!?\n]{2,60}\s+[I|·–—-]\s+\d{1,2}\.\d{1,2}\.\d{2,4}\s*[-–|·]\s*\d{1,2}:\d{2}$/iu];
+//# sourceMappingURL=de.js.map

package/server/dist/feeds/content/patterns/snippet-metadata/en.js

@@ -0,0 +1,3 @@
+/** Whole bylines with an explicit timestamp, only ahead of the first editorial line. */
+export default [/^(?:by|written by)\s+[^.!?\n]{2,60}\s+[I|·–—-]\s+(?:[A-Za-z]{3,9}\s+\d{1,2},?\s+\d{4}|\d{4}-\d{2}-\d{2})\s*[-–|·]\s*\d{1,2}:\d{2}(?:\s*[ap]m)?$/iu];
+//# sourceMappingURL=en.js.map

package/server/dist/feeds/content/patterns/snippet-metadata/index.js

@@ -0,0 +1,10 @@
+import de from "./de.js";
+import en from "./en.js";
+import ja from "./ja.js";
+export const SNIPPET_BYLINES = [...de, ...en, ...ja];
+/** File names and copyright notices are metadata only when they stand alone before the text. */
+export const SNIPPET_IMAGE_METADATA = [
+    /^[\p{L}\p{N}_. -]{3,120}\.(?:jpe?g|png|webp|gif|avif)$/iu,
+    /^(?:©|\(c\)|copyright(?:\s*©)?)\s*[^.!?\n]{1,100}$/iu,
+];
+//# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/snippet-metadata/ja.js

@@ -0,0 +1,3 @@
+/** Whole bylines with an explicit timestamp, only ahead of the first editorial line. */
+export default [/^(?:著者|執筆|文)[:：]\s*[^。！？\n]{1,40}\s+[I|·–—-]\s*\d{4}年\d{1,2}月\d{1,2}日\s*[-–|·]\s*\d{1,2}:\d{2}$/u];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/content/patterns/video-pages/de.js

@@ -0,0 +1,3 @@
+/** Whole metadata labels of video landing pages, never words inside prose. */
+export default ["Video", "Videobeitrag"];
+//# sourceMappingURL=de.js.map

package/server/dist/feeds/content/patterns/video-pages/en.js

@@ -0,0 +1,3 @@
+/** Whole metadata labels of video landing pages, never words inside prose. */
+export default ["Video", "Video report"];
+//# sourceMappingURL=en.js.map

package/server/dist/feeds/content/patterns/video-pages/ja.js

@@ -0,0 +1,3 @@
+/** Whole metadata labels of video landing pages, never words inside prose. */
+export default ["動画", "映像", "動画ニュース"];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/contentNoise.js

@@ -1,4 +1,7 @@
+import { cleanVideoPage } from "./content/noise/videoPages.js";
+import { lazyImages } from "./content/noise/lazyImages.js";
 import { JSDOM } from "jsdom";
+import { SNIPPET_BYLINES, SNIPPET_IMAGE_METADATA } from "./content/patterns/snippet-metadata/index.js";
 import { CONSENT_ANCHORS, CONSENT_COMPANIONS } from "./content/patterns/consent/index.js";
 import { SIDEBAR_HEADINGS } from "./content/patterns/sidebars/index.js";
 import { resolveEmbeds } from "./content/embeds/index.js";
@@ -186,7 +189,7 @@ function removeReaderNoise(root, title, stats, author) {
     let pastStart = false;
     for (const block of leafBlocks(root)) {
         const text = flat(block.textContent);
-        if (!text)
+        if (!text || block.querySelector("a[data-media]"))
             continue;
         const position = leading++;
         const heading = /^H[1-3]$/.test(block.tagName);
@@ -245,7 +248,7 @@ function removeReaderNoise(root, title, stats, author) {
         }
         if (!pastStart && text.length <= 70 && block.querySelector("time") && charsBefore(root, block) <= 600) {
             const rest = flat(text.replace(flat(block.querySelector("time")?.textContent), ""));
-            if (/^(stand|zuletzt aktualisiert|aktualisiert|ver(ö|oe)ffentlicht|updated|published|posted)?\s*[:·|•-]*\s*$/i.test(rest)) {
+            if (/^(news|ニュース|nachrichten|stand|zuletzt aktualisiert|aktualisiert|ver(ö|oe)ffentlicht|updated|published|posted)?\s*[:·|•-]*\s*$/i.test(rest)) {
                 drop(block, "dateline");
                 continue;
             }
@@ -452,17 +455,19 @@ function serialize(root) {
     return holder.innerHTML;
 }
 /** The article HTML as a reader should show it. Unchanged input is returned as is. */
-export function cleanReaderHtml(html, title, stats = {}, author) {
+export function cleanReaderHtml(html, title, stats = {}, author, url) {
     if (!html)
         return html;
     const root = JSDOM.fragment(html);
+    const videoPage = cleanVideoPage(root, url, title, stats);
+    const lazy = lazyImages.apply(root, stats);
     const images = resolveCustomImages(root, stats);
     const duplicates = collapseAdjacentImages(root, stats);
     const variants = collapseImageVariants(root, stats);
     const embeds = resolveEmbeds(root, stats);
     const noise = removeReaderNoise(root, title, stats, author);
     const typography = removeTypography(root, stats);
-    return images || duplicates || variants || embeds || noise || typography ? serialize(root) : html;
+    return videoPage || lazy || images || duplicates || variants || embeds || noise || typography ? serialize(root) : html;
 }
 /**
  * Some publishers wrap a picture in their own element and let a script fill the image: `<a-img src="/imgs/…">`
@@ -780,12 +785,23 @@ export function cleanSnippet(snippet, title) {
     for (const pattern of SNIPPET_INLINE_PATTERNS)
         text = text.replace(pattern, "");
     const headline = normalized(title ?? "");
+    let inMetadata = true;
+    text = text.split("\n").filter(line => {
+        const value = line.trim();
+        if (!value)
+            return false;
+        if (inMetadata && (matchesAny(value, SNIPPET_BYLINES) || matchesAny(value, SNIPPET_IMAGE_METADATA) ||
+            (headline && normalized(value) === headline)))
+            return false;
+        inMetadata = false;
+        return true;
+    }).join("\n");
     const lines = text.split("\n").map(line => line.trim()).filter((line, index) => line && !SNIPPET_LINE_PATTERNS.some(pattern => pattern.test(line)) && !isSocialList(line) &&
         !(index === 0 && headline && normalized(line) === headline));
     return lines.join("\n").trim();
 }
 /** Raised whenever the rules above change what they remove, so stored leads are derived again. */
-export const LEAD_VERSION = 2;
+export const LEAD_VERSION = 3;
 const LEAD_LENGTH = 600;
 function shorten(text, max) {
     if (text.length <= max)

package/server/dist/feeds/extractor.js

@@ -1,3 +1,4 @@
+import { SCRIPT_IMAGE_SOURCE } from "./content/noise/lazyImages.js";
 import { Readability } from "@mozilla/readability";
 import { JSDOM } from "jsdom";
 import { fetch as undiciFetch } from "undici";
@@ -44,7 +45,7 @@ function resolveLazyImages(doc) {
             img.getAttribute("data-original") ||
             img.getAttribute("data-actualsrc");
         const src = img.getAttribute("src");
-        if (lazySrc && (!src || src.startsWith("data:") || src.length < 10)) {
+        if (lazySrc && (!src || src.startsWith("data:") || src.length < 10 || SCRIPT_IMAGE_SOURCE.test(src))) {
             img.setAttribute("src", lazySrc);
         }
         const lazySrcset = img.getAttribute("data-lazy-srcset") || img.getAttribute("data-srcset");

package/server/dist/mcp/items.js

@@ -51,7 +51,7 @@ export function getItemForMcp(userId, itemId, maxChars = MAX_CONTENT_LENGTH, off
         return null;
     if (revision !== undefined && revision !== item.content_revision)
         throw new Error("content_revision_conflict");
-    const content = readerHtml(item.full_content_html ?? item.content_html, item.title, item.author);
+    const content = readerHtml(item.full_content_html ?? item.content_html, item.title, item.author, item.link);
     const body = format === "text" ? sourceText(item) : content ?? "";
     let end = Math.min(offset + Math.min(maxChars, MAX_CONTENT_LENGTH), body.length);
     if (end < body.length && /[\uD800-\uDBFF]/.test(body[end - 1]))

package/server/dist/mcp/server.js

@@ -42,7 +42,11 @@ function selectedItemIds({ itemId, itemIds }) {
 // instance is created per request (see mcp/http.ts) so tool handlers can
 // safely close over `userId` without leaking data between users.
 export function createMcpServerForUser(userId, scope) {
-    const server = new McpServer({ name: "feedkeeper", version: APP_VERSION });
+    const server = new McpServer({ name: "feedkeeper", version: APP_VERSION }, {
+        instructions: "FeedKeeper is a personal RSS reader, article archiver, and news curation service. " +
+            "Use it to read, search, and triage RSS/Atom feeds, fetch full article text, manage reading lists, " +
+            "and read or publish curated newspaper editions.",
+    });
     server.registerTool("list_feeds", {
         title: "List subscribed feeds",
         description: "Lists all RSS/Atom feeds the current user is subscribed to, including unread counts and polling health.",
@@ -116,8 +120,8 @@ export function createMcpServerForUser(userId, scope) {
         });
     if (scope === "write")
         server.registerTool("save_to_inbox", {
-            title: "Save article or note to Inbox",
-            description: "Saves a web article URL, raw HTML, text clipping, or markdown note directly into the user's Universal Inbox.",
+            title: "Save article or clipping to Inbox",
+            description: "Saves a web article URL, raw HTML, text clipping, or note directly into the user's Universal Inbox.",
             inputSchema: {
                 url: mcpUrlSchema.optional().describe("Web page URL to fetch, extract, and save"),
                 title: z.string().max(500).optional().describe("Optional custom title"),

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.35.3",
+  "version": "0.35.4",
   "private": true,
   "type": "module"
 }