feedkeeper 0.35.1 → 0.35.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.35.1",
+  "version": "0.35.3",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/backup-cli.js

@@ -31,7 +31,11 @@ async function main() {
         process.umask(0o077);
         const { db } = await import("./db/index.js");
         try {
-            await db.backup(filePath);
+            await db.backup(filePath, {
+                progress({ remainingPages }) {
+                    return remainingPages;
+                },
+            });
             const saved = new Database(filePath, { readonly: true, fileMustExist: true });
             try {
                 assertHealthy(saved);
@@ -72,7 +76,11 @@ async function main() {
         assertHealthy(source);
         mkdirSync(dirname(databasePath), { recursive: true, mode: 0o700 });
         process.umask(0o077);
-        await source.backup(databasePath);
+        await source.backup(databasePath, {
+            progress({ remainingPages }) {
+                return remainingPages;
+            },
+        });
     }
     finally {
         source.close();

package/server/dist/feeds/content/patterns/boilerplate/index.js

@@ -1,6 +1,7 @@
 import * as de from "./de.js";
 import * as en from "./en.js";
+import * as ja from "./ja.js";
 /** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
-export const BOILERPLATE_ANYWHERE = [de.ANYWHERE, en.ANYWHERE].flat();
-export const BOILERPLATE_TRAILING = [de.TRAILING, en.TRAILING].flat();
+export const BOILERPLATE_ANYWHERE = [de.ANYWHERE, en.ANYWHERE, ja.ANYWHERE].flat();
+export const BOILERPLATE_TRAILING = [de.TRAILING, en.TRAILING, ja.TRAILING].flat();
 //# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/boilerplate/ja.js

@@ -0,0 +1,19 @@
+/**
+ * Blocks, in Japanese, that publishers put into their text without being part of the story. A block counts only when its whole text is one
+ * of these lines (see `noise/boilerplate.ts`), so a sentence of the story that contains the words stays.
+ */
+export const ANYWHERE = [
+    // Calls to follow the site on Google or elsewhere.
+    /^(?:Google(?: ニュース)?でフォロー(?:する)?|Google ニュースで[^.!?]{2,40}をフォロー)$/i,
+    // The text of download buttons whose picture is gone.
+    /^(?:App Storeからダウンロード|Google Playで手に入れよう|Google Playで入手|App Storeでダウンロード)$/i,
+    // A picture's filename shown as text.
+    /^[\w\-. ]{3,120}\.(?:jpe?g|png|webp|gif|avif)$/i,
+];
+/** Footers at the end of the text; they count only in the second half. */
+export const TRAILING = [
+    /^(?:この記事|本記事|本稿).{3,120}(?:で初出|に初出|で公開|に掲載|転載).*$/i,
+    /^(?:この記事について(?:話し合う|語る|コメント)|コメントする)\b.{0,40}$/i,
+    /^(?:関連記事|トピック|タグ|関連タグ):.{0,200}$/i,
+];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/content/patterns/consent/index.js

@@ -1,10 +1,11 @@
 import de from "./de.js";
 import en from "./en.js";
+import ja from "./ja.js";
 /**
  * One file per language. A notice is looked for in every language at once, because an article in one language can
  * sit on a site in another. To add a language, add a file next to these and list it here.
  */
-const LANGUAGES = [de, en];
+const LANGUAGES = [de, en, ja];
 export const CONSENT_ANCHORS = LANGUAGES.flatMap(language => language.anchors);
 export const CONSENT_COMPANIONS = LANGUAGES.flatMap(language => language.companions);
 //# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/consent/ja.js

@@ -0,0 +1,14 @@
+/** Consent notices that sit where an embed would be, in Japanese. See `de.ts` for what the two lists mean. */
+export default {
+    anchors: [
+        /^(?:外部コンテンツ|おすすめの外部コンテンツ|埋め込みコンテンツ)$/i,
+        /^(?:YouTube|Vimeo|Instagram|X|Twitter|TikTok|Spotify|Facebook)のコンテンツ$/i,
+        /^(?:外部コンテンツを表示|コンテンツを表示|外部コンテンツを読み込む)$/i,
+    ],
+    companions: [
+        /^(?:外部コンテンツの読み込みに同意する|同意して表示|コンテンツを表示する)$/i,
+        /個人データが(?:外部|サードパーティ)/i,
+        /プライバシーポリシー/i,
+    ],
+};
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/content/patterns/loading/index.js

@@ -1,5 +1,6 @@
 import de from "./de.js";
 import en from "./en.js";
+import ja from "./ja.js";
 /** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
-export const LOADING_LABELS = [de, en].flat();
+export const LOADING_LABELS = [de, en, ja].flat();
 //# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/loading/ja.js

@@ -0,0 +1,8 @@
+/**
+ * Whole-line placeholders, in Japanese, that a script replaces with content. A reader that does not run the script
+ * keeps them as text. Only a block that holds nothing else counts.
+ */
+export default [
+    /^(?:読み込み中|ロード中|少々お待ちください)(?:\s*(?:\.{3}|…))?$/i,
+];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/content/patterns/play-labels/index.js

@@ -1,5 +1,6 @@
 import de from "./de.js";
 import en from "./en.js";
+import ja from "./ja.js";
 /** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
-export const PLAY_LABEL = new RegExp(`^(?:${[de, en].flat().map(label => label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})$`, "i");
+export const PLAY_LABEL = new RegExp(`^(?:${[de, en, ja].flat().map(label => label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|")})$`, "i");
 //# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/play-labels/ja.js

@@ -0,0 +1,3 @@
+/** Labels, in Japanese, that a player's play-button picture carries as its alt text. See `en.ts`. */
+export default ["動画を再生", "動画を再生する", "動画を見る", "再生", "再生する", "動画再生"];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/content/patterns/sidebars/index.js

@@ -1,6 +1,7 @@
 import de from "./de.js";
 import en from "./en.js";
+import ja from "./ja.js";
 /** One file per language, searched all at once. To add a language, add a file next to these and list it here. */
-export const SIDEBAR_HEADINGS = [de, en].flat();
+export const SIDEBAR_HEADINGS = [de, en, ja].flat();
 export { SIDEBAR_SELECTORS } from "./selectors.js";
 //# sourceMappingURL=index.js.map

package/server/dist/feeds/content/patterns/sidebars/ja.js

@@ -0,0 +1,7 @@
+/** Headings, in Japanese, that start a list of other articles. See `en.ts` for how they are used. */
+export default [
+    /^(?:関連記事|おすすめ記事|オススメ記事|あわせて読みたい|人気の記事|人気記事|アクセスランキング|注目の記事|こちらもおすすめ|最新記事|新着記事|関連リンク)$/i,
+    // A site's welcome with logo and links at the end of the text.
+    /^.+へようこそ!?$/i,
+];
+//# sourceMappingURL=ja.js.map

package/server/dist/feeds/contentNoise.js

@@ -1,6 +1,9 @@
 import { JSDOM } from "jsdom";
 import { CONSENT_ANCHORS, CONSENT_COMPANIONS } from "./content/patterns/consent/index.js";
+import { SIDEBAR_HEADINGS } from "./content/patterns/sidebars/index.js";
 import { resolveEmbeds } from "./content/embeds/index.js";
+import { loadingPlaceholders } from "./content/noise/loading.js";
+import { trackingPixels } from "./content/noise/pixels.js";
 import { teaserLinks } from "./content/noise/teaserLinks.js";
 import { markVideoTeasers } from "./content/noise/videoTeasers.js";
 import { flat, pruneEmpty } from "./content/shared.js";
@@ -76,6 +79,7 @@ function isStructuredData(text) {
 }
 function removeReaderNoise(root, title, stats, author) {
     let changed = false;
+    const total = (root.textContent ?? "").length;
     const drop = (element, rule) => {
         if (!element.parentNode)
             return;
@@ -88,16 +92,25 @@ function removeReaderNoise(root, title, stats, author) {
     // A feed's own text ends with "[mehr]": the link to the article is no part of what is read.
     if (teaserLinks.apply(root, stats))
         changed = true;
+    if (loadingPlaceholders.apply(root, stats))
+        changed = true;
+    if (trackingPixels.apply(root, stats))
+        changed = true;
     for (const selector of BREADCRUMB_MARKERS) {
         try {
             root.querySelectorAll(selector).forEach(element => drop(element, "breadcrumb"));
         }
         catch { /* selector unsupported by the DOM */ }
     }
-    // Breadcrumbs without markup hints: a short list near the start whose items are single short links.
+    // Short list of single links near the start (breadcrumbs) or in the tail (topic/tag chips).
     for (const list of root.querySelectorAll("ol, ul")) {
         const items = [...list.children].filter(child => child.tagName === "LI");
-        if (items.length < 2 || items.length > 8 || charsBefore(root, list) > 300)
+        if (items.length < 2 || items.length > 8)
+            continue;
+        const before = charsBefore(root, list);
+        const isHead = before <= 300;
+        const isTail = (total - before) <= TAIL_CHARACTERS;
+        if (!isHead && !isTail)
             continue;
         const onlyLinks = items.every(item => {
             const links = item.querySelectorAll("a");
@@ -105,7 +118,7 @@ function removeReaderNoise(root, title, stats, author) {
             return links.length === 1 && label.length > 0 && label.length <= 30 && flat(links[0].textContent) === label;
         });
         if (onlyLinks)
-            drop(list, "breadcrumb");
+            drop(list, isHead ? "breadcrumb" : "topic-list");
     }
     // A product card of a shop link: its picture is a CSS background, which no reader draws, and its price is a snapshot
     // that is out of date when the article is read later. What is left would be a label, a link and a number.
@@ -260,7 +273,6 @@ function removeReaderNoise(root, title, stats, author) {
         }
     }
     // A comment section's house rules at the end of the text: no reader has comments, so heading and what follows go.
-    const total = (root.textContent ?? "").length;
     for (const block of leafBlocks(root)) {
         if (!block.parentNode || !COMMENT_RULES_HEADING.test(flat(block.textContent)))
             continue;
@@ -275,6 +287,36 @@ function removeReaderNoise(root, title, stats, author) {
         drop(block, "comment-rules");
         break;
     }
+    // A consent wall, tracking choice or publisher welcome at the end: heading and everything below go.
+    for (const block of leafBlocks(root)) {
+        if (!block.parentNode || !CONSENT_WALL_HEADING.test(flat(block.textContent)))
+            continue;
+        if (total - charsBefore(root, block) > TAIL_CHARACTERS)
+            continue;
+        let node = block;
+        while (node.parentNode && node !== root) {
+            while (node.nextSibling)
+                node.nextSibling.remove();
+            node = node.parentNode;
+        }
+        drop(block, "consent-wall");
+        break;
+    }
+    // An orphan recommendation heading left without subsequent links or content.
+    for (const block of leafBlocks(root)) {
+        if (!block.parentNode)
+            continue;
+        const text = flat(block.textContent);
+        if (!SIDEBAR_HEADINGS.some(pattern => pattern.test(text)))
+            continue;
+        const parent = block.parentElement;
+        if (parent && !parent.querySelector("a[href], img, picture, video, iframe") && flat(parent.textContent) === text) {
+            drop(parent, "empty-section-label");
+        }
+        else if (!block.querySelector("a[href], img, picture, video, iframe") && block.nextElementSibling === null) {
+            drop(block, "empty-section-label");
+        }
+    }
     // Empty nested slots are structural evidence: the next story paragraph is outside this ad container.
     // Do not infer an empty ad from a standalone heading followed by text (which may itself be sponsored).
     for (const container of [...root.querySelectorAll("div, aside, section, p")]) {
@@ -591,6 +633,7 @@ const AD_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Advertorial|Ads?|S
 const AD_SLOT_LABEL = /^(Anzeige|Werbung|Advertisement|Advertising|Ads?|広告|広告掲載|宣伝)\s*[:：]?$/i;
 const PRODUCT_LABEL = /^(Produkthinweis|Produktempfehlung|Product (notice|recommendation)|商品紹介|商品案内|製品紹介)\s*[:：]?$/i;
 const COMMENT_RULES_HEADING = /^(Regeln für Kommentare|Kommentarregeln|Kommentar-Richtlinien|Community[- ]Richtlinien|(Comment(ing)?|Community|Discussion|House) (rules|guidelines|policy|policies)|Rules for comments|Commenting rules)\s*:?$/i;
+const CONSENT_WALL_HEADING = /^(?:Weiter mit Werbung|Mit Werbung weitersurfen|Weiterlesen mit Werbung|Zustimmen und weiter(?:lesen)?|Zustimmung erforderlich|Willkommen bei [^.!?]{2,40}!?|Einwilligung zu (?:Cookies|Tracking|Datenverarbeitung)|Cookie[- ]Einstellungen|Datenschutzoptionen|Continue with (?:ads|advertising|personalized advertising)|(?:Accept|Agree) and continue|Welcome to [^.!?]{2,40}!?|Cookie (?:and|&) (?:privacy|tracking) settings|Privacy (?:and|&) cookie policy|広告(?:付き|に同意して)で?続ける|同意して(?:続ける|進む)|.+へようこそ!?|Cookie(?:と|および)プライバシーの?設定)\s*:?$/i;
 const TAG_LABEL = /^(Schlagw(ö|oe)rter|Schlagworte|Tags?|Keywords|Themen|Topics)\s*:?$/i;
 const VIDEO_CARD = /^\d{1,2}:\d{2}(:\d{2})?(?=\D)\s*\S/;
 const SECTION_LABEL = /^(Auch interessant|Das könnte (dich|Sie|euch) (auch )?interessieren|Mehr zum Thema|Lesen Sie auch|Lies auch|Lest auch|Aktuelle News|Weitere (Artikel|Meldungen|News)|Popular Stories|Related (Stories|Articles|Posts|Reading)|More (Stories|from)|Top Stories|Meistgelesen|Topaktuell|Regeln für Kommentare|Kommentare|Comments?)\b\s*:?/i;

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.35.1",
+  "version": "0.35.3",
   "private": true,
   "type": "module"
 }