feedkeeper 0.35.5 → 0.35.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "feedkeeper",
-  "version": "0.35.5",
+  "version": "0.35.7",
   "description": "A lightweight, self-hosted RSS reader and remote MCP server for your news, blogs, and AI workflows.",
   "keywords": [
     "rss",

package/server/dist/feeds/content/index.js

@@ -4,6 +4,7 @@ import { embedHeadings } from "./embeds/headings.js";
 import { boilerplate } from "./noise/boilerplate.js";
 import { loadingPlaceholders } from "./noise/loading.js";
 import { trackingPixels } from "./noise/pixels.js";
+import { permalinkGlyphs } from "./noise/permalinkGlyphs.js";
 import { sidebars } from "./noise/sidebars.js";
 import { teaserLinks } from "./noise/teaserLinks.js";
 /**
@@ -11,7 +12,7 @@ import { teaserLinks } from "./noise/teaserLinks.js";
  * tags that are removed next), then the clutter around the story. The tests check each rule against its fixtures in
  * `test/fixtures/content/<rule name>` and that running it twice changes nothing. See `docs/content-rules.md`.
  */
-export const PAGE_RULES = [lazyImages, ...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate, teaserLinks];
+export const PAGE_RULES = [lazyImages, ...EMBED_RULES, embedHeadings, sidebars, loadingPlaceholders, trackingPixels, boilerplate, teaserLinks, permalinkGlyphs];
 export { resolveEmbeds };
 /** What a fetched page goes through before its scripts, frames and `noscript` blocks are removed. */
 export function cleanPage(root, stats = {}) {

package/server/dist/feeds/content/noise/permalinkGlyphs.js

@@ -0,0 +1,52 @@
+import { count, flat, pruneEmpty } from "../shared.js";
+/** One to three characters that are neither letters nor digits: "★", "∞", "¶", "#", "↩". Non-breaking spaces around them count as blanks. */
+const GLYPH_ONLY = /^[^\p{L}\p{N}]{1,3}$/u;
+/** Whether any text follows the element in the document: only the tail of a story is looked at. */
+function hasTextAfter(root, element) {
+    const document = root.nodeType === 9 ? root : root.ownerDocument;
+    if (!document)
+        return false;
+    const walker = document.createTreeWalker(root, 4);
+    let reached = false;
+    for (let node = walker.nextNode(); node; node = walker.nextNode()) {
+        if (element.contains(node)) {
+            reached = true;
+            continue;
+        }
+        if (reached && flat(node.textContent) !== "")
+            return true;
+    }
+    return false;
+}
+/**
+ * Some blogs end every post with a permanent link that is only a symbol ("★", "∞", "¶"): a link to the post itself, of no use inside a reader.
+ * The block must hold nothing but that one link, come after the text of the story and have nothing after it. A symbol inside a sentence,
+ * a link with a picture and a block with more text stay.
+ */
+export const permalinkGlyphs = {
+    name: "permalink-glyphs",
+    description: "A block at the end of a story that holds only one link made of a single symbol such as \"★\" or \"¶\" is removed.",
+    apply(root, stats) {
+        let changed = false;
+        const total = flat(root.textContent).length;
+        for (const block of [...root.querySelectorAll("p, div")].reverse()) {
+            if (!root.contains(block) || block.querySelector("p, div, img, picture, video, iframe"))
+                continue;
+            const links = block.querySelectorAll("a[href]");
+            if (links.length !== 1)
+                continue;
+            const text = (block.textContent ?? "").replace(/[\s\u00a0]+/g, "");
+            if (!GLYPH_ONLY.test(text) || flat(links[0].textContent).replace(/[\s\u00a0]+/g, "") !== text)
+                continue;
+            if (total - text.length < 120 || hasTextAfter(root, block))
+                continue;
+            const parent = block.parentNode;
+            block.remove();
+            pruneEmpty(parent);
+            count(stats, "permalink-glyphs");
+            changed = true;
+        }
+        return changed;
+    },
+};
+//# sourceMappingURL=permalinkGlyphs.js.map

package/server/dist/feeds/contentNoise.js

@@ -7,6 +7,7 @@ import { SIDEBAR_HEADINGS } from "./content/patterns/sidebars/index.js";
 import { resolveEmbeds } from "./content/embeds/index.js";
 import { loadingPlaceholders } from "./content/noise/loading.js";
 import { trackingPixels } from "./content/noise/pixels.js";
+import { permalinkGlyphs } from "./content/noise/permalinkGlyphs.js";
 import { teaserLinks } from "./content/noise/teaserLinks.js";
 import { markVideoTeasers } from "./content/noise/videoTeasers.js";
 import { flat, pruneEmpty } from "./content/shared.js";
@@ -95,6 +96,8 @@ function removeReaderNoise(root, title, stats, author) {
     // A feed's own text ends with "[mehr]": the link to the article is no part of what is read.
     if (teaserLinks.apply(root, stats))
         changed = true;
+    if (permalinkGlyphs.apply(root, stats))
+        changed = true;
     if (loadingPlaceholders.apply(root, stats))
         changed = true;
     if (trackingPixels.apply(root, stats))

package/server/dist/feeds/imageSources.js

@@ -42,6 +42,8 @@ export function imageSource(attribute) {
     }
     return null;
 }
+/** The body of a tag up to its closing `>`: a `>` inside a quoted value, as in `media="(resolution >= 3x)"`, does not end it. */
+const TAG_BODY = `(?:[^>"']|"[^"]*"|'[^']*')*`;
 function attribute(tag, name) {
     const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i").exec(tag);
     return match ? match[1] ?? match[2] ?? match[3] : null;
@@ -56,14 +58,14 @@ export function normalizeImageSources(html) {
         const withoutSources = tag.replace(/\s+(?:(?:data-(?:lazy-)?)?src(?:set)?|sizes|data-sizes|data-original|data-actualsrc)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'>]+)/gi, "");
         return withoutSources.replace(/\s*\/?>$/, (end) => ` src="${escaped}"${end}`);
     };
-    const pictures = html.replace(/<picture\b[^>]*>[\s\S]*?<\/picture\s*>/gi, (picture) => {
+    const pictures = html.replace(new RegExp(`<picture\\b${TAG_BODY}>[\\s\\S]*?<\\/picture\\s*>`, "gi"), (picture) => {
         let fallback = null;
-        const withoutSources = picture.replace(/<source\b[^>]*>/gi, (tag) => {
+        const withoutSources = picture.replace(new RegExp(`<source\\b${TAG_BODY}>`, "gi"), (tag) => {
             fallback ??= firstSrcsetSource(attribute(tag, "data-lazy-srcset") ?? attribute(tag, "data-srcset") ?? attribute(tag, "srcset"));
             return "";
         });
-        return withoutSources.replace(/<img\b[^>]*>/gi, (tag) => normalize(tag, fallback));
+        return withoutSources.replace(new RegExp(`<img\\b${TAG_BODY}>`, "gi"), (tag) => normalize(tag, fallback));
     });
-    return pictures.replace(/<img\b[^>]*>/gi, (tag) => normalize(tag));
+    return pictures.replace(new RegExp(`<img\\b${TAG_BODY}>`, "gi"), (tag) => normalize(tag));
 }
 //# sourceMappingURL=imageSources.js.map

package/server/dist/feeds/poller.js

@@ -3,6 +3,7 @@ import { fetchFeed } from "./fetcher.js";
 import { discoverIconUrls, iconCheckDue } from "./icon.js";
 import { refreshFeedIcon } from "./feedIcon.js";
 import { decodeEntities, plainTitle } from "./text.js";
+import { unwrapRedirectLink } from "./redirectLinks.js";
 import { enqueueNewItemNotifications, startPushOutbox, stopPushOutbox } from "../pushOutbox.js";
 import { hasUserCapability } from "../auth/capabilities.js";
 import { findFeedById, listSubscriberIds, updateFeedAfterPoll, updateFeedIcon, upsertItems } from "./repository.js";
@@ -123,8 +124,8 @@ export function itemGuid(item) {
 /** The article address: its link, or its guid when the feed gives only that and the guid is a web address. */
 export function itemLink(item) {
     if (item.link)
-        return item.link;
-    return item.guid && /^https?:\/\//i.test(item.guid) ? item.guid : undefined;
+        return unwrapRedirectLink(item.link);
+    return item.guid && /^https?:\/\//i.test(item.guid) ? unwrapRedirectLink(item.guid) : undefined;
 }
 /** One attempt; the queue owns retries, admission, single-flight and the total deadline. */
 async function pollAttempt(feed, signal) {

package/server/dist/feeds/redirectLinks.js

@@ -0,0 +1,22 @@
+/**
+ * Some feeds wrap every article address in a redirect through the publisher's tracker. Google Alerts links through
+ * `https://www.google.com/url?...&url=<article>`, which is not the page to read and fails the full-text fetch. The article address
+ * inside the wrapper is what an item should point to.
+ */
+const GOOGLE_HOST = /^(?:www\.)?google\.[a-z]{2,3}(?:\.[a-z]{2})?$/i;
+export function unwrapRedirectLink(link) {
+    if (!link)
+        return link;
+    let url;
+    try {
+        url = new URL(link);
+    }
+    catch {
+        return link;
+    }
+    if (!GOOGLE_HOST.test(url.hostname) || url.pathname !== "/url")
+        return link;
+    const target = url.searchParams.get("url") ?? url.searchParams.get("q");
+    return target && /^https?:\/\//i.test(target) ? target : link;
+}
+//# sourceMappingURL=redirectLinks.js.map

package/server/dist/feeds/repository.js

@@ -3,7 +3,8 @@ import { articleText } from "./articleContent.js";
 import { LEAD_VERSION } from "./contentNoise.js";
 import { largerHeroImage } from "./heroImage.js";
 import { refreshGalleryClassification, refreshLead } from "./lead.js";
-import { decodeEntities } from "./text.js";
+import { decodeEntities, plainTitle } from "./text.js";
+import { unwrapRedirectLink } from "./redirectLinks.js";
 export function findFeedByUrl(url) {
     return db.prepare("SELECT * FROM feeds WHERE url = ?").get(url);
 }
@@ -570,19 +571,24 @@ export function moveSubscription(userId, fromFeedId, toFeedId) {
 export function updateFeedIcon(feedId, iconUrl) {
     db.prepare("UPDATE feeds SET icon_url = ?, icon_checked_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE id = ?").run(iconUrl, feedId);
 }
-/** Fix item titles, snippets, feed titles and labels stored before entity decoding existed; cheap once nothing matches. */
+/**
+ * Fix item titles, snippets, links, feed titles and labels stored before entity decoding, title markup removal and redirect unwrapping
+ * existed; cheap once nothing matches.
+ */
 export function repairEncodedText() {
     const rows = db
-        .prepare("SELECT id, title, content_snippet FROM items WHERE title LIKE '%&%;%' OR content_snippet LIKE '%&%;%'")
+        .prepare(`SELECT id, title, content_snippet, link FROM items
+       WHERE title LIKE '%&%;%' OR title LIKE '%<%>%' OR content_snippet LIKE '%&%;%' OR link LIKE 'https://www.google.%/url?%'`)
         .all();
-    const update = db.prepare("UPDATE items SET title = ?, content_snippet = ? WHERE id = ?");
+    const update = db.prepare("UPDATE items SET title = ?, content_snippet = ?, link = ? WHERE id = ?");
     let fixed = 0;
     db.transaction(() => {
         for (const row of rows) {
-            const title = decodeEntities(row.title);
+            const title = row.title === null ? null : (plainTitle(row.title) ?? row.title);
             const snippet = decodeEntities(row.content_snippet);
-            if (title !== row.title || snippet !== row.content_snippet) {
-                update.run(title ?? null, snippet ?? null, row.id);
+            const link = unwrapRedirectLink(row.link);
+            if (title !== row.title || snippet !== row.content_snippet || link !== row.link) {
+                update.run(title, snippet ?? null, link ?? null, row.id);
                 refreshLead(row.id);
                 fixed++;
             }

package/server/dist/feeds/text.js

@@ -15,9 +15,14 @@ function decodeOnce(text) {
         return NAMED_ENTITIES[name] ?? match;
     });
 }
-/** A title as one line of text: entities decoded, line breaks and runs of spaces collapsed to one space, trimmed. */
+/**
+ * Inline formatting that some feeds put into a title, such as the `<b>` Google Alerts wraps around the search term. Only these tags are
+ * removed, so a title that really contains an angle bracket stays as it is.
+ */
+const TITLE_MARKUP = /<\/?(?:b|i|u|em|strong|mark|span|small|sub|sup|font|a|br)\b[^>]*>/gi;
+/** A title as one line of text: entities decoded, inline markup removed, line breaks and runs of spaces collapsed to one space, trimmed. */
 export function plainTitle(text) {
-    const cleaned = decodeEntities(text)?.replace(/\s+/g, " ").trim();
+    const cleaned = decodeEntities(text)?.replace(TITLE_MARKUP, tag => (/^<br/i.test(tag) ? " " : "")).replace(/\s+/g, " ").trim();
     return cleaned || undefined;
 }
 export function decodeEntities(text) {

package/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@feedkeeper/server",
-  "version": "0.35.5",
+  "version": "0.35.7",
   "private": true,
   "type": "module"
 }