@rmdes/indiekit-endpoint-rss 1.1.4 → 1.2.0

package/index.js

@@ -86,6 +86,11 @@ export default class RssEndpoint {
       express.urlencoded({ extended: false }),
       dashboardController.backfillFeed,
     );
+    router.post(
+      "/feeds/:id/republish",
+      express.urlencoded({ extended: false }),
+      dashboardController.republishFeed,
+    );
 
     // Feed management (protected - requires auth)
     router.post("/api/feeds", express.json(), feedsController.add);

package/lib/controllers/dashboard.js

@@ -1,4 +1,8 @@
-import { applyFeedSettings, rewindWatermark } from "../feed-settings.js";
+import {
+  applyFeedSettings,
+  resetPublishState,
+  rewindWatermark,
+} from "../feed-settings.js";
 import { getSyncState, runSync } from "../sync.js";
 import { formatFeed, formatItem } from "../utils.js";
 
@@ -130,6 +134,7 @@ export const dashboardController = {
           content: request.body.content,
           linkProperty: request.body.linkProperty || null,
           status: request.body.status,
+          dateSource: request.body.dateSource,
         },
       };
 
@@ -214,6 +219,44 @@ export const dashboardController = {
     }
   },
 
+
+  /**
+   * Forget what a feed has published, then publish it again
+   * POST /feeds/:id/republish
+   */
+  async republishFeed(request, response) {
+    try {
+      const { rssConfig, getRssDb } = request.app.locals.application;
+      const db = getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const reset = await resetPublishState(db, request.params.id);
+      if (reset.error) {
+        return flashBack(request, response, "error", reset.error);
+      }
+
+      const sync = await runSync(db, rssConfig, publishContextFrom(request));
+      if (sync.error) {
+        return flashBack(request, response, "error", sync.error);
+      }
+
+      const published = sync.itemsPublished || 0;
+      flashBack(
+        request,
+        response,
+        published > 0 ? "success" : "warning",
+        published > 0
+          ? `Cleared ${reset.reset} item(s) and republished ${published}.`
+          : `Cleared ${reset.reset} item(s), but none published — check the watermark and the logs.`,
+      );
+    } catch (error) {
+      console.error("[RSS] Error republishing feed:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
   /**
    * Clear all items and re-sync
    * POST /clear-resync

package/lib/feed-settings.js

@@ -181,3 +181,34 @@ export async function rewindWatermark(db, id, body) {
 
   return { status: 200, since: watermark, eligible };
 }
+
+/**
+ * Clear a feed's publish state so its items can be published again.
+ *
+ * Without this a feed is stuck the moment its items carry postedAt: changing
+ * the template, the post type or the date source has no way to reach items
+ * already sent. Existing posts are not deleted — republishing writes over the
+ * same URL, because the slug is derived from the item's guid.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @returns {Promise<object>} Result carrying status, error, reset
+ */
+export async function resetPublishState(db, id) {
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  const feedId = new ObjectId(id);
+  const feed = await db.collection("rssFeeds").findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  const result = await db.collection("rssItems").updateMany(
+    { feedId },
+    { $unset: { postedAt: "", postUrl: "", postSkipped: "", postError: "", postErrorAt: "", postAttempts: "" } },
+  );
+
+  return { status: 200, reset: result.modifiedCount };
+}

package/lib/jf2-builder.js

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { sanitizeHtml, stripHtml } from "./utils.js";
 
 /**
@@ -14,14 +16,44 @@ const LINK_PROPERTY_DEFAULTS = {
 };
 
 const PLACEHOLDERS = {
-  title: (item) => stripHtml(item.title),
+  title: (item) => oneLine(stripHtml(item.title)),
   link: (item) => item.link,
-  description: (item) => stripHtml(item.description),
-  content: (item) => sanitizeHtml(item.content),
-  author: (item) => stripHtml(item.author),
-  sourceTitle: (item) => stripHtml(item.sourceTitle),
+  description: (item) => oneLine(stripHtml(item.description)),
+  // The item's own link is the right base: a feed's markup is relative to
+  // the site that published it, not to ours.
+  content: (item) => sanitizeHtml(item.content, { baseUrl: item.link }),
+  author: (item) => oneLine(stripHtml(item.author)),
+  sourceTitle: (item) => oneLine(stripHtml(item.sourceTitle)),
 };
 
+
+/**
+ * Collapse whitespace runs so an excerpt stays one line.
+ *
+ * Feed descriptions often arrive indented. Markdown reads an indented line as
+ * a code block, which is how a GitHub timeline entry rendered as <pre><code>.
+ * @param {string} value - Text
+ * @returns {string} Single-line text
+ */
+const oneLine = (value) => (value ? String(value).replace(/\s+/g, " ").trim() : "");
+
+/**
+ * Short, stable discriminator for an item.
+ *
+ * Indiekit derives a slug from the first five words of the name, so feed items
+ * sharing a title prefix produce the same URL — and postData.create replaces
+ * on that URL, so the second item silently overwrites the first. Three GitHub
+ * timeline entries collapsed into one post that way. The guid is already the
+ * deduplication key, so it is the right thing to discriminate on.
+ * @param {object} item - RSS item document
+ * @returns {string} Six hex characters
+ */
+const discriminator = (item) =>
+  createHash("sha1")
+    .update(String(item.guid ?? item.link ?? item.title ?? ""))
+    .digest("hex")
+    .slice(0, 6);
+
 /**
  * Link property offered when a post type is chosen
  * @param {string} postType - Post type key
@@ -73,7 +105,7 @@ export function buildJf2(item, publish) {
   // A `name` next to content makes discovery return "article" — see
   // post-type-discovery.js: `if (content && properties.name) return "article"`.
   // A note must therefore carry none.
-  const name = stripHtml(item.title);
+  const name = oneLine(stripHtml(item.title));
   if (publish.postType !== "note" && name) {
     properties.name = name;
   }
@@ -91,7 +123,14 @@ export function buildJf2(item, publish) {
     properties[publish.linkProperty] = value;
   }
 
-  if (item.pubDate) {
+  // dateSource decides whether the post carries the item's own date or the
+  // moment it was published here. The item's date is truthful — the article
+  // really was written then — but it buries the post at its chronological
+  // place, so a feed of older items produces posts nobody sees. "created"
+  // trades that accuracy for visibility.
+  if (publish.dateSource === "created") {
+    properties.published = new Date().toISOString();
+  } else if (item.pubDate) {
     properties.published =
       item.pubDate instanceof Date ? item.pubDate.toISOString() : item.pubDate;
   }
@@ -104,6 +143,13 @@ export function buildJf2(item, publish) {
     properties.syndication = [item.link];
   }
 
+  // Five title words plus a guid-derived suffix: readable, and impossible for
+  // two distinct items to collide on.
+  properties["mp-slug"] = [
+    ...name.split(/\s+/).filter(Boolean).slice(0, 5),
+    discriminator(item),
+  ].join(" ");
+
   properties["post-status"] =
     publish.status === "published" ? "published" : "draft";
 

package/lib/utils.js

@@ -14,13 +14,34 @@ function toISO(value) {
 }
 
 /**
- * Sanitize HTML content - strip dangerous tags, keep basic formatting
+ * Sanitize HTML content, keeping as much of it as is safe to keep.
+ *
+ * Feeds carry real formatting — headings, images, links, code — and dropping
+ * it produces a flat paraphrase of the source. The job here is to remove what
+ * is dangerous, not what is rich.
  * @param {string} html - Raw HTML
- * @returns {string}
+ * @param {object} [options] - Options
+ * @param {string} [options.baseUrl] - Resolves relative hrefs and image srcs
+ * @returns {string} Sanitized HTML
  */
-export function sanitizeHtml(html) {
+export function sanitizeHtml(html, { baseUrl } = {}) {
   if (!html) return "";
-  return sanitizeHtmlLib(html, {
+
+  /**
+   * Make a feed-relative URL absolute against its source.
+   * @param {string} value - href or src
+   * @returns {string} Absolute URL where possible
+   */
+  const absolute = (value) => {
+    if (!value || !baseUrl) return value;
+    try {
+      return new URL(value, baseUrl).href;
+    } catch {
+      return value;
+    }
+  };
+
+  const clean = sanitizeHtmlLib(html, {
     allowedTags: [
       "p",
       "br",
@@ -35,9 +56,17 @@ export function sanitizeHtml(html) {
       "blockquote",
       "code",
       "pre",
+      "img",
+      "figure",
+      "figcaption",
+      "h2",
+      "h3",
+      "h4",
+      "hr",
     ],
     allowedAttributes: {
       a: ["href", "title", "rel"],
+      img: ["src", "alt", "title", "width", "height", "loading"],
       code: ["class"],
       pre: ["class"],
     },
@@ -46,7 +75,47 @@ export function sanitizeHtml(html) {
       pre: ["language-*"],
     },
     allowedSchemes: ["http", "https", "mailto"],
+    transformTags: {
+      // A feed's markup is relative to the feed's own site. Left as-is these
+      // resolve against the publishing site instead, so every link and image
+      // silently points at the wrong host.
+      a: (tagName, attribs) => ({
+        tagName,
+        attribs: { ...attribs, ...(attribs.href && { href: absolute(attribs.href) }) },
+      }),
+      img: (tagName, attribs) => ({
+        tagName,
+        attribs: {
+          ...attribs,
+          ...(attribs.src && { src: absolute(attribs.src) }),
+          loading: "lazy",
+        },
+      }),
+    },
+    // An anchor whose only child was a stripped element leaves an empty link.
+    exclusiveFilter: (frame) =>
+      frame.tag === "a" && !frame.text.trim() && !frame.mediaChildren?.length,
   });
+
+  return collapseIndentation(clean).trim();
+}
+
+/**
+ * Collapse whitespace runs outside <pre> blocks.
+ *
+ * Micropub turns HTML into the markdown that gets stored, normalising through
+ * markdown-it first — where a line starting with four spaces is an indented
+ * code block. Feed markup is usually pretty-printed, so whole paragraphs came
+ * back wrapped in a fence with their tags escaped. Inside <pre> the
+ * indentation is the content, so it is left alone.
+ * @param {string} html - Sanitized HTML
+ * @returns {string} HTML without cosmetic indentation
+ */
+function collapseIndentation(html) {
+  return html
+    .split(/(<pre[\s\S]*?<\/pre>)/i)
+    .map((part, index) => (index % 2 ? part : part.replace(/\s+/g, " ")))
+    .join("");
 }
 
 /**

package/locales/en.json

@@ -62,7 +62,11 @@
       "status": "Post status",
       "draft": "Draft",
       "published": "Published",
-      "save": "Save settings"
+      "save": "Save settings",
+      "dateSource": "Post date",
+      "dateFromItem": "The item's own publication date",
+      "dateFromCreation": "When it was published here",
+      "dateSourceHint": "The item's own date is accurate but sorts the post into the past, where a feed of older items produces posts nobody sees."
     },
     "settings": {
       "title": "Feed settings",
@@ -73,6 +77,10 @@
       "label": "Publish existing items",
       "hint": "How many of the newest cached items to publish. They are created with this feed's post status, a few per sync cycle.",
       "submit": "Publish existing items"
+    },
+    "republish": {
+      "submit": "Forget and republish",
+      "hint": "Clears what this feed has published so the current settings apply to every item again. Existing posts are overwritten, not duplicated."
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-rss",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "RSS feed reader endpoint for Indiekit. Aggregates multiple feeds, caches in MongoDB, displays on frontend.",
   "keywords": [
     "indiekit",

package/views/rss.njk

@@ -175,6 +175,25 @@
                   ]
                 }) }}
 
+                {{ select({
+                  id: "publish-date-source-" + feed.id,
+                  name: "dateSource",
+                  label: __("rss.publish.dateSource"),
+                  hint: __("rss.publish.dateSourceHint"),
+                  items: [
+                    {
+                      text: __("rss.publish.dateFromItem"),
+                      value: "item",
+                      selected: feed.publish.dateSource != "created"
+                    },
+                    {
+                      text: __("rss.publish.dateFromCreation"),
+                      value: "created",
+                      selected: feed.publish.dateSource == "created"
+                    }
+                  ]
+                }) }}
+
                 {{ button({
                   type: "submit",
                   text: __("rss.publish.save")
@@ -197,6 +216,15 @@
                   text: __("rss.backfill.submit")
                 }) }}
               </form>
+
+              <form class="rss-republish" action="{{ mountPath }}/feeds/{{ feed.id }}/republish" method="post">
+                <p class="hint">{{ __("rss.republish.hint") }}</p>
+                {{ button({
+                  classes: "button--secondary",
+                  type: "submit",
+                  text: __("rss.republish.submit")
+                }) }}
+              </form>
               {% endif %}
             {% endcall %}
           </div>