@rmdes/indiekit-endpoint-rss 1.1.3 → 1.2.0

package/index.js

@@ -74,6 +74,24 @@ export default class RssEndpoint {
     // Clear items and re-sync
     router.post("/clear-resync", dashboardController.clearResync);
 
+    // Dashboard forms: plain POST and redirect, so results land in
+    // Indiekit's own notification banner rather than a browser alert.
+    router.post(
+      "/feeds/:id/settings",
+      express.urlencoded({ extended: false }),
+      dashboardController.saveFeedSettings,
+    );
+    router.post(
+      "/feeds/:id/backfill",
+      express.urlencoded({ extended: false }),
+      dashboardController.backfillFeed,
+    );
+    router.post(
+      "/feeds/:id/republish",
+      express.urlencoded({ extended: false }),
+      dashboardController.republishFeed,
+    );
+
     // Feed management (protected - requires auth)
     router.post("/api/feeds", express.json(), feedsController.add);
     router.delete("/api/feeds/:id", feedsController.remove);

package/lib/controllers/dashboard.js

@@ -1,3 +1,8 @@
+import {
+  applyFeedSettings,
+  resetPublishState,
+  rewindWatermark,
+} from "../feed-settings.js";
 import { getSyncState, runSync } from "../sync.js";
 import { formatFeed, formatItem } from "../utils.js";
 
@@ -17,6 +22,32 @@ function consumeFlashMessage(request) {
   return result;
 }
 
+
+/**
+ * Publish context for a request, or null when the site is not configured.
+ * @param {object} request - Request
+ * @returns {object|null} Micropub endpoint and publication URL
+ */
+function publishContextFrom(request) {
+  const { application, publication } = request.app.locals;
+
+  return application?.micropubEndpoint && publication?.me
+    ? { micropubEndpoint: application.micropubEndpoint, me: publication.me }
+    : null;
+}
+
+/**
+ * Record a flash message and return to the dashboard.
+ * @param {object} request - Request
+ * @param {object} response - Response
+ * @param {string} type - success, error or warning
+ * @param {string} content - Message
+ */
+function flashBack(request, response, type, content) {
+  request.session.messages = [{ type, content }];
+  response.redirect(request.baseUrl);
+}
+
 export const dashboardController = {
   /**
    * Render admin dashboard
@@ -81,6 +112,151 @@ export const dashboardController = {
     }
   },
 
+
+  /**
+   * Save a feed's settings from the dashboard form
+   * POST /feeds/:id/settings
+   */
+  async saveFeedSettings(request, response) {
+    try {
+      const db = request.app.locals.application.getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const body = {
+        url: request.body.url,
+        publish: {
+          // An unchecked checkbox is omitted from a form post entirely, so
+          // presence is the signal.
+          enabled: request.body.enabled !== undefined,
+          postType: request.body.postType,
+          content: request.body.content,
+          linkProperty: request.body.linkProperty || null,
+          status: request.body.status,
+          dateSource: request.body.dateSource,
+        },
+      };
+
+      const result = await applyFeedSettings(
+        db,
+        request.params.id,
+        body,
+        request.app.locals.publication,
+      );
+
+      if (result.error) {
+        return flashBack(request, response, "error", result.error);
+      }
+
+      // Saying "nothing is waiting" beats silence: a feed can be correctly
+      // configured and still publish nothing, because the watermark only
+      // admits items dated after publishing was switched on.
+      const message =
+        result.eligible === 0
+          ? "Settings saved. No items are waiting — only items published after you enabled this will post. Use Publish existing items to include the ones already cached."
+          : result.eligible > 0
+            ? `Settings saved. ${result.eligible} item(s) waiting to publish.`
+            : "Settings saved.";
+
+      flashBack(request, response, "success", message);
+    } catch (error) {
+      console.error("[RSS] Error saving feed settings:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
+  /**
+   * Rewind a feed's watermark and publish straight away
+   * POST /feeds/:id/backfill
+   */
+  async backfillFeed(request, response) {
+    try {
+      const { rssConfig, getRssDb } = request.app.locals.application;
+      const db = getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const result = await rewindWatermark(db, request.params.id, {
+        last: Number(request.body.last),
+      });
+
+      if (result.error) {
+        return flashBack(request, response, "error", result.error);
+      }
+
+      if (result.eligible === 0) {
+        return flashBack(
+          request,
+          response,
+          "warning",
+          "Nothing to publish: those items are already published or were skipped after repeated failures.",
+        );
+      }
+
+      // Run the sync now rather than leaving the user to wait a cycle and
+      // guess. This is the only way the answer can be "3 published" instead
+      // of "queued, come back later".
+      const sync = await runSync(db, rssConfig, publishContextFrom(request));
+
+      if (sync.error) {
+        return flashBack(request, response, "error", sync.error);
+      }
+
+      const published = sync.itemsPublished || 0;
+      flashBack(
+        request,
+        response,
+        published > 0 ? "success" : "warning",
+        published > 0
+          ? `Published ${published} item(s) from the cache.`
+          : `${result.eligible} item(s) became eligible but none published — check the logs for the reason.`,
+      );
+    } catch (error) {
+      console.error("[RSS] Error running backfill:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
+
+  /**
+   * Forget what a feed has published, then publish it again
+   * POST /feeds/:id/republish
+   */
+  async republishFeed(request, response) {
+    try {
+      const { rssConfig, getRssDb } = request.app.locals.application;
+      const db = getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const reset = await resetPublishState(db, request.params.id);
+      if (reset.error) {
+        return flashBack(request, response, "error", reset.error);
+      }
+
+      const sync = await runSync(db, rssConfig, publishContextFrom(request));
+      if (sync.error) {
+        return flashBack(request, response, "error", sync.error);
+      }
+
+      const published = sync.itemsPublished || 0;
+      flashBack(
+        request,
+        response,
+        published > 0 ? "success" : "warning",
+        published > 0
+          ? `Cleared ${reset.reset} item(s) and republished ${published}.`
+          : `Cleared ${reset.reset} item(s), but none published — check the watermark and the logs.`,
+      );
+    } catch (error) {
+      console.error("[RSS] Error republishing feed:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
   /**
    * Clear all items and re-sync
    * POST /clear-resync

package/lib/controllers/feeds.js

@@ -1,8 +1,7 @@
 import { ObjectId } from "mongodb";
 import { RssClient } from "../rss-client.js";
-import { defaultContent, defaultLinkProperty } from "../jf2-builder.js";
+import { applyFeedSettings, rewindWatermark } from "../feed-settings.js";
 import { formatFeed, isValidUrl, normalizeUrl } from "../utils.js";
-import { pendingQuery, watermarkFor } from "../publisher.js";
 
 export const feedsController = {
   /**
@@ -149,138 +148,31 @@ export const feedsController = {
   },
 
   /**
-   * Update a feed: enable/disable, or change its publish configuration
+   * Update a feed: enable/disable, change its url, or its publish config
    * PATCH /api/feeds/:id
-   * Body: { enabled?: boolean, publish?: object }
    */
   async toggle(request, response) {
     try {
-      const { id } = request.params;
-      const { enabled, publish, url } = request.body;
-
-      if (!ObjectId.isValid(id)) {
-        return response.status(400).json({ error: "Invalid feed ID" });
-      }
-
-      if (enabled === undefined && publish === undefined && url === undefined) {
-        return response.status(400).json({ error: "Nothing to update" });
-      }
-
-      if (enabled !== undefined && typeof enabled !== "boolean") {
-        return response.status(400).json({ error: "enabled must be boolean" });
-      }
-
       const db = request.app.locals.application.getRssDb?.();
       if (!db) {
         return response.status(500).json({ error: "Database not available" });
       }
 
-      const feedsCollection = db.collection("rssFeeds");
-      const feedId = new ObjectId(id);
-      const feed = await feedsCollection.findOne({ _id: feedId });
-
-      if (!feed) {
-        return response.status(404).json({
-          error: response.locals.__("rss.error.feedNotFound"),
-        });
-      }
-
-      const update = {};
-
-      if (enabled !== undefined) {
-        update.enabled = enabled;
-      }
-
-      if (url !== undefined) {
-        if (!isValidUrl(url)) {
-          return response.status(400).json({
-            error: response.locals.__("rss.error.invalidUrl"),
-          });
-        }
-
-        const normalizedUrl = normalizeUrl(url);
-        const clash = await feedsCollection.findOne({
-          url: normalizedUrl,
-          _id: { $ne: feedId },
-        });
-
-        if (clash) {
-          return response.status(409).json({
-            error: response.locals.__("rss.error.feedExists"),
-          });
-        }
-
-        update.url = normalizedUrl;
-        // Items key on feedId, so they survive the change. Any error from the
-        // old URL is stale the moment it is corrected.
-        update.lastError = null;
-      }
-
-      if (publish !== undefined) {
-        const { postTypes } = request.app.locals.publication || {};
-
-        if (publish.enabled) {
-          if (!postTypes) {
-            return response.status(500).json({
-              error: "Post type configuration unavailable",
-            });
-          }
-
-          // postData.create() throws notImplemented for an unconfigured type,
-          // and the failure would be buried deep in the sync loop. Sites
-          // differ: chardonsbleus has audio, event, jam and rsvp disabled.
-          if (!Object.hasOwn(postTypes, publish.postType)) {
-            return response.status(400).json({
-              error: `Post type "${publish.postType}" is not enabled on this site`,
-            });
-          }
-        }
-
-        const merged = { ...feed.publish, ...publish };
-
-        update.publish = {
-          ...merged,
-          // Pre-fill from the post type when the caller left them out, so a
-          // minimal PATCH of { enabled, postType } yields a working config.
-          // An explicit null is respected: it means "no link property".
-          content: merged.content || defaultContent(merged.postType),
-          linkProperty:
-            publish.linkProperty === undefined
-              ? (merged.linkProperty ?? defaultLinkProperty(merged.postType))
-              : publish.linkProperty,
-          // Stamp the watermark the first time publishing is switched on, so
-          // enabling a feed does not publish its entire cached backlog.
-          since:
-            publish.enabled && !feed.publish?.since
-              ? new Date().toISOString()
-              : (feed.publish?.since ?? null),
-        };
-      }
-
-      const result = await feedsCollection.findOneAndUpdate(
-        { _id: feedId },
-        { $set: update },
-        { returnDocument: "after" },
+      const result = await applyFeedSettings(
+        db,
+        request.params.id,
+        request.body,
+        request.app.locals.publication,
       );
 
-      // Enabling publishing stamps a watermark, so a feed can be correctly
-      // configured and still have nothing to publish. Saying so beats silence.
-      let eligible;
-      if (result.publish?.enabled) {
-        eligible = await db
-          .collection("rssItems")
-          .countDocuments(pendingQuery(result));
+      if (result.error) {
+        return response.status(result.status).json({ error: result.error });
       }
 
       response.json({
-        eligible,
-        message:
-          enabled === undefined
-            ? response.locals.__("rss.success.feedUpdated")
-            : enabled
-              ? response.locals.__("rss.success.feedEnabled")
-              : response.locals.__("rss.success.feedDisabled"),
-        feed: formatFeed(result),
+        eligible: result.eligible,
+        message: response.locals.__("rss.success.feedUpdated"),
+        feed: formatFeed(result.feed),
       });
     } catch (error) {
       console.error("[RSS] Error updating feed:", error.message);
@@ -291,69 +183,24 @@ export const feedsController = {
   /**
    * Rewind a feed's watermark so past items get published
    * POST /api/feeds/:id/backfill
-   * Body: { since?: string } or { last?: number }
    */
   async backfill(request, response) {
     try {
-      const { id } = request.params;
-      const { since, last } = request.body;
-
-      if (!ObjectId.isValid(id)) {
-        return response.status(400).json({ error: "Invalid feed ID" });
-      }
-
       const db = request.app.locals.application.getRssDb?.();
       if (!db) {
         return response.status(500).json({ error: "Database not available" });
       }
 
-      const feedsCollection = db.collection("rssFeeds");
-      const itemsCollection = db.collection("rssItems");
-      const feedId = new ObjectId(id);
-      const feed = await feedsCollection.findOne({ _id: feedId });
-
-      if (!feed) {
-        return response.status(404).json({
-          error: response.locals.__("rss.error.feedNotFound"),
-        });
-      }
-
-      if (!feed.publish?.enabled) {
-        return response.status(400).json({
-          error: "Enable publishing on this feed first",
-        });
-      }
+      const result = await rewindWatermark(db, request.params.id, request.body);
 
-      let oldestPubDate = null;
-      if (last) {
-        const items = await itemsCollection
-          .find({ feedId })
-          .sort({ pubDate: -1 })
-          .limit(Number(last))
-          .toArray();
-        oldestPubDate = items.at(-1)?.pubDate || null;
+      if (result.error) {
+        return response.status(result.status).json({ error: result.error });
       }
 
-      let watermark;
-      try {
-        watermark = watermarkFor({ since, last }, oldestPubDate);
-      } catch (error) {
-        return response.status(400).json({ error: error.message });
-      }
-
-      await feedsCollection.updateOne(
-        { _id: feedId },
-        {
-          // The feed's own post status is respected: silently forcing drafts
-          // would override a deliberate choice. The guard here is the count
-          // the user asked for, and maxPostsPerCycle pacing the catch-up.
-          $set: { "publish.since": watermark },
-        },
-      );
-
       response.json({
+        eligible: result.eligible,
         message: response.locals.__("rss.success.backfillQueued"),
-        since: watermark,
+        since: result.since,
       });
     } catch (error) {
       console.error("[RSS] Error queueing backfill:", error.message);

package/lib/feed-settings.js

@@ -0,0 +1,214 @@
+import { ObjectId } from "mongodb";
+
+import { defaultContent, defaultLinkProperty } from "./jf2-builder.js";
+import { pendingQuery, watermarkFor } from "./publisher.js";
+import { isValidUrl, normalizeUrl } from "./utils.js";
+
+/**
+ * Apply a settings change to one feed.
+ *
+ * Shared by the JSON API and the dashboard form so the two cannot drift: two
+ * copies of this validation would eventually disagree, and the symptom would
+ * be a form that accepts what the API rejects.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @param {object} body - Fields to change: enabled, url, publish
+ * @param {object} [publication] - Publication configuration
+ * @returns {Promise<object>} Result carrying status, error, feed, eligible
+ */
+export async function applyFeedSettings(db, id, body, publication = {}) {
+  const { enabled, publish, url } = body;
+
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  if (enabled === undefined && publish === undefined && url === undefined) {
+    return { status: 400, error: "Nothing to update" };
+  }
+
+  if (enabled !== undefined && typeof enabled !== "boolean") {
+    return { status: 400, error: "enabled must be boolean" };
+  }
+
+  const feedsCollection = db.collection("rssFeeds");
+  const feedId = new ObjectId(id);
+  const feed = await feedsCollection.findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  const update = {};
+
+  if (enabled !== undefined) {
+    update.enabled = enabled;
+  }
+
+  if (url !== undefined) {
+    if (!isValidUrl(url)) {
+      return { status: 400, error: "Invalid feed URL" };
+    }
+
+    const normalizedUrl = normalizeUrl(url);
+    const clash = await feedsCollection.findOne({
+      url: normalizedUrl,
+      _id: { $ne: feedId },
+    });
+
+    if (clash) {
+      return { status: 409, error: "Another feed already uses that URL" };
+    }
+
+    update.url = normalizedUrl;
+    // Items key on feedId, so they survive the change. Any error from the old
+    // URL is stale the moment it is corrected.
+    update.lastError = null;
+  }
+
+  if (publish !== undefined) {
+    const { postTypes } = publication;
+
+    if (publish.enabled) {
+      if (!postTypes) {
+        return { status: 500, error: "Post type configuration unavailable" };
+      }
+
+      // postData.create() throws notImplemented for an unconfigured type, and
+      // the failure would be buried deep in the sync loop. Sites differ:
+      // chardonsbleus has audio, event, jam and rsvp disabled.
+      if (!Object.hasOwn(postTypes, publish.postType)) {
+        return {
+          status: 400,
+          error: `Post type "${publish.postType}" is not enabled on this site`,
+        };
+      }
+    }
+
+    const merged = { ...feed.publish, ...publish };
+
+    update.publish = {
+      ...merged,
+      // Pre-fill from the post type when the caller left them out, so a
+      // minimal update of enabled plus postType yields a working config. An
+      // explicit null is respected: it means "no link property".
+      content: merged.content || defaultContent(merged.postType),
+      linkProperty:
+        publish.linkProperty === undefined
+          ? (merged.linkProperty ?? defaultLinkProperty(merged.postType))
+          : publish.linkProperty,
+      // Stamp the watermark the first time publishing is switched on, so
+      // enabling a feed does not publish its entire cached backlog.
+      since:
+        publish.enabled && !feed.publish?.since
+          ? new Date().toISOString()
+          : (feed.publish?.since ?? null),
+    };
+  }
+
+  const result = await feedsCollection.findOneAndUpdate(
+    { _id: feedId },
+    { $set: update },
+    { returnDocument: "after" },
+  );
+
+  // A feed can be correctly configured and still have nothing to publish, so
+  // report the count rather than leaving the user to guess.
+  let eligible;
+  if (result.publish?.enabled) {
+    eligible = await db
+      .collection("rssItems")
+      .countDocuments(pendingQuery(result));
+  }
+
+  return { status: 200, feed: result, eligible };
+}
+
+/**
+ * Move a feed's watermark back so already-cached items become eligible.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @param {object} body - Either a since date or a last item count
+ * @returns {Promise<object>} Result carrying status, error, since, eligible
+ */
+export async function rewindWatermark(db, id, body) {
+  const { since, last } = body;
+
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  const feedsCollection = db.collection("rssFeeds");
+  const itemsCollection = db.collection("rssItems");
+  const feedId = new ObjectId(id);
+  const feed = await feedsCollection.findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  if (!feed.publish?.enabled) {
+    return { status: 400, error: "Enable publishing on this feed first" };
+  }
+
+  let oldestPubDate = null;
+  if (last) {
+    const items = await itemsCollection
+      .find({ feedId })
+      .sort({ pubDate: -1 })
+      .limit(Number(last))
+      .toArray();
+    oldestPubDate = items.at(-1)?.pubDate || null;
+  }
+
+  let watermark;
+  try {
+    watermark = watermarkFor({ since, last }, oldestPubDate);
+  } catch (error) {
+    return { status: 400, error: error.message };
+  }
+
+  // The feed's own post status is respected: silently forcing drafts would
+  // override a deliberate choice. The guard is the count the user asked for,
+  // and maxPostsPerCycle pacing the catch-up.
+  const updated = await feedsCollection.findOneAndUpdate(
+    { _id: feedId },
+    { $set: { "publish.since": watermark } },
+    { returnDocument: "after" },
+  );
+
+  const eligible = await itemsCollection.countDocuments(pendingQuery(updated));
+
+  return { status: 200, since: watermark, eligible };
+}
+
+/**
+ * Clear a feed's publish state so its items can be published again.
+ *
+ * Without this a feed is stuck the moment its items carry postedAt: changing
+ * the template, the post type or the date source has no way to reach items
+ * already sent. Existing posts are not deleted — republishing writes over the
+ * same URL, because the slug is derived from the item's guid.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @returns {Promise<object>} Result carrying status, error, reset
+ */
+export async function resetPublishState(db, id) {
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  const feedId = new ObjectId(id);
+  const feed = await db.collection("rssFeeds").findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  const result = await db.collection("rssItems").updateMany(
+    { feedId },
+    { $unset: { postedAt: "", postUrl: "", postSkipped: "", postError: "", postErrorAt: "", postAttempts: "" } },
+  );
+
+  return { status: 200, reset: result.modifiedCount };
+}

package/lib/jf2-builder.js

@@ -1,3 +1,5 @@
+import { createHash } from "node:crypto";
+
 import { sanitizeHtml, stripHtml } from "./utils.js";
 
 /**
@@ -14,14 +16,44 @@ const LINK_PROPERTY_DEFAULTS = {
 };
 
 const PLACEHOLDERS = {
-  title: (item) => stripHtml(item.title),
+  title: (item) => oneLine(stripHtml(item.title)),
   link: (item) => item.link,
-  description: (item) => stripHtml(item.description),
-  content: (item) => sanitizeHtml(item.content),
-  author: (item) => stripHtml(item.author),
-  sourceTitle: (item) => stripHtml(item.sourceTitle),
+  description: (item) => oneLine(stripHtml(item.description)),
+  // The item's own link is the right base: a feed's markup is relative to
+  // the site that published it, not to ours.
+  content: (item) => sanitizeHtml(item.content, { baseUrl: item.link }),
+  author: (item) => oneLine(stripHtml(item.author)),
+  sourceTitle: (item) => oneLine(stripHtml(item.sourceTitle)),
 };
 
+
+/**
+ * Collapse whitespace runs so an excerpt stays one line.
+ *
+ * Feed descriptions often arrive indented. Markdown reads an indented line as
+ * a code block, which is how a GitHub timeline entry rendered as <pre><code>.
+ * @param {string} value - Text
+ * @returns {string} Single-line text
+ */
+const oneLine = (value) => (value ? String(value).replace(/\s+/g, " ").trim() : "");
+
+/**
+ * Short, stable discriminator for an item.
+ *
+ * Indiekit derives a slug from the first five words of the name, so feed items
+ * sharing a title prefix produce the same URL — and postData.create replaces
+ * on that URL, so the second item silently overwrites the first. Three GitHub
+ * timeline entries collapsed into one post that way. The guid is already the
+ * deduplication key, so it is the right thing to discriminate on.
+ * @param {object} item - RSS item document
+ * @returns {string} Six hex characters
+ */
+const discriminator = (item) =>
+  createHash("sha1")
+    .update(String(item.guid ?? item.link ?? item.title ?? ""))
+    .digest("hex")
+    .slice(0, 6);
+
 /**
  * Link property offered when a post type is chosen
  * @param {string} postType - Post type key
@@ -73,7 +105,7 @@ export function buildJf2(item, publish) {
   // A `name` next to content makes discovery return "article" — see
   // post-type-discovery.js: `if (content && properties.name) return "article"`.
   // A note must therefore carry none.
-  const name = stripHtml(item.title);
+  const name = oneLine(stripHtml(item.title));
   if (publish.postType !== "note" && name) {
     properties.name = name;
   }
@@ -91,7 +123,14 @@ export function buildJf2(item, publish) {
     properties[publish.linkProperty] = value;
   }
 
-  if (item.pubDate) {
+  // dateSource decides whether the post carries the item's own date or the
+  // moment it was published here. The item's date is truthful — the article
+  // really was written then — but it buries the post at its chronological
+  // place, so a feed of older items produces posts nobody sees. "created"
+  // trades that accuracy for visibility.
+  if (publish.dateSource === "created") {
+    properties.published = new Date().toISOString();
+  } else if (item.pubDate) {
     properties.published =
       item.pubDate instanceof Date ? item.pubDate.toISOString() : item.pubDate;
   }
@@ -104,6 +143,13 @@ export function buildJf2(item, publish) {
     properties.syndication = [item.link];
   }
 
+  // Five title words plus a guid-derived suffix: readable, and impossible for
+  // two distinct items to collide on.
+  properties["mp-slug"] = [
+    ...name.split(/\s+/).filter(Boolean).slice(0, 5),
+    discriminator(item),
+  ].join(" ");
+
   properties["post-status"] =
     publish.status === "published" ? "published" : "draft";
 

package/lib/publisher.js

@@ -119,12 +119,21 @@ export function pendingQuery(feed) {
   };
 
   if (feed.publish?.since) {
-    // An item with no parsable date would be excluded forever by a bare
-    // `pubDate: { $gt: since }`, because null sorts below every Date in BSON.
-    // fetchedAt is always set on insert and stands in when pubDate is missing.
+    const since = feed.publish.since;
+
+    // One watermark, two operand types, because the two fields are stored
+    // differently: pubDate is a BSON Date (rss-client builds real Dates and
+    // pruneOldItems compares it against one), while fetchedAt — like every
+    // other date here — is an ISO string. MongoDB orders by BSON type before
+    // value, so a Date is never $gt a String: passing the ISO string to
+    // pubDate matches nothing at all, with no error to show for it.
+    //
+    // An item with no parsable date would also be excluded forever by a bare
+    // comparison, because null sorts below every Date. fetchedAt is always set
+    // on insert and stands in when pubDate is missing.
     query.$or = [
-      { pubDate: { $gt: feed.publish.since } },
-      { pubDate: null, fetchedAt: { $gt: feed.publish.since } },
+      { pubDate: { $gt: new Date(since) } },
+      { pubDate: null, fetchedAt: { $gt: since } },
     ];
   }
 

package/lib/utils.js

@@ -14,13 +14,34 @@ function toISO(value) {
 }
 
 /**
- * Sanitize HTML content - strip dangerous tags, keep basic formatting
+ * Sanitize HTML content, keeping as much of it as is safe to keep.
+ *
+ * Feeds carry real formatting — headings, images, links, code — and dropping
+ * it produces a flat paraphrase of the source. The job here is to remove what
+ * is dangerous, not what is rich.
  * @param {string} html - Raw HTML
- * @returns {string}
+ * @param {object} [options] - Options
+ * @param {string} [options.baseUrl] - Resolves relative hrefs and image srcs
+ * @returns {string} Sanitized HTML
  */
-export function sanitizeHtml(html) {
+export function sanitizeHtml(html, { baseUrl } = {}) {
   if (!html) return "";
-  return sanitizeHtmlLib(html, {
+
+  /**
+   * Make a feed-relative URL absolute against its source.
+   * @param {string} value - href or src
+   * @returns {string} Absolute URL where possible
+   */
+  const absolute = (value) => {
+    if (!value || !baseUrl) return value;
+    try {
+      return new URL(value, baseUrl).href;
+    } catch {
+      return value;
+    }
+  };
+
+  const clean = sanitizeHtmlLib(html, {
     allowedTags: [
       "p",
       "br",
@@ -35,9 +56,17 @@ export function sanitizeHtml(html) {
       "blockquote",
       "code",
       "pre",
+      "img",
+      "figure",
+      "figcaption",
+      "h2",
+      "h3",
+      "h4",
+      "hr",
     ],
     allowedAttributes: {
       a: ["href", "title", "rel"],
+      img: ["src", "alt", "title", "width", "height", "loading"],
       code: ["class"],
       pre: ["class"],
     },
@@ -46,7 +75,47 @@ export function sanitizeHtml(html) {
       pre: ["language-*"],
     },
     allowedSchemes: ["http", "https", "mailto"],
+    transformTags: {
+      // A feed's markup is relative to the feed's own site. Left as-is these
+      // resolve against the publishing site instead, so every link and image
+      // silently points at the wrong host.
+      a: (tagName, attribs) => ({
+        tagName,
+        attribs: { ...attribs, ...(attribs.href && { href: absolute(attribs.href) }) },
+      }),
+      img: (tagName, attribs) => ({
+        tagName,
+        attribs: {
+          ...attribs,
+          ...(attribs.src && { src: absolute(attribs.src) }),
+          loading: "lazy",
+        },
+      }),
+    },
+    // An anchor whose only child was a stripped element leaves an empty link.
+    exclusiveFilter: (frame) =>
+      frame.tag === "a" && !frame.text.trim() && !frame.mediaChildren?.length,
   });
+
+  return collapseIndentation(clean).trim();
+}
+
+/**
+ * Collapse whitespace runs outside <pre> blocks.
+ *
+ * Micropub turns HTML into the markdown that gets stored, normalising through
+ * markdown-it first — where a line starting with four spaces is an indented
+ * code block. Feed markup is usually pretty-printed, so whole paragraphs came
+ * back wrapped in a fence with their tags escaped. Inside <pre> the
+ * indentation is the content, so it is left alone.
+ * @param {string} html - Sanitized HTML
+ * @returns {string} HTML without cosmetic indentation
+ */
+function collapseIndentation(html) {
+  return html
+    .split(/(<pre[\s\S]*?<\/pre>)/i)
+    .map((part, index) => (index % 2 ? part : part.replace(/\s+/g, " ")))
+    .join("");
 }
 
 /**

package/locales/en.json

@@ -42,7 +42,7 @@
       "feedUpdated": "Feed updated",
       "feedEnabled": "Feed enabled",
       "feedDisabled": "Feed disabled",
-      "backfillQueued": "Backfill queued; items will be published as drafts over the next sync cycles",
+      "backfillQueued": "Backfill queued: the newest cached items will be published over the next sync cycles, using this feed's post status.",
       "syncComplete": "Sync complete",
       "clearResync": "Items cleared and re-synced"
     },
@@ -62,7 +62,11 @@
       "status": "Post status",
       "draft": "Draft",
       "published": "Published",
-      "save": "Save settings"
+      "save": "Save settings",
+      "dateSource": "Post date",
+      "dateFromItem": "The item's own publication date",
+      "dateFromCreation": "When it was published here",
+      "dateSourceHint": "The item's own date is accurate but sorts the post into the past, where a feed of older items produces posts nobody sees."
     },
     "settings": {
       "title": "Feed settings",
@@ -73,6 +77,10 @@
       "label": "Publish existing items",
       "hint": "How many of the newest cached items to publish. They are created with this feed's post status, a few per sync cycle.",
       "submit": "Publish existing items"
+    },
+    "republish": {
+      "submit": "Forget and republish",
+      "hint": "Clears what this feed has published so the current settings apply to every item again. Existing posts are overwritten, not duplicated."
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-rss",
-  "version": "1.1.3",
+  "version": "1.2.0",
   "description": "RSS feed reader endpoint for Indiekit. Aggregates multiple feeds, caches in MongoDB, displays on frontend.",
   "keywords": [
     "indiekit",

package/views/rss.njk

@@ -113,7 +113,7 @@
               classes: "rss-publish",
               summary: publishSummary
             }) %}
-              <form data-publish-feed="{{ feed.id }}">
+              <form action="{{ mountPath }}/feeds/{{ feed.id }}/settings" method="post">
                 {{ input({
                   id: "feed-url-" + feed.id,
                   name: "url",
@@ -175,6 +175,25 @@
                   ]
                 }) }}
 
+                {{ select({
+                  id: "publish-date-source-" + feed.id,
+                  name: "dateSource",
+                  label: __("rss.publish.dateSource"),
+                  hint: __("rss.publish.dateSourceHint"),
+                  items: [
+                    {
+                      text: __("rss.publish.dateFromItem"),
+                      value: "item",
+                      selected: feed.publish.dateSource != "created"
+                    },
+                    {
+                      text: __("rss.publish.dateFromCreation"),
+                      value: "created",
+                      selected: feed.publish.dateSource == "created"
+                    }
+                  ]
+                }) }}
+
                 {{ button({
                   type: "submit",
                   text: __("rss.publish.save")
@@ -182,7 +201,7 @@
               </form>
 
               {% if feed.publish.enabled %}
-              <form class="rss-backfill" data-backfill-feed="{{ feed.id }}">
+              <form class="rss-backfill" action="{{ mountPath }}/feeds/{{ feed.id }}/backfill" method="post">
                 {{ input({
                   id: "backfill-last-" + feed.id,
                   name: "last",
@@ -197,6 +216,15 @@
                   text: __("rss.backfill.submit")
                 }) }}
               </form>
+
+              <form class="rss-republish" action="{{ mountPath }}/feeds/{{ feed.id }}/republish" method="post">
+                <p class="hint">{{ __("rss.republish.hint") }}</p>
+                {{ button({
+                  classes: "button--secondary",
+                  type: "submit",
+                  text: __("rss.republish.submit")
+                }) }}
+              </form>
               {% endif %}
             {% endcall %}
           </div>
@@ -356,38 +384,6 @@
     }
   });
 
-  // Handle backfill
-  document.querySelectorAll('[data-backfill-feed]').forEach(form => {
-    form.addEventListener('submit', async (e) => {
-      e.preventDefault();
-      const feedId = e.target.dataset.backfillFeed;
-      const last = Number(new FormData(e.target).get('last'));
-
-      if (!last || last < 1) {
-        alert('Enter how many recent items to publish.');
-        return;
-      }
-
-      try {
-        const response = await fetch(`{{ mountPath }}/api/feeds/${feedId}/backfill`, {
-          method: 'POST',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ last })
-        });
-
-        const body = await response.json();
-        if (response.ok) {
-          alert(body.message || 'Backfill queued.');
-          location.reload();
-        } else {
-          alert(body.error || 'Failed to queue backfill');
-        }
-      } catch (err) {
-        alert('Failed to queue backfill: ' + err.message);
-      }
-    });
-  });
-
   // Handle delete feed
   document.querySelectorAll('[data-delete-feed]').forEach(btn => {
     btn.addEventListener('click', async (e) => {
@@ -412,51 +408,5 @@
     });
   });
 
-  // Handle publish configuration
-  document.querySelectorAll('[data-publish-feed]').forEach(form => {
-    form.addEventListener('submit', async (e) => {
-      e.preventDefault();
-      const feedId = e.target.dataset.publishFeed;
-      const data = new FormData(e.target);
-
-      const publish = {
-        // The checkboxes component sets value="true"; an unchecked box is
-        // omitted from FormData entirely, so presence is the signal.
-        enabled: data.get('enabled') !== null,
-        postType: data.get('postType'),
-        content: data.get('content'),
-        linkProperty: data.get('linkProperty') || null,
-        status: data.get('status'),
-      };
-
-      try {
-        const response = await fetch(`{{ mountPath }}/api/feeds/${feedId}`, {
-          method: 'PATCH',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ url: data.get('url'), publish })
-        });
-
-        const body = await response.json();
-
-        if (response.ok) {
-          // A feed can be correctly configured and still have nothing to
-          // publish: the watermark only lets through items dated after the
-          // moment publishing was switched on.
-          if (body.eligible === 0) {
-            alert(
-              'Saved. No items are waiting: only items published after you ' +
-              'enabled this will post. Use "Publish existing items" to ' +
-              'include the ones already cached.'
-            );
-          }
-          location.reload();
-        } else {
-          alert(body.error || 'Failed to save publish settings');
-        }
-      } catch (err) {
-        alert('Failed to save publish settings: ' + err.message);
-      }
-    });
-  });
 </script>
 {% endblock %}