@rmdes/indiekit-endpoint-rss 1.1.2 → 1.1.4

package/index.js

@@ -74,6 +74,19 @@ export default class RssEndpoint {
     // Clear items and re-sync
     router.post("/clear-resync", dashboardController.clearResync);
 
+    // Dashboard forms: plain POST and redirect, so results land in
+    // Indiekit's own notification banner rather than a browser alert.
+    router.post(
+      "/feeds/:id/settings",
+      express.urlencoded({ extended: false }),
+      dashboardController.saveFeedSettings,
+    );
+    router.post(
+      "/feeds/:id/backfill",
+      express.urlencoded({ extended: false }),
+      dashboardController.backfillFeed,
+    );
+
     // Feed management (protected - requires auth)
     router.post("/api/feeds", express.json(), feedsController.add);
     router.delete("/api/feeds/:id", feedsController.remove);

package/lib/controllers/dashboard.js

@@ -1,3 +1,4 @@
+import { applyFeedSettings, rewindWatermark } from "../feed-settings.js";
 import { getSyncState, runSync } from "../sync.js";
 import { formatFeed, formatItem } from "../utils.js";
 
@@ -17,6 +18,32 @@ function consumeFlashMessage(request) {
   return result;
 }
 
+
+/**
+ * Publish context for a request, or null when the site is not configured.
+ * @param {object} request - Request
+ * @returns {object|null} Micropub endpoint and publication URL
+ */
+function publishContextFrom(request) {
+  const { application, publication } = request.app.locals;
+
+  return application?.micropubEndpoint && publication?.me
+    ? { micropubEndpoint: application.micropubEndpoint, me: publication.me }
+    : null;
+}
+
+/**
+ * Record a flash message and return to the dashboard.
+ * @param {object} request - Request
+ * @param {object} response - Response
+ * @param {string} type - success, error or warning
+ * @param {string} content - Message
+ */
+function flashBack(request, response, type, content) {
+  request.session.messages = [{ type, content }];
+  response.redirect(request.baseUrl);
+}
+
 export const dashboardController = {
   /**
    * Render admin dashboard
@@ -81,6 +108,112 @@ export const dashboardController = {
     }
   },
 
+
+  /**
+   * Save a feed's settings from the dashboard form
+   * POST /feeds/:id/settings
+   */
+  async saveFeedSettings(request, response) {
+    try {
+      const db = request.app.locals.application.getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const body = {
+        url: request.body.url,
+        publish: {
+          // An unchecked checkbox is omitted from a form post entirely, so
+          // presence is the signal.
+          enabled: request.body.enabled !== undefined,
+          postType: request.body.postType,
+          content: request.body.content,
+          linkProperty: request.body.linkProperty || null,
+          status: request.body.status,
+        },
+      };
+
+      const result = await applyFeedSettings(
+        db,
+        request.params.id,
+        body,
+        request.app.locals.publication,
+      );
+
+      if (result.error) {
+        return flashBack(request, response, "error", result.error);
+      }
+
+      // Saying "nothing is waiting" beats silence: a feed can be correctly
+      // configured and still publish nothing, because the watermark only
+      // admits items dated after publishing was switched on.
+      const message =
+        result.eligible === 0
+          ? "Settings saved. No items are waiting — only items published after you enabled this will post. Use Publish existing items to include the ones already cached."
+          : result.eligible > 0
+            ? `Settings saved. ${result.eligible} item(s) waiting to publish.`
+            : "Settings saved.";
+
+      flashBack(request, response, "success", message);
+    } catch (error) {
+      console.error("[RSS] Error saving feed settings:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
+  /**
+   * Rewind a feed's watermark and publish straight away
+   * POST /feeds/:id/backfill
+   */
+  async backfillFeed(request, response) {
+    try {
+      const { rssConfig, getRssDb } = request.app.locals.application;
+      const db = getRssDb?.();
+      if (!db) {
+        return flashBack(request, response, "error", "Database not available");
+      }
+
+      const result = await rewindWatermark(db, request.params.id, {
+        last: Number(request.body.last),
+      });
+
+      if (result.error) {
+        return flashBack(request, response, "error", result.error);
+      }
+
+      if (result.eligible === 0) {
+        return flashBack(
+          request,
+          response,
+          "warning",
+          "Nothing to publish: those items are already published or were skipped after repeated failures.",
+        );
+      }
+
+      // Run the sync now rather than leaving the user to wait a cycle and
+      // guess. This is the only way the answer can be "3 published" instead
+      // of "queued, come back later".
+      const sync = await runSync(db, rssConfig, publishContextFrom(request));
+
+      if (sync.error) {
+        return flashBack(request, response, "error", sync.error);
+      }
+
+      const published = sync.itemsPublished || 0;
+      flashBack(
+        request,
+        response,
+        published > 0 ? "success" : "warning",
+        published > 0
+          ? `Published ${published} item(s) from the cache.`
+          : `${result.eligible} item(s) became eligible but none published — check the logs for the reason.`,
+      );
+    } catch (error) {
+      console.error("[RSS] Error running backfill:", error.message);
+      flashBack(request, response, "error", error.message);
+    }
+  },
+
   /**
    * Clear all items and re-sync
    * POST /clear-resync

package/lib/controllers/feeds.js

@@ -1,7 +1,6 @@
 import { ObjectId } from "mongodb";
 import { RssClient } from "../rss-client.js";
-import { defaultContent, defaultLinkProperty } from "../jf2-builder.js";
-import { watermarkFor } from "../publisher.js";
+import { applyFeedSettings, rewindWatermark } from "../feed-settings.js";
 import { formatFeed, isValidUrl, normalizeUrl } from "../utils.js";
 
 export const feedsController = {
@@ -149,128 +148,31 @@ export const feedsController = {
   },
 
   /**
-   * Update a feed: enable/disable, or change its publish configuration
+   * Update a feed: enable/disable, change its url, or its publish config
    * PATCH /api/feeds/:id
-   * Body: { enabled?: boolean, publish?: object }
    */
   async toggle(request, response) {
     try {
-      const { id } = request.params;
-      const { enabled, publish, url } = request.body;
-
-      if (!ObjectId.isValid(id)) {
-        return response.status(400).json({ error: "Invalid feed ID" });
-      }
-
-      if (enabled === undefined && publish === undefined && url === undefined) {
-        return response.status(400).json({ error: "Nothing to update" });
-      }
-
-      if (enabled !== undefined && typeof enabled !== "boolean") {
-        return response.status(400).json({ error: "enabled must be boolean" });
-      }
-
       const db = request.app.locals.application.getRssDb?.();
       if (!db) {
         return response.status(500).json({ error: "Database not available" });
       }
 
-      const feedsCollection = db.collection("rssFeeds");
-      const feedId = new ObjectId(id);
-      const feed = await feedsCollection.findOne({ _id: feedId });
-
-      if (!feed) {
-        return response.status(404).json({
-          error: response.locals.__("rss.error.feedNotFound"),
-        });
-      }
-
-      const update = {};
-
-      if (enabled !== undefined) {
-        update.enabled = enabled;
-      }
-
-      if (url !== undefined) {
-        if (!isValidUrl(url)) {
-          return response.status(400).json({
-            error: response.locals.__("rss.error.invalidUrl"),
-          });
-        }
-
-        const normalizedUrl = normalizeUrl(url);
-        const clash = await feedsCollection.findOne({
-          url: normalizedUrl,
-          _id: { $ne: feedId },
-        });
-
-        if (clash) {
-          return response.status(409).json({
-            error: response.locals.__("rss.error.feedExists"),
-          });
-        }
-
-        update.url = normalizedUrl;
-        // Items key on feedId, so they survive the change. Any error from the
-        // old URL is stale the moment it is corrected.
-        update.lastError = null;
-      }
-
-      if (publish !== undefined) {
-        const { postTypes } = request.app.locals.publication || {};
-
-        if (publish.enabled) {
-          if (!postTypes) {
-            return response.status(500).json({
-              error: "Post type configuration unavailable",
-            });
-          }
-
-          // postData.create() throws notImplemented for an unconfigured type,
-          // and the failure would be buried deep in the sync loop. Sites
-          // differ: chardonsbleus has audio, event, jam and rsvp disabled.
-          if (!Object.hasOwn(postTypes, publish.postType)) {
-            return response.status(400).json({
-              error: `Post type "${publish.postType}" is not enabled on this site`,
-            });
-          }
-        }
-
-        const merged = { ...feed.publish, ...publish };
+      const result = await applyFeedSettings(
+        db,
+        request.params.id,
+        request.body,
+        request.app.locals.publication,
+      );
 
-        update.publish = {
-          ...merged,
-          // Pre-fill from the post type when the caller left them out, so a
-          // minimal PATCH of { enabled, postType } yields a working config.
-          // An explicit null is respected: it means "no link property".
-          content: merged.content || defaultContent(merged.postType),
-          linkProperty:
-            publish.linkProperty === undefined
-              ? (merged.linkProperty ?? defaultLinkProperty(merged.postType))
-              : publish.linkProperty,
-          // Stamp the watermark the first time publishing is switched on, so
-          // enabling a feed does not publish its entire cached backlog.
-          since:
-            publish.enabled && !feed.publish?.since
-              ? new Date().toISOString()
-              : (feed.publish?.since ?? null),
-        };
+      if (result.error) {
+        return response.status(result.status).json({ error: result.error });
       }
 
-      const result = await feedsCollection.findOneAndUpdate(
-        { _id: feedId },
-        { $set: update },
-        { returnDocument: "after" },
-      );
-
       response.json({
-        message:
-          enabled === undefined
-            ? response.locals.__("rss.success.feedUpdated")
-            : enabled
-              ? response.locals.__("rss.success.feedEnabled")
-              : response.locals.__("rss.success.feedDisabled"),
-        feed: formatFeed(result),
+        eligible: result.eligible,
+        message: response.locals.__("rss.success.feedUpdated"),
+        feed: formatFeed(result.feed),
       });
     } catch (error) {
       console.error("[RSS] Error updating feed:", error.message);
@@ -281,70 +183,24 @@ export const feedsController = {
   /**
    * Rewind a feed's watermark so past items get published
    * POST /api/feeds/:id/backfill
-   * Body: { since?: string } or { last?: number }
    */
   async backfill(request, response) {
     try {
-      const { id } = request.params;
-      const { since, last } = request.body;
-
-      if (!ObjectId.isValid(id)) {
-        return response.status(400).json({ error: "Invalid feed ID" });
-      }
-
       const db = request.app.locals.application.getRssDb?.();
       if (!db) {
         return response.status(500).json({ error: "Database not available" });
       }
 
-      const feedsCollection = db.collection("rssFeeds");
-      const itemsCollection = db.collection("rssItems");
-      const feedId = new ObjectId(id);
-      const feed = await feedsCollection.findOne({ _id: feedId });
-
-      if (!feed) {
-        return response.status(404).json({
-          error: response.locals.__("rss.error.feedNotFound"),
-        });
-      }
-
-      if (!feed.publish?.enabled) {
-        return response.status(400).json({
-          error: "Enable publishing on this feed first",
-        });
-      }
+      const result = await rewindWatermark(db, request.params.id, request.body);
 
-      let oldestPubDate = null;
-      if (last) {
-        const items = await itemsCollection
-          .find({ feedId })
-          .sort({ pubDate: -1 })
-          .limit(Number(last))
-          .toArray();
-        oldestPubDate = items.at(-1)?.pubDate || null;
+      if (result.error) {
+        return response.status(result.status).json({ error: result.error });
       }
 
-      let watermark;
-      try {
-        watermark = watermarkFor({ since, last }, oldestPubDate);
-      } catch (error) {
-        return response.status(400).json({ error: error.message });
-      }
-
-      await feedsCollection.updateOne(
-        { _id: feedId },
-        {
-          $set: {
-            "publish.since": watermark,
-            // Imported history is always drafts, whatever the feed is set to.
-            "publish.status": "draft",
-          },
-        },
-      );
-
       response.json({
+        eligible: result.eligible,
         message: response.locals.__("rss.success.backfillQueued"),
-        since: watermark,
+        since: result.since,
       });
     } catch (error) {
       console.error("[RSS] Error queueing backfill:", error.message);

package/lib/feed-settings.js

@@ -0,0 +1,183 @@
+import { ObjectId } from "mongodb";
+
+import { defaultContent, defaultLinkProperty } from "./jf2-builder.js";
+import { pendingQuery, watermarkFor } from "./publisher.js";
+import { isValidUrl, normalizeUrl } from "./utils.js";
+
+/**
+ * Apply a settings change to one feed.
+ *
+ * Shared by the JSON API and the dashboard form so the two cannot drift: two
+ * copies of this validation would eventually disagree, and the symptom would
+ * be a form that accepts what the API rejects.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @param {object} body - Fields to change: enabled, url, publish
+ * @param {object} [publication] - Publication configuration
+ * @returns {Promise<object>} Result carrying status, error, feed, eligible
+ */
+export async function applyFeedSettings(db, id, body, publication = {}) {
+  const { enabled, publish, url } = body;
+
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  if (enabled === undefined && publish === undefined && url === undefined) {
+    return { status: 400, error: "Nothing to update" };
+  }
+
+  if (enabled !== undefined && typeof enabled !== "boolean") {
+    return { status: 400, error: "enabled must be boolean" };
+  }
+
+  const feedsCollection = db.collection("rssFeeds");
+  const feedId = new ObjectId(id);
+  const feed = await feedsCollection.findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  const update = {};
+
+  if (enabled !== undefined) {
+    update.enabled = enabled;
+  }
+
+  if (url !== undefined) {
+    if (!isValidUrl(url)) {
+      return { status: 400, error: "Invalid feed URL" };
+    }
+
+    const normalizedUrl = normalizeUrl(url);
+    const clash = await feedsCollection.findOne({
+      url: normalizedUrl,
+      _id: { $ne: feedId },
+    });
+
+    if (clash) {
+      return { status: 409, error: "Another feed already uses that URL" };
+    }
+
+    update.url = normalizedUrl;
+    // Items key on feedId, so they survive the change. Any error from the old
+    // URL is stale the moment it is corrected.
+    update.lastError = null;
+  }
+
+  if (publish !== undefined) {
+    const { postTypes } = publication;
+
+    if (publish.enabled) {
+      if (!postTypes) {
+        return { status: 500, error: "Post type configuration unavailable" };
+      }
+
+      // postData.create() throws notImplemented for an unconfigured type, and
+      // the failure would be buried deep in the sync loop. Sites differ:
+      // chardonsbleus has audio, event, jam and rsvp disabled.
+      if (!Object.hasOwn(postTypes, publish.postType)) {
+        return {
+          status: 400,
+          error: `Post type "${publish.postType}" is not enabled on this site`,
+        };
+      }
+    }
+
+    const merged = { ...feed.publish, ...publish };
+
+    update.publish = {
+      ...merged,
+      // Pre-fill from the post type when the caller left them out, so a
+      // minimal update of enabled plus postType yields a working config. An
+      // explicit null is respected: it means "no link property".
+      content: merged.content || defaultContent(merged.postType),
+      linkProperty:
+        publish.linkProperty === undefined
+          ? (merged.linkProperty ?? defaultLinkProperty(merged.postType))
+          : publish.linkProperty,
+      // Stamp the watermark the first time publishing is switched on, so
+      // enabling a feed does not publish its entire cached backlog.
+      since:
+        publish.enabled && !feed.publish?.since
+          ? new Date().toISOString()
+          : (feed.publish?.since ?? null),
+    };
+  }
+
+  const result = await feedsCollection.findOneAndUpdate(
+    { _id: feedId },
+    { $set: update },
+    { returnDocument: "after" },
+  );
+
+  // A feed can be correctly configured and still have nothing to publish, so
+  // report the count rather than leaving the user to guess.
+  let eligible;
+  if (result.publish?.enabled) {
+    eligible = await db
+      .collection("rssItems")
+      .countDocuments(pendingQuery(result));
+  }
+
+  return { status: 200, feed: result, eligible };
+}
+
+/**
+ * Move a feed's watermark back so already-cached items become eligible.
+ * @param {object} db - Database
+ * @param {string} id - Feed id
+ * @param {object} body - Either a since date or a last item count
+ * @returns {Promise<object>} Result carrying status, error, since, eligible
+ */
+export async function rewindWatermark(db, id, body) {
+  const { since, last } = body;
+
+  if (!ObjectId.isValid(id)) {
+    return { status: 400, error: "Invalid feed ID" };
+  }
+
+  const feedsCollection = db.collection("rssFeeds");
+  const itemsCollection = db.collection("rssItems");
+  const feedId = new ObjectId(id);
+  const feed = await feedsCollection.findOne({ _id: feedId });
+
+  if (!feed) {
+    return { status: 404, error: "Feed not found" };
+  }
+
+  if (!feed.publish?.enabled) {
+    return { status: 400, error: "Enable publishing on this feed first" };
+  }
+
+  let oldestPubDate = null;
+  if (last) {
+    const items = await itemsCollection
+      .find({ feedId })
+      .sort({ pubDate: -1 })
+      .limit(Number(last))
+      .toArray();
+    oldestPubDate = items.at(-1)?.pubDate || null;
+  }
+
+  let watermark;
+  try {
+    watermark = watermarkFor({ since, last }, oldestPubDate);
+  } catch (error) {
+    return { status: 400, error: error.message };
+  }
+
+  // The feed's own post status is respected: silently forcing drafts would
+  // override a deliberate choice. The guard is the count the user asked for,
+  // and maxPostsPerCycle pacing the catch-up.
+  const updated = await feedsCollection.findOneAndUpdate(
+    { _id: feedId },
+    { $set: { "publish.since": watermark } },
+    { returnDocument: "after" },
+  );
+
+  const eligible = await itemsCollection.countDocuments(pendingQuery(updated));
+
+  return { status: 200, since: watermark, eligible };
+}

package/lib/publisher.js

@@ -102,6 +102,44 @@ export function resolvePublishContext(application = {}, publication = {}) {
 
 const MAX_ATTEMPTS = 3;
 
+/**
+ * Query selecting a feed's items that are still waiting to be published.
+ *
+ * Exported so the dashboard can count them with exactly the same rule the
+ * publish loop applies. Two copies of this would drift, and the symptom would
+ * be a count that disagrees with what actually publishes.
+ * @param {object} feed - Feed document
+ * @returns {object} MongoDB query
+ */
+export function pendingQuery(feed) {
+  const query = {
+    feedId: feed._id,
+    postedAt: { $exists: false },
+    postSkipped: { $ne: true },
+  };
+
+  if (feed.publish?.since) {
+    const since = feed.publish.since;
+
+    // One watermark, two operand types, because the two fields are stored
+    // differently: pubDate is a BSON Date (rss-client builds real Dates and
+    // pruneOldItems compares it against one), while fetchedAt — like every
+    // other date here — is an ISO string. MongoDB orders by BSON type before
+    // value, so a Date is never $gt a String: passing the ISO string to
+    // pubDate matches nothing at all, with no error to show for it.
+    //
+    // An item with no parsable date would also be excluded forever by a bare
+    // comparison, because null sorts below every Date. fetchedAt is always set
+    // on insert and stands in when pubDate is missing.
+    query.$or = [
+      { pubDate: { $gt: new Date(since) } },
+      { pubDate: null, fetchedAt: { $gt: since } },
+    ];
+  }
+
+  return query;
+}
+
 /**
  * Publish the pending items of one feed.
  *
@@ -131,22 +169,7 @@ export async function publishPending(feed, itemsCollection, options) {
     mintImpl = mintToken,
   } = options;
 
-  const query = {
-    feedId: feed._id,
-    postedAt: { $exists: false },
-    postSkipped: { $ne: true },
-  };
-
-  if (feed.publish.since) {
-    // An item with no parsable date would be excluded forever by a bare
-    // `pubDate: { $gt: since }`, because null sorts below every Date in BSON —
-    // a silent permanent no-op. fetchedAt is always set on insert and is
-    // monotonic with cache entry, so it stands in when pubDate is missing.
-    query.$or = [
-      { pubDate: { $gt: feed.publish.since } },
-      { pubDate: null, fetchedAt: { $gt: feed.publish.since } },
-    ];
-  }
+  const query = pendingQuery(feed);
 
   const items = await itemsCollection
     .find(query)

package/locales/en.json

@@ -42,7 +42,7 @@
       "feedUpdated": "Feed updated",
       "feedEnabled": "Feed enabled",
       "feedDisabled": "Feed disabled",
-      "backfillQueued": "Backfill queued; items will be published as drafts over the next sync cycles",
+      "backfillQueued": "Backfill queued: the newest cached items will be published over the next sync cycles, using this feed's post status.",
       "syncComplete": "Sync complete",
       "clearResync": "Items cleared and re-synced"
     },
@@ -68,6 +68,11 @@
       "title": "Feed settings",
       "url": "Feed URL",
       "urlHint": "Correcting this keeps the cached items; removing and re-adding the feed would lose them."
+    },
+    "backfill": {
+      "label": "Publish existing items",
+      "hint": "How many of the newest cached items to publish. They are created with this feed's post status, a few per sync cycle.",
+      "submit": "Publish existing items"
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-rss",
-  "version": "1.1.2",
+  "version": "1.1.4",
   "description": "RSS feed reader endpoint for Indiekit. Aggregates multiple feeds, caches in MongoDB, displays on frontend.",
   "keywords": [
     "indiekit",

package/views/rss.njk

@@ -113,7 +113,7 @@
               classes: "rss-publish",
               summary: publishSummary
             }) %}
-              <form data-publish-feed="{{ feed.id }}">
+              <form action="{{ mountPath }}/feeds/{{ feed.id }}/settings" method="post">
                 {{ input({
                   id: "feed-url-" + feed.id,
                   name: "url",
@@ -180,6 +180,24 @@
                   text: __("rss.publish.save")
                 }) }}
               </form>
+
+              {% if feed.publish.enabled %}
+              <form class="rss-backfill" action="{{ mountPath }}/feeds/{{ feed.id }}/backfill" method="post">
+                {{ input({
+                  id: "backfill-last-" + feed.id,
+                  name: "last",
+                  type: "number",
+                  label: __("rss.backfill.label"),
+                  hint: __("rss.backfill.hint"),
+                  value: "10"
+                }) }}
+                {{ button({
+                  classes: "button--secondary",
+                  type: "submit",
+                  text: __("rss.backfill.submit")
+                }) }}
+              </form>
+              {% endif %}
             {% endcall %}
           </div>
           <div class="rss-feed-actions">
@@ -362,40 +380,5 @@
     });
   });
 
-  // Handle publish configuration
-  document.querySelectorAll('[data-publish-feed]').forEach(form => {
-    form.addEventListener('submit', async (e) => {
-      e.preventDefault();
-      const feedId = e.target.dataset.publishFeed;
-      const data = new FormData(e.target);
-
-      const publish = {
-        // The checkboxes component sets value="true"; an unchecked box is
-        // omitted from FormData entirely, so presence is the signal.
-        enabled: data.get('enabled') !== null,
-        postType: data.get('postType'),
-        content: data.get('content'),
-        linkProperty: data.get('linkProperty') || null,
-        status: data.get('status'),
-      };
-
-      try {
-        const response = await fetch(`{{ mountPath }}/api/feeds/${feedId}`, {
-          method: 'PATCH',
-          headers: { 'Content-Type': 'application/json' },
-          body: JSON.stringify({ url: data.get('url'), publish })
-        });
-
-        if (response.ok) {
-          location.reload();
-        } else {
-          const body = await response.json();
-          alert(body.error || 'Failed to save publish settings');
-        }
-      } catch (err) {
-        alert('Failed to save publish settings: ' + err.message);
-      }
-    });
-  });
 </script>
 {% endblock %}