@rmdes/indiekit-endpoint-microsub 1.0.61 → 1.0.64

package/lib/storage/items-retention.js

@@ -3,18 +3,21 @@
  * @module storage/items-retention
  */
 
+import { MAX_FULL_READ_ITEMS } from "../utils/constants.js";
 import { getCollection } from "./items.js";
 
-// Maximum number of full read items to keep per channel before stripping content.
-const MAX_FULL_READ_ITEMS = 200;
-
-// Maximum age (in days) for stripped skeletons and unread items.
-// After this period, both are hard-deleted to prevent unbounded growth.
-const MAX_ITEM_AGE_DAYS = 30;
+// Global retention defaults. Each can be overridden per channel via
+// channel.settings.{maxItems,maxItemsPerFeed,maxUnreadAgeDays}. The "notifications"
+// channel is exempt from these caps entirely — webmentions are high-signal and
+// users may want long history there.
+export const DEFAULT_MAX_ITEMS = 1000;
+export const DEFAULT_MAX_ITEMS_PER_FEED = 50;
+export const DEFAULT_MAX_UNREAD_AGE_DAYS = 30;
 
 /**
  * Cleanup all read items across all channels (startup cleanup).
- * RSS items are stripped to dedup skeletons; AP items are hard-deleted.
+ * Read items beyond MAX_FULL_READ_ITEMS are stripped to skeletons (kept for
+ * dedup, content removed).
  * @param {object} application - Indiekit application
  * @returns {Promise<number>} Total number of items cleaned up
  */
@@ -49,59 +52,36 @@ export async function cleanupAllReadItems(application) {
           })
           .sort({ published: -1, _id: -1 })
           .skip(MAX_FULL_READ_ITEMS)
-          .project({ _id: 1, feedId: 1 })
+          .project({ _id: 1 })
           .toArray();
 
         if (itemsToCleanup.length > 0) {
-          const apItemIds = [];
-          const rssItemIds = [];
-          for (const item of itemsToCleanup) {
-            if (item.feedId) {
-              rssItemIds.push(item._id);
-            } else {
-              apItemIds.push(item._id);
-            }
-          }
-
-          // Hard-delete AP items
-          if (apItemIds.length > 0) {
-            const deleted = await collection.deleteMany({
-              _id: { $in: apItemIds },
-            });
-            totalCleaned += deleted.deletedCount;
-            console.info(
-              `[Microsub] Startup cleanup: deleted ${deleted.deletedCount} AP items from channel "${channel.name}"`,
-            );
-          }
-
-          // Strip RSS items to skeletons
-          if (rssItemIds.length > 0) {
-            const stripped = await collection.updateMany(
-              { _id: { $in: rssItemIds } },
-              {
-                $set: { _stripped: true },
-                $unset: {
-                  name: "",
-                  content: "",
-                  summary: "",
-                  author: "",
-                  category: "",
-                  photo: "",
-                  video: "",
-                  audio: "",
-                  likeOf: "",
-                  repostOf: "",
-                  bookmarkOf: "",
-                  inReplyTo: "",
-                  source: "",
-                },
+          const ids = itemsToCleanup.map((item) => item._id);
+          const stripped = await collection.updateMany(
+            { _id: { $in: ids } },
+            {
+              $set: { _stripped: true },
+              $unset: {
+                name: "",
+                content: "",
+                summary: "",
+                author: "",
+                category: "",
+                photo: "",
+                video: "",
+                audio: "",
+                likeOf: "",
+                repostOf: "",
+                bookmarkOf: "",
+                inReplyTo: "",
+                source: "",
               },
-            );
-            totalCleaned += stripped.modifiedCount;
-            console.info(
-              `[Microsub] Startup cleanup: stripped ${stripped.modifiedCount} RSS items from channel "${channel.name}"`,
-            );
-          }
+            },
+          );
+          totalCleaned += stripped.modifiedCount;
+          console.info(
+            `[Microsub] Startup cleanup: stripped ${stripped.modifiedCount} items from channel "${channel.name}"`,
+          );
         }
       }
     }
@@ -117,58 +97,187 @@ export async function cleanupAllReadItems(application) {
 }
 
 /**
- * Delete stale items: stripped skeletons and unread items older than MAX_ITEM_AGE_DAYS.
- * Stripped skeletons have served their dedup purpose; stale unread items are unlikely
- * to be read. Both are hard-deleted to prevent unbounded collection growth.
+ * Per-channel retention cleanup. For each channel (excluding `notifications`):
+ *   1. Drop unread items + stripped skeletons older than `maxUnreadAgeDays`.
+ *   2. Per-feed cap: keep most recent `maxItemsPerFeed` items per feed, drop the rest.
+ *   3. Channel-wide cap: keep most recent `maxItems` items total, drop the rest.
+ *
+ * Each channel uses its own `channel.settings.{maxItems,maxItemsPerFeed,maxUnreadAgeDays}`
+ * when present; otherwise the module-level defaults apply. This makes the policy
+ * configurable per channel — a noisy aggregator channel can set tight caps while
+ * a low-volume curated channel keeps a long tail.
+ *
+ * The order matters: per-feed cap runs before channel cap so a single prolific
+ * feed cannot starve other feeds in the channel of representation after the
+ * channel-wide trim.
+ *
  * @param {object} application - Indiekit application
- * @returns {Promise<number>} Total number of items deleted
+ * @returns {Promise<number>} Total number of items deleted across all channels
  */
 export async function cleanupStaleItems(application) {
-  const collection = getCollection(application);
-  const cutoff = new Date();
-  cutoff.setDate(cutoff.getDate() - MAX_ITEM_AGE_DAYS);
-
-  // Delete stripped skeletons older than cutoff
-  const strippedResult = await collection.deleteMany({
-    _stripped: true,
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
+  const itemsCollection = getCollection(application);
+  const channelsCollection = application.collections.get("microsub_channels");
 
-  // Delete unread items older than cutoff
-  const unreadResult = await collection.deleteMany({
-    readBy: { $in: [null, []] },
-    _stripped: { $ne: true },
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
+  const channels = await channelsCollection.find({}).toArray();
+  let totalDeleted = 0;
 
-  // Also catch items with no readBy field at all
-  const noReadByResult = await collection.deleteMany({
-    readBy: { $exists: false },
-    _stripped: { $ne: true },
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
+  for (const channel of channels) {
+    // Notifications channel (webmentions) is exempt — high-signal, kept indefinitely.
+    if (channel.uid === "notifications") continue;
+
+    const settings = channel.settings || {};
+    const maxItems = settings.maxItems ?? DEFAULT_MAX_ITEMS;
+    const maxItemsPerFeed =
+      settings.maxItemsPerFeed ?? DEFAULT_MAX_ITEMS_PER_FEED;
+    const maxUnreadAgeDays =
+      settings.maxUnreadAgeDays ?? DEFAULT_MAX_UNREAD_AGE_DAYS;
+
+    const cutoff = new Date();
+    cutoff.setDate(cutoff.getDate() - maxUnreadAgeDays);
+    const cutoffIso = cutoff.toISOString();
+    let channelDeleted = 0;
+    let staleDeleted = 0;
+    let perFeedDeleted = 0;
+    let channelCapDeleted = 0;
+
+    // 1. Drop stripped skeletons older than cutoff (served their dedup purpose).
+    const strippedResult = await itemsCollection.deleteMany({
+      channelId: channel._id,
+      _stripped: true,
+      $or: [
+        { published: { $lt: cutoff } },
+        {
+          published: { $exists: false },
+          createdAt: { $lt: cutoffIso },
+        },
+      ],
+    });
+    staleDeleted += strippedResult.deletedCount;
+
+    // 1b. Drop unread (or never-read) items older than cutoff.
+    const unreadAgeResult = await itemsCollection.deleteMany({
+      channelId: channel._id,
+      _stripped: { $ne: true },
+      $and: [
+        {
+          $or: [
+            { readBy: { $exists: false } },
+            { readBy: { $size: 0 } },
+            { readBy: null },
+          ],
+        },
+        {
+          $or: [
+            { published: { $lt: cutoff } },
+            {
+              published: { $exists: false },
+              createdAt: { $lt: cutoffIso },
+            },
+          ],
+        },
+      ],
+    });
+    staleDeleted += unreadAgeResult.deletedCount;
+    channelDeleted += staleDeleted;
+
+    // 2. Per-feed cap. Iterate feeds in the channel; for each, delete oldest
+    //    items beyond maxItemsPerFeed regardless of read state.
+    const feedIds = await itemsCollection.distinct("feedId", {
+      channelId: channel._id,
+      feedId: { $exists: true, $ne: null },
+    });
+
+    for (const feedId of feedIds) {
+      const excess = await itemsCollection
+        .find({ channelId: channel._id, feedId })
+        .sort({ published: -1, _id: -1 })
+        .skip(maxItemsPerFeed)
+        .project({ _id: 1 })
+        .toArray();
+
+      if (excess.length > 0) {
+        const ids = excess.map((item) => item._id);
+        const result = await itemsCollection.deleteMany({
+          _id: { $in: ids },
+        });
+        perFeedDeleted += result.deletedCount;
+        channelDeleted += result.deletedCount;
+      }
+    }
+
+    // 3. Channel-wide cap. Catches items without feedId plus anything still over
+    //    the per-channel ceiling after the per-feed pass.
+    const excessChannel = await itemsCollection
+      .find({ channelId: channel._id })
+      .sort({ published: -1, _id: -1 })
+      .skip(maxItems)
+      .project({ _id: 1 })
+      .toArray();
+
+    if (excessChannel.length > 0) {
+      const ids = excessChannel.map((item) => item._id);
+      const result = await itemsCollection.deleteMany({ _id: { $in: ids } });
+      channelCapDeleted += result.deletedCount;
+      channelDeleted += result.deletedCount;
+    }
 
-  const total =
-    strippedResult.deletedCount +
-    unreadResult.deletedCount +
-    noReadByResult.deletedCount;
+    if (channelDeleted > 0) {
+      console.info(
+        `[Microsub] Retention cleanup "${channel.name}": deleted ${channelDeleted} items ` +
+          `(stale: ${staleDeleted}, per-feed: ${perFeedDeleted}, channel-cap: ${channelCapDeleted}; ` +
+          `maxItems=${maxItems}, maxItemsPerFeed=${maxItemsPerFeed}, maxUnreadAgeDays=${maxUnreadAgeDays})`,
+      );
+    }
+    totalDeleted += channelDeleted;
+  }
 
-  if (total > 0) {
+  if (totalDeleted > 0) {
     console.info(
-      `[Microsub] Stale cleanup: deleted ${strippedResult.deletedCount} stripped skeletons, ` +
-        `${unreadResult.deletedCount + noReadByResult.deletedCount} stale unread items ` +
-        `(cutoff: ${MAX_ITEM_AGE_DAYS} days)`,
+      `[Microsub] Retention cleanup complete: ${totalDeleted} total items deleted across ${channels.length} channels`,
     );
   }
 
-  return total;
+  return totalDeleted;
+}
+
+/**
+ * One-time migration: remove the abandoned "Fediverse" channel and its items.
+ * The microsub reader briefly tried to ingest ActivityPub outboxes into a
+ * dedicated channel (uid: "activitypub"). That feature was abandoned — fediverse
+ * federation lives entirely in the separate `indiekit-endpoint-activitypub`
+ * plugin now. This migration cleans up the leftover channel and items.
+ * Idempotent — safe to run on every startup.
+ * @param {object} application - Indiekit application
+ * @returns {Promise<{ channelsRemoved: number, itemsRemoved: number }>}
+ */
+export async function removeActivityPubData(application) {
+  const itemsCollection = getCollection(application);
+  const channelsCollection = application.collections.get("microsub_channels");
+
+  const apChannels = await channelsCollection
+    .find({ uid: "activitypub" })
+    .toArray();
+
+  if (apChannels.length === 0) {
+    return { channelsRemoved: 0, itemsRemoved: 0 };
+  }
+
+  const channelIds = apChannels.map((c) => c._id);
+
+  const itemsResult = await itemsCollection.deleteMany({
+    channelId: { $in: channelIds },
+  });
+
+  const channelsResult = await channelsCollection.deleteMany({
+    _id: { $in: channelIds },
+  });
+
+  console.info(
+    `[Microsub] Removed abandoned Fediverse channel: ${channelsResult.deletedCount} channel(s), ${itemsResult.deletedCount} item(s)`,
+  );
+
+  return {
+    channelsRemoved: channelsResult.deletedCount,
+    itemsRemoved: itemsResult.deletedCount,
+  };
 }

package/lib/utils/blogroll-notify.js

@@ -36,7 +36,7 @@ export async function notifyBlogroll(application, action, data) {
       status: "deleted",
     });
     if (deleted) {
-      console.log(
+      console.info(
         `[Microsub→Blogroll] Skipping follow for ${data.url} — previously deleted by user`,
       );
       return;
@@ -76,7 +76,7 @@ export async function notifyBlogroll(application, action, data) {
       { upsert: true },
     );
 
-    console.log(`[Microsub→Blogroll] Added/updated feed ${data.url}`);
+    console.info(`[Microsub→Blogroll] Added/updated feed ${data.url}`);
   } else if (action === "unfollow") {
     // Soft-delete the blog entry if it came from microsub
     const result = await collection.updateOne(
@@ -96,7 +96,7 @@ export async function notifyBlogroll(application, action, data) {
     );
 
     if (result.modifiedCount > 0) {
-      console.log(`[Microsub→Blogroll] Soft-deleted feed ${data.url}`);
+      console.info(`[Microsub→Blogroll] Soft-deleted feed ${data.url}`);
     }
   }
 }

package/lib/utils/constants.js

@@ -5,3 +5,10 @@
 
 /** Retention period for unread count queries (only count recent items) */
 export const UNREAD_RETENTION_DAYS = 30;
+
+/**
+ * Maximum number of full read items to keep per channel/user before stripping
+ * content. Items beyond this limit are converted to lightweight dedup skeletons
+ * (channelId, uid, readBy) so the poller doesn't re-ingest them as new unread.
+ */
+export const MAX_FULL_READ_ITEMS = 200;

package/lib/utils/jf2.js

@@ -29,115 +29,6 @@ export function generateChannelUid() {
   return result;
 }
 
-/**
- * Create a jf2 Item from normalized feed data
- * @param {object} data - Normalized item data
- * @param {object} source - Feed source metadata
- * @returns {object} jf2 Item object
- */
-export function createJf2Item(data, source) {
-  return {
-    type: "entry",
-    uid: data.uid,
-    url: data.url,
-    name: data.name || undefined,
-    content: data.content || undefined,
-    summary: data.summary || undefined,
-    published: data.published,
-    updated: data.updated || undefined,
-    author: data.author || undefined,
-    category: data.category || [],
-    photo: data.photo || [],
-    video: data.video || [],
-    audio: data.audio || [],
-    // Interaction types
-    "like-of": data.likeOf || [],
-    "repost-of": data.repostOf || [],
-    "bookmark-of": data.bookmarkOf || [],
-    "in-reply-to": data.inReplyTo || [],
-    // Internal properties (prefixed with _)
-    _id: data._id,
-    _is_read: data._is_read || false,
-    _source: source,
-  };
-}
-
-/**
- * Create a jf2 Card (author/person)
- * @param {object} data - Author data
- * @returns {object} jf2 Card object
- */
-export function createJf2Card(data) {
-  if (!data) return;
-
-  return {
-    type: "card",
-    name: data.name || undefined,
-    url: data.url || undefined,
-    photo: data.photo || undefined,
-  };
-}
-
-/**
- * Create a jf2 Content object
- * @param {string} text - Plain text content
- * @param {string} html - HTML content
- * @returns {object|undefined} jf2 Content object
- */
-export function createJf2Content(text, html) {
-  if (!text && !html) return;
-
-  return {
-    text: text || stripHtml(html),
-    html: html || undefined,
-  };
-}
-
-/**
- * Strip HTML tags from string
- * @param {string} html - HTML string
- * @returns {string} Plain text
- */
-export function stripHtml(html) {
-  if (!html) return "";
-  return html.replaceAll(/<[^>]*>/g, "").trim();
-}
-
-/**
- * Create a jf2 Feed response
- * @param {object} options - Feed options
- * @param {Array} options.items - Array of jf2 items
- * @param {object} options.paging - Pagination cursors
- * @returns {object} jf2 Feed object
- */
-export function createJf2Feed({ items, paging }) {
-  const feed = {
-    items: items || [],
-  };
-
-  if (paging) {
-    feed.paging = {};
-    if (paging.before) feed.paging.before = paging.before;
-    if (paging.after) feed.paging.after = paging.after;
-  }
-
-  return feed;
-}
-
-/**
- * Create a Channel response object
- * @param {object} channel - Channel data
- * @param {number} unreadCount - Number of unread items
- * @returns {object} Channel object for API response
- */
-export function createChannelResponse(channel, unreadCount = 0) {
-  return {
-    uid: channel.uid,
-    name: channel.name,
-    unread: unreadCount > 0 ? unreadCount : false,
-  };
-}
-
 /**
  * Create a Feed response object
  * @param {object} feed - Feed data

package/lib/utils/sanitize.js

@@ -1,6 +1,5 @@
 /**
- * Shared HTML sanitization configuration
- * Used by both RSS/Atom normalizer and ActivityPub outbox fetcher
+ * Shared HTML sanitization configuration used by the feed normalizers.
  * @module utils/sanitize
  */
 

package/lib/utils/validation.js

@@ -163,6 +163,31 @@ export function validateExcludeTypes(types) {
   return types.filter((type) => VALID_EXCLUDE_TYPES.includes(type));
 }
 
+/**
+ * Validate a per-channel retention numeric setting.
+ * Accepts a positive integer; returns undefined for empty/invalid input so the
+ * caller can fall back to the global default.
+ * @param {string|number|undefined} value - Raw form value
+ * @param {object} [options]
+ * @param {number} [options.min] - Minimum allowed value (inclusive). Default 1.
+ * @param {number} [options.max] - Maximum allowed value (inclusive). Default 1_000_000.
+ * @returns {number|undefined} Validated integer, or undefined if empty/invalid
+ */
+export function validateRetentionSetting(value, options = {}) {
+  const { min = 1, max = 1_000_000 } = options;
+
+  if (value === undefined || value === null || value === "") {
+    return undefined;
+  }
+
+  const parsed = Number.parseInt(String(value), 10);
+  if (!Number.isFinite(parsed) || parsed < min || parsed > max) {
+    return undefined;
+  }
+
+  return parsed;
+}
+
 /**
  * Validate regex pattern
  * @param {string} pattern - Regex pattern to validate

package/lib/webmention/processor.js

@@ -30,7 +30,7 @@ export async function processWebmention(application, source, target, userId) {
   const verification = await verifyWebmention(source, target);
 
   if (!verification.verified) {
-    console.log(
+    console.info(
       `[Microsub] Webmention verification failed: ${verification.error}`,
     );
     return {
@@ -87,7 +87,7 @@ export async function processWebmention(application, source, target, userId) {
     });
   }
 
-  console.log(
+  console.info(
     `[Microsub] Webmention processed: ${verification.type} from ${source}`,
   );
 

package/lib/websub/handler.js

@@ -64,7 +64,7 @@ export async function verify(request, response) {
     });
   }
 
-  console.log(`[Microsub] WebSub subscription verified for ${feed.url}`);
+  console.info(`[Microsub] WebSub subscription verified for ${feed.url}`);
 
   // Return challenge to verify subscription
   response.type("text/plain").send(challenge);
@@ -143,7 +143,7 @@ async function processWebsubContent(application, feed, contentType, body) {
     // Parse the pushed content
     const parsed = await parseFeed(content, feed.url, { contentType });
 
-    console.log(
+    console.info(
       `[Microsub] Processing ${parsed.items.length} items from WebSub push for ${feed.url}`,
     );
 

package/locales/en.json

@@ -10,7 +10,6 @@
       "showRead": "Show read ({{count}})",
       "hideRead": "Hide read items",
       "allRead": "All caught up!",
-      "newer": "Newer",
       "older": "Older"
     },
     "channels": {
@@ -18,34 +17,21 @@
       "name": "Channel name",
       "new": "New channel",
       "create": "Create channel",
-      "delete": "Delete channel",
       "settings": "Channel settings",
-      "empty": "No channels yet. Create one to get started.",
-      "notifications": "Notifications"
+      "empty": "No channels yet. Create one to get started."
     },
     "timeline": {
-      "title": "Timeline",
       "empty": "No items in this channel",
-      "markRead": "Mark as read",
-      "markUnread": "Mark as unread",
-      "remove": "Remove"
+      "markRead": "Mark as read"
     },
     "feeds": {
       "title": "Feeds",
       "follow": "Follow",
       "subscribe": "Subscribe to a feed",
-      "unfollow": "Unfollow",
       "empty": "No feeds followed in this channel",
       "url": "Feed URL",
       "urlPlaceholder": "https://example.com/feed.xml",
-      "edit": "Edit feed",
-      "rediscover": "Rediscover feed",
-      "refresh": "Refresh now",
-      "status": {
-        "active": "Active",
-        "error": "Error",
-        "stale": "Stale"
-      }
+      "edit": "Edit feed"
     },
     "item": {
       "reply": "Reply",
@@ -75,6 +61,14 @@
       "excludeRegex": "Exclude pattern",
       "excludeRegexHelp": "Regular expression to filter out matching content",
       "save": "Save settings",
+      "retentionTitle": "Retention",
+      "retentionHelp": "Control how many items this channel keeps in MongoDB. Useful for high-volume aggregator channels that would otherwise grow without bound. Leave any field blank to use the global default.",
+      "maxItems": "Maximum items in channel",
+      "maxItemsHelp": "Keep at most this many items total. Oldest items are deleted regardless of read state. Default: {{default}}.",
+      "maxItemsPerFeed": "Maximum items per feed",
+      "maxItemsPerFeedHelp": "Keep at most this many items per feed inside this channel. Prevents one prolific feed from monopolising the channel cap. Default: {{default}}.",
+      "maxUnreadAgeDays": "Drop unread items older than (days)",
+      "maxUnreadAgeDaysHelp": "Unread items older than this are deleted, even if the channel is below its item cap. Default: {{default}}.",
       "dangerZone": "Danger zone",
       "deleteWarning": "Deleting this channel will permanently remove all feeds and items. This action cannot be undone.",
       "deleteConfirm": "Are you sure you want to delete this channel and all its content?",
@@ -93,20 +87,10 @@
       "submit": "Search",
       "noResults": "No results found"
     },
-    "preview": {
-      "title": "Preview",
-      "subscribe": "Subscribe to this feed"
-    },
     "views": {
       "channels": "Channels",
       "deck": "Deck",
       "timeline": "Timeline"
-    },
-    "error": {
-      "channelNotFound": "Channel not found",
-      "feedNotFound": "Feed not found",
-      "invalidUrl": "Invalid URL",
-      "invalidAction": "Invalid action"
     }
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-microsub",
-  "version": "1.0.61",
+  "version": "1.0.64",
   "description": "Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the Microsub protocol.",
   "keywords": [
     "indiekit",