@rmdes/indiekit-endpoint-microsub 1.0.30 → 1.0.31

package/lib/storage/channels.js

@@ -115,6 +115,7 @@ export async function getChannels(application, userId) {
         channelId: channel._id,
         readBy: { $ne: userId },
         published: { $gte: cutoffDate },
+        _stripped: { $ne: true },
       });
 
       return {

package/lib/storage/items.js

@@ -87,8 +87,9 @@ export async function getTimelineItems(application, channelId, options = {}) {
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
   const limit = parseLimit(options.limit);
 
-  // Base query - filter out read items unless showRead is true
-  const baseQuery = { channelId: objectId };
+  // Base query - filter out read items unless showRead is true,
+  // and always exclude stripped dedup skeletons (no content to display)
+  const baseQuery = { channelId: objectId, _stripped: { $ne: true } };
   if (options.userId && !options.showRead) {
     baseQuery.readBy = { $ne: options.userId };
   }
@@ -288,61 +289,83 @@ export async function countReadItems(application, channelId, userId) {
  * @param {string} userId - User ID
  * @returns {Promise<number>} Number of items updated
  */
-// Maximum number of read items to keep per channel
-const MAX_READ_ITEMS = 30;
+// Maximum number of full read items to keep per channel before stripping content.
+// Items beyond this limit are converted to lightweight dedup skeletons (channelId,
+// uid, readBy) so the poller doesn't re-ingest them as new unread entries.
+const MAX_FULL_READ_ITEMS = 200;
 
 /**
- * Cleanup old read items, keeping only the most recent MAX_READ_ITEMS
+ * Cleanup old read items by stripping content but preserving dedup skeletons.
+ * This prevents the vicious cycle where deleted read items get re-ingested as
+ * unread by the poller because the dedup record (channelId + uid) was destroyed.
  * @param {object} collection - MongoDB collection
  * @param {ObjectId} channelObjectId - Channel ObjectId
  * @param {string} userId - User ID
  */
 async function cleanupOldReadItems(collection, channelObjectId, userId) {
-  // Count read items in this channel
   const readCount = await collection.countDocuments({
     channelId: channelObjectId,
     readBy: userId,
   });
 
-  if (readCount > MAX_READ_ITEMS) {
-    // Find the oldest read items to delete
-    const itemsToDelete = await collection
+  if (readCount > MAX_FULL_READ_ITEMS) {
+    // Find old read items beyond the retention limit
+    const itemsToStrip = await collection
       .find({
         channelId: channelObjectId,
         readBy: userId,
+        _stripped: { $ne: true }, // Don't re-strip already-stripped items
       })
-      .sort({ published: -1, _id: -1 }) // Newest first
-      .skip(MAX_READ_ITEMS) // Skip the ones we want to keep
+      .sort({ published: -1, _id: -1 })
+      .skip(MAX_FULL_READ_ITEMS)
       .project({ _id: 1 })
       .toArray();
 
-    if (itemsToDelete.length > 0) {
-      const idsToDelete = itemsToDelete.map((item) => item._id);
-      const deleteResult = await collection.deleteMany({
-        _id: { $in: idsToDelete },
-      });
+    if (itemsToStrip.length > 0) {
+      const idsToStrip = itemsToStrip.map((item) => item._id);
+      // Strip content but keep dedup skeleton (channelId, uid, feedId, readBy)
+      const result = await collection.updateMany(
+        { _id: { $in: idsToStrip } },
+        {
+          $set: { _stripped: true },
+          $unset: {
+            name: "",
+            content: "",
+            summary: "",
+            author: "",
+            category: "",
+            photo: "",
+            video: "",
+            audio: "",
+            likeOf: "",
+            repostOf: "",
+            bookmarkOf: "",
+            inReplyTo: "",
+            source: "",
+          },
+        },
+      );
       console.info(
-        `[Microsub] Cleaned up ${deleteResult.deletedCount} old read items (keeping ${MAX_READ_ITEMS})`,
+        `[Microsub] Stripped content from ${result.modifiedCount} old read items (keeping ${MAX_FULL_READ_ITEMS} full)`,
       );
     }
   }
 }
 
 /**
- * Cleanup all read items across all channels (startup cleanup)
+ * Cleanup all read items across all channels (startup cleanup).
+ * Strips content from old read items but preserves dedup skeletons.
  * @param {object} application - Indiekit application
- * @returns {Promise<number>} Total number of items deleted
+ * @returns {Promise<number>} Total number of items stripped
  */
 export async function cleanupAllReadItems(application) {
   const collection = getCollection(application);
   const channelsCollection = application.collections.get("microsub_channels");
 
-  // Get all channels
   const channels = await channelsCollection.find({}).toArray();
-  let totalDeleted = 0;
+  let totalStripped = 0;
 
   for (const channel of channels) {
-    // Get unique userIds who have read items in this channel
     const readByUsers = await collection.distinct("readBy", {
       channelId: channel._id,
       readBy: { $exists: true, $ne: [] },
@@ -354,40 +377,60 @@ export async function cleanupAllReadItems(application) {
       const readCount = await collection.countDocuments({
         channelId: channel._id,
         readBy: userId,
+        _stripped: { $ne: true },
       });
 
-      if (readCount > MAX_READ_ITEMS) {
-        const itemsToDelete = await collection
+      if (readCount > MAX_FULL_READ_ITEMS) {
+        const itemsToStrip = await collection
           .find({
             channelId: channel._id,
             readBy: userId,
+            _stripped: { $ne: true },
           })
           .sort({ published: -1, _id: -1 })
-          .skip(MAX_READ_ITEMS)
+          .skip(MAX_FULL_READ_ITEMS)
           .project({ _id: 1 })
           .toArray();
 
-        if (itemsToDelete.length > 0) {
-          const idsToDelete = itemsToDelete.map((item) => item._id);
-          const deleteResult = await collection.deleteMany({
-            _id: { $in: idsToDelete },
-          });
-          totalDeleted += deleteResult.deletedCount;
+        if (itemsToStrip.length > 0) {
+          const idsToStrip = itemsToStrip.map((item) => item._id);
+          const result = await collection.updateMany(
+            { _id: { $in: idsToStrip } },
+            {
+              $set: { _stripped: true },
+              $unset: {
+                name: "",
+                content: "",
+                summary: "",
+                author: "",
+                category: "",
+                photo: "",
+                video: "",
+                audio: "",
+                likeOf: "",
+                repostOf: "",
+                bookmarkOf: "",
+                inReplyTo: "",
+                source: "",
+              },
+            },
+          );
+          totalStripped += result.modifiedCount;
           console.info(
-            `[Microsub] Startup cleanup: deleted ${deleteResult.deletedCount} old items from channel "${channel.name}"`,
+            `[Microsub] Startup cleanup: stripped ${result.modifiedCount} old items from channel "${channel.name}"`,
           );
         }
       }
     }
   }
 
-  if (totalDeleted > 0) {
+  if (totalStripped > 0) {
     console.info(
-      `[Microsub] Startup cleanup complete: ${totalDeleted} total items deleted`,
+      `[Microsub] Startup cleanup complete: ${totalStripped} total items stripped`,
     );
   }
 
-  return totalDeleted;
+  return totalStripped;
 }
 
 export async function markItemsRead(application, channelId, entryIds, userId) {
@@ -446,9 +489,6 @@ export async function markItemsRead(application, channelId, entryIds, userId) {
     `[Microsub] markItemsRead result: ${result.modifiedCount} items updated`,
   );
 
-  // Cleanup old read items, keeping only the most recent
-  await cleanupOldReadItems(collection, channelObjectId, userId);
-
   return result.modifiedCount;
 }
 
@@ -577,7 +617,7 @@ export async function getUnreadCount(application, channelId, userId) {
   const objectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
-  // Only count items from the last UNREAD_RETENTION_DAYS
+  // Only count items from the last UNREAD_RETENTION_DAYS, exclude stripped skeletons
   const cutoffDate = new Date();
   cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
 
@@ -585,6 +625,7 @@ export async function getUnreadCount(application, channelId, userId) {
     channelId: objectId,
     readBy: { $ne: userId },
     published: { $gte: cutoffDate },
+    _stripped: { $ne: true },
   });
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-microsub",
-  "version": "1.0.30",
+  "version": "1.0.31",
   "description": "Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the Microsub protocol.",
   "keywords": [
     "indiekit",