@rmdes/indiekit-endpoint-microsub 1.0.55 → 1.0.57

package/lib/storage/items-read-state.js

@@ -0,0 +1,287 @@
+/**
+ * Timeline item read state management
+ * @module storage/items-read-state
+ */
+
+import { ObjectId } from "mongodb";
+
+import { UNREAD_RETENTION_DAYS } from "../utils/constants.js";
+import { getCollection } from "./items.js";
+
+// Maximum number of full read items to keep per channel before stripping content.
+// Items beyond this limit are converted to lightweight dedup skeletons (channelId,
+// uid, readBy) so the poller doesn't re-ingest them as new unread entries.
+const MAX_FULL_READ_ITEMS = 200;
+
+/**
+ * Cleanup old read items by stripping content but preserving dedup skeletons.
+ * This prevents the vicious cycle where deleted read items get re-ingested as
+ * unread by the poller because the dedup record (channelId + uid) was destroyed.
+ *
+ * AP items (feedId: null) are hard-deleted instead of stripped, since no poller
+ * re-ingests them — they arrive via inbox push and don't need dedup skeletons.
+ *
+ * @param {object} collection - MongoDB collection
+ * @param {ObjectId} channelObjectId - Channel ObjectId
+ * @param {string} userId - User ID
+ */
+async function cleanupOldReadItems(collection, channelObjectId, userId) {
+  const readCount = await collection.countDocuments({
+    channelId: channelObjectId,
+    readBy: userId,
+  });
+
+  if (readCount > MAX_FULL_READ_ITEMS) {
+    // Find old read items beyond the retention limit
+    const itemsToCleanup = await collection
+      .find({
+        channelId: channelObjectId,
+        readBy: userId,
+        _stripped: { $ne: true },
+      })
+      .sort({ published: -1, _id: -1 })
+      .skip(MAX_FULL_READ_ITEMS)
+      .project({ _id: 1, feedId: 1 })
+      .toArray();
+
+    if (itemsToCleanup.length === 0) return;
+
+    // Separate AP items (feedId: null) from RSS items (feedId: ObjectId)
+    const apItemIds = [];
+    const rssItemIds = [];
+    for (const item of itemsToCleanup) {
+      if (item.feedId) {
+        rssItemIds.push(item._id);
+      } else {
+        apItemIds.push(item._id);
+      }
+    }
+
+    // Hard-delete AP items — no poller to re-ingest, skeletons are useless
+    if (apItemIds.length > 0) {
+      const deleted = await collection.deleteMany({
+        _id: { $in: apItemIds },
+      });
+      console.info(
+        `[Microsub] Deleted ${deleted.deletedCount} old AP read items`,
+      );
+    }
+
+    // Strip RSS items to dedup skeletons — poller would re-ingest if deleted
+    if (rssItemIds.length > 0) {
+      const stripped = await collection.updateMany(
+        { _id: { $in: rssItemIds } },
+        {
+          $set: { _stripped: true },
+          $unset: {
+            name: "",
+            content: "",
+            summary: "",
+            author: "",
+            category: "",
+            photo: "",
+            video: "",
+            audio: "",
+            likeOf: "",
+            repostOf: "",
+            bookmarkOf: "",
+            inReplyTo: "",
+            source: "",
+          },
+        },
+      );
+      console.info(
+        `[Microsub] Stripped ${stripped.modifiedCount} old RSS read items (keeping ${MAX_FULL_READ_ITEMS} full)`,
+      );
+    }
+  }
+}
+
+/**
+ * Mark items as read
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {Array} entryIds - Array of entry IDs to mark as read (can be ObjectId, uid, or URL)
+ * @param {string} userId - User ID
+ * @returns {Promise<number>} Number of items updated
+ */
+export async function markItemsRead(application, channelId, entryIds, userId) {
+  const collection = getCollection(application);
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  console.info(
+    `[Microsub] markItemsRead called for channel ${channelId}, entries:`,
+    entryIds,
+    `userId: ${userId}`,
+  );
+
+  // Handle "last-read-entry" special value
+  if (entryIds.includes("last-read-entry")) {
+    // Mark all items in channel as read
+    const result = await collection.updateMany(
+      { channelId: channelObjectId },
+      { $addToSet: { readBy: userId } },
+    );
+    console.info(
+      `[Microsub] Marked all items as read: ${result.modifiedCount} updated`,
+    );
+
+    // Cleanup old read items, keeping only the most recent
+    await cleanupOldReadItems(collection, channelObjectId, userId);
+
+    return result.modifiedCount;
+  }
+
+  // Convert string IDs to ObjectIds where possible
+  const objectIds = entryIds
+    .map((id) => {
+      try {
+        return new ObjectId(id);
+      } catch {
+        return;
+      }
+    })
+    .filter(Boolean);
+
+  // Build query to match by _id, uid, or url (Microsub spec uses URLs as entry identifiers)
+  const result = await collection.updateMany(
+    {
+      channelId: channelObjectId,
+      $or: [
+        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
+        { uid: { $in: entryIds } },
+        { url: { $in: entryIds } },
+      ],
+    },
+    { $addToSet: { readBy: userId } },
+  );
+
+  console.info(
+    `[Microsub] markItemsRead result: ${result.modifiedCount} items updated`,
+  );
+
+  return result.modifiedCount;
+}
+
+/**
+ * Mark all items from a specific feed as read in a channel
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {ObjectId|string} feedId - Feed ObjectId
+ * @param {string} userId - User ID
+ * @returns {Promise<number>} Number of items updated
+ */
+export async function markFeedItemsRead(
+  application,
+  channelId,
+  feedId,
+  userId,
+) {
+  const collection = getCollection(application);
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+  const feedObjectId =
+    typeof feedId === "string" ? new ObjectId(feedId) : feedId;
+
+  const result = await collection.updateMany(
+    { channelId: channelObjectId, feedId: feedObjectId },
+    { $addToSet: { readBy: userId } },
+  );
+
+  console.info(
+    `[Microsub] markFeedItemsRead: marked ${result.modifiedCount} items from feed ${feedId} as read`,
+  );
+
+  // Cleanup old read items
+  await cleanupOldReadItems(collection, channelObjectId, userId);
+
+  return result.modifiedCount;
+}
+
+/**
+ * Mark items as unread
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {Array} entryIds - Array of entry IDs to mark as unread (can be ObjectId, uid, or URL)
+ * @param {string} userId - User ID
+ * @returns {Promise<number>} Number of items updated
+ */
+export async function markItemsUnread(
+  application,
+  channelId,
+  entryIds,
+  userId,
+) {
+  const collection = getCollection(application);
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  // Convert string IDs to ObjectIds where possible
+  const objectIds = entryIds
+    .map((id) => {
+      try {
+        return new ObjectId(id);
+      } catch {
+        return;
+      }
+    })
+    .filter(Boolean);
+
+  // Match by _id, uid, or url
+  const result = await collection.updateMany(
+    {
+      channelId: channelObjectId,
+      $or: [
+        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
+        { uid: { $in: entryIds } },
+        { url: { $in: entryIds } },
+      ],
+    },
+    { $pull: { readBy: userId } },
+  );
+
+  return result.modifiedCount;
+}
+
+/**
+ * Count read items in a channel
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} userId - User ID
+ * @returns {Promise<number>} Number of read items
+ */
+export async function countReadItems(application, channelId, userId) {
+  const collection = getCollection(application);
+  const objectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  return collection.countDocuments({
+    channelId: objectId,
+    readBy: userId,
+  });
+}
+
+/**
+ * Get unread count for a channel
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} userId - User ID
+ * @returns {Promise<number>} Unread count
+ */
+export async function getUnreadCount(application, channelId, userId) {
+  const collection = getCollection(application);
+  const objectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  // Only count items from the last UNREAD_RETENTION_DAYS, exclude stripped skeletons
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
+
+  return collection.countDocuments({
+    channelId: objectId,
+    readBy: { $ne: userId },
+    published: { $gte: cutoffDate },
+    _stripped: { $ne: true },
+  });
+}

package/lib/storage/items-retention.js

@@ -0,0 +1,174 @@
+/**
+ * Timeline item cleanup and retention
+ * @module storage/items-retention
+ */
+
+import { getCollection } from "./items.js";
+
+// Maximum number of full read items to keep per channel before stripping content.
+const MAX_FULL_READ_ITEMS = 200;
+
+// Maximum age (in days) for stripped skeletons and unread items.
+// After this period, both are hard-deleted to prevent unbounded growth.
+const MAX_ITEM_AGE_DAYS = 30;
+
+/**
+ * Cleanup all read items across all channels (startup cleanup).
+ * RSS items are stripped to dedup skeletons; AP items are hard-deleted.
+ * @param {object} application - Indiekit application
+ * @returns {Promise<number>} Total number of items cleaned up
+ */
+export async function cleanupAllReadItems(application) {
+  const collection = getCollection(application);
+  const channelsCollection = application.collections.get("microsub_channels");
+
+  const channels = await channelsCollection.find({}).toArray();
+  let totalCleaned = 0;
+
+  for (const channel of channels) {
+    const readByUsers = await collection.distinct("readBy", {
+      channelId: channel._id,
+      readBy: { $exists: true, $ne: [] },
+    });
+
+    for (const userId of readByUsers) {
+      if (!userId) continue;
+
+      const readCount = await collection.countDocuments({
+        channelId: channel._id,
+        readBy: userId,
+        _stripped: { $ne: true },
+      });
+
+      if (readCount > MAX_FULL_READ_ITEMS) {
+        const itemsToCleanup = await collection
+          .find({
+            channelId: channel._id,
+            readBy: userId,
+            _stripped: { $ne: true },
+          })
+          .sort({ published: -1, _id: -1 })
+          .skip(MAX_FULL_READ_ITEMS)
+          .project({ _id: 1, feedId: 1 })
+          .toArray();
+
+        if (itemsToCleanup.length > 0) {
+          const apItemIds = [];
+          const rssItemIds = [];
+          for (const item of itemsToCleanup) {
+            if (item.feedId) {
+              rssItemIds.push(item._id);
+            } else {
+              apItemIds.push(item._id);
+            }
+          }
+
+          // Hard-delete AP items
+          if (apItemIds.length > 0) {
+            const deleted = await collection.deleteMany({
+              _id: { $in: apItemIds },
+            });
+            totalCleaned += deleted.deletedCount;
+            console.info(
+              `[Microsub] Startup cleanup: deleted ${deleted.deletedCount} AP items from channel "${channel.name}"`,
+            );
+          }
+
+          // Strip RSS items to skeletons
+          if (rssItemIds.length > 0) {
+            const stripped = await collection.updateMany(
+              { _id: { $in: rssItemIds } },
+              {
+                $set: { _stripped: true },
+                $unset: {
+                  name: "",
+                  content: "",
+                  summary: "",
+                  author: "",
+                  category: "",
+                  photo: "",
+                  video: "",
+                  audio: "",
+                  likeOf: "",
+                  repostOf: "",
+                  bookmarkOf: "",
+                  inReplyTo: "",
+                  source: "",
+                },
+              },
+            );
+            totalCleaned += stripped.modifiedCount;
+            console.info(
+              `[Microsub] Startup cleanup: stripped ${stripped.modifiedCount} RSS items from channel "${channel.name}"`,
+            );
+          }
+        }
+      }
+    }
+  }
+
+  if (totalCleaned > 0) {
+    console.info(
+      `[Microsub] Startup cleanup complete: ${totalCleaned} total items cleaned`,
+    );
+  }
+
+  return totalCleaned;
+}
+
+/**
+ * Delete stale items: stripped skeletons and unread items older than MAX_ITEM_AGE_DAYS.
+ * Stripped skeletons have served their dedup purpose; stale unread items are unlikely
+ * to be read. Both are hard-deleted to prevent unbounded collection growth.
+ * @param {object} application - Indiekit application
+ * @returns {Promise<number>} Total number of items deleted
+ */
+export async function cleanupStaleItems(application) {
+  const collection = getCollection(application);
+  const cutoff = new Date();
+  cutoff.setDate(cutoff.getDate() - MAX_ITEM_AGE_DAYS);
+
+  // Delete stripped skeletons older than cutoff
+  const strippedResult = await collection.deleteMany({
+    _stripped: true,
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  // Delete unread items older than cutoff
+  const unreadResult = await collection.deleteMany({
+    readBy: { $in: [null, []] },
+    _stripped: { $ne: true },
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  // Also catch items with no readBy field at all
+  const noReadByResult = await collection.deleteMany({
+    readBy: { $exists: false },
+    _stripped: { $ne: true },
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  const total =
+    strippedResult.deletedCount +
+    unreadResult.deletedCount +
+    noReadByResult.deletedCount;
+
+  if (total > 0) {
+    console.info(
+      `[Microsub] Stale cleanup: deleted ${strippedResult.deletedCount} stripped skeletons, ` +
+        `${unreadResult.deletedCount + noReadByResult.deletedCount} stale unread items ` +
+        `(cutoff: ${MAX_ITEM_AGE_DAYS} days)`,
+    );
+  }
+
+  return total;
+}

package/lib/storage/items-search.js

@@ -0,0 +1,34 @@
+/**
+ * Timeline item search
+ * @module storage/items-search
+ */
+
+import { ObjectId } from "mongodb";
+
+import { getCollection, transformToJf2 } from "./items.js";
+
+/**
+ * Search items by text
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} query - Search query
+ * @param {number} [limit] - Max results
+ * @returns {Promise<Array>} Array of matching items
+ */
+export async function searchItems(application, channelId, query, limit = 20) {
+  const collection = getCollection(application);
+  const objectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  // Use MongoDB text index for efficient full-text search
+  const items = await collection
+    .find({
+      channelId: objectId,
+      $text: { $search: query },
+    })
+    .sort({ score: { $meta: "textScore" } })
+    .limit(limit)
+    .toArray();
+
+  return items.map((item) => transformToJf2(item));
+}