@rmdes/indiekit-endpoint-microsub 1.0.56 → 1.0.57

package/lib/storage/items.js

@@ -1,10 +1,11 @@
 /**
- * Timeline item storage operations
+ * Timeline item storage operations — core CRUD
  * @module storage/items
  */
 
 import { ObjectId } from "mongodb";
 
+import { extractImagesFromHtml } from "../utils/html.js";
 import {
   buildPaginationQuery,
   buildPaginationSort,
@@ -13,33 +14,111 @@ import {
 } from "../utils/pagination.js";
 
 /**
- * Extract image URLs from HTML content (fallback for items without explicit photos)
- * @param {string} html - HTML content
- * @returns {string[]} Array of image URLs
+ * Get items collection from application
+ * @param {object} application - Indiekit application
+ * @returns {object} MongoDB collection
+ */
+export function getCollection(application) {
+  return application.collections.get("microsub_items");
+}
+
+/**
+ * Extract URL string from a media value
+ * @param {object|string} media - Media value (can be string URL or object)
+ * @returns {string|undefined} URL string
  */
-function extractImagesFromHtml(html) {
-  if (!html) {
+function extractMediaUrl(media) {
+  if (!media) {
+    return;
+  }
+  if (typeof media === "string") {
+    return media;
+  }
+  if (typeof media === "object") {
+    return media.value || media.url || media.src;
+  }
+}
+
+/**
+ * Normalize media array to URL strings
+ * @param {Array} mediaArray - Array of media items
+ * @returns {Array} Array of URL strings
+ */
+function normalizeMediaArray(mediaArray) {
+  if (!mediaArray || !Array.isArray(mediaArray)) {
     return [];
   }
-  const urls = [];
-  const imgRegex = /<img[^>]+src=["']([^"']+)["'][^>]*>/gi;
-  let match;
-  while ((match = imgRegex.exec(html)) !== null) {
-    const src = match[1];
-    if (src && !urls.includes(src)) {
-      urls.push(src);
-    }
+  return mediaArray.map((media) => extractMediaUrl(media)).filter(Boolean);
+}
+
+/**
+ * Normalize author object to ensure photo is a URL string
+ * @param {object} author - Author object
+ * @returns {object} Normalized author
+ */
+function normalizeAuthor(author) {
+  if (!author) {
+    return;
   }
-  return urls;
+  return {
+    ...author,
+    photo: extractMediaUrl(author.photo),
+  };
 }
 
 /**
- * Get items collection from application
- * @param {object} application - Indiekit application
- * @returns {object} MongoDB collection
+ * Transform database item to jf2 format
+ * @param {object} item - Database item
+ * @param {string} [userId] - User ID for read state
+ * @returns {object} jf2 item
  */
-function getCollection(application) {
-  return application.collections.get("microsub_items");
+export function transformToJf2(item, userId) {
+  const jf2 = {
+    type: item.type,
+    uid: item.uid,
+    url: item.url,
+    published: item.published?.toISOString(), // Convert Date to ISO string
+    _id: item._id.toString(),
+    _is_read: userId ? item.readBy?.includes(userId) : false,
+    _channelId: item.channelId?.toString(),
+    _feedId: item.feedId?.toString(),
+  };
+
+  // Optional fields
+  if (item.name) jf2.name = item.name;
+  if (item.content) jf2.content = item.content;
+  if (item.summary) jf2.summary = item.summary;
+  if (item.updated) jf2.updated = item.updated.toISOString(); // Convert Date to ISO string
+  if (item.author) jf2.author = normalizeAuthor(item.author);
+  if (item.category?.length > 0) jf2.category = item.category;
+
+  // Normalize media arrays to ensure they contain URL strings
+  const photos = normalizeMediaArray(item.photo);
+  const videos = normalizeMediaArray(item.video);
+  const audios = normalizeMediaArray(item.audio);
+
+  // Fallback: extract images from HTML content if no explicit photos
+  if (photos.length === 0 && item.content?.html) {
+    const extracted = extractImagesFromHtml(item.content.html);
+    if (extracted.length > 0) {
+      photos.push(...extracted);
+    }
+  }
+
+  if (photos.length > 0) jf2.photo = photos;
+  if (videos.length > 0) jf2.video = videos;
+  if (audios.length > 0) jf2.audio = audios;
+
+  // Interaction types
+  if (item.likeOf?.length > 0) jf2["like-of"] = item.likeOf;
+  if (item.repostOf?.length > 0) jf2["repost-of"] = item.repostOf;
+  if (item.bookmarkOf?.length > 0) jf2["bookmark-of"] = item.bookmarkOf;
+  if (item.inReplyTo?.length > 0) jf2["in-reply-to"] = item.inReplyTo;
+
+  // Source
+  if (item.source) jf2._source = item.source;
+
+  return jf2;
 }
 
 /**
@@ -212,105 +291,6 @@ export async function getAllTimelineItems(application, options = {}) {
   };
 }
 
-/**
- * Extract URL string from a media value
- * @param {object|string} media - Media value (can be string URL or object)
- * @returns {string|undefined} URL string
- */
-function extractMediaUrl(media) {
-  if (!media) {
-    return;
-  }
-  if (typeof media === "string") {
-    return media;
-  }
-  if (typeof media === "object") {
-    return media.value || media.url || media.src;
-  }
-}
-
-/**
- * Normalize media array to URL strings
- * @param {Array} mediaArray - Array of media items
- * @returns {Array} Array of URL strings
- */
-function normalizeMediaArray(mediaArray) {
-  if (!mediaArray || !Array.isArray(mediaArray)) {
-    return [];
-  }
-  return mediaArray.map((media) => extractMediaUrl(media)).filter(Boolean);
-}
-
-/**
- * Normalize author object to ensure photo is a URL string
- * @param {object} author - Author object
- * @returns {object} Normalized author
- */
-function normalizeAuthor(author) {
-  if (!author) {
-    return;
-  }
-  return {
-    ...author,
-    photo: extractMediaUrl(author.photo),
-  };
-}
-
-/**
- * Transform database item to jf2 format
- * @param {object} item - Database item
- * @param {string} [userId] - User ID for read state
- * @returns {object} jf2 item
- */
-function transformToJf2(item, userId) {
-  const jf2 = {
-    type: item.type,
-    uid: item.uid,
-    url: item.url,
-    published: item.published?.toISOString(), // Convert Date to ISO string
-    _id: item._id.toString(),
-    _is_read: userId ? item.readBy?.includes(userId) : false,
-    _channelId: item.channelId?.toString(),
-    _feedId: item.feedId?.toString(),
-  };
-
-  // Optional fields
-  if (item.name) jf2.name = item.name;
-  if (item.content) jf2.content = item.content;
-  if (item.summary) jf2.summary = item.summary;
-  if (item.updated) jf2.updated = item.updated.toISOString(); // Convert Date to ISO string
-  if (item.author) jf2.author = normalizeAuthor(item.author);
-  if (item.category?.length > 0) jf2.category = item.category;
-
-  // Normalize media arrays to ensure they contain URL strings
-  const photos = normalizeMediaArray(item.photo);
-  const videos = normalizeMediaArray(item.video);
-  const audios = normalizeMediaArray(item.audio);
-
-  // Fallback: extract images from HTML content if no explicit photos
-  if (photos.length === 0 && item.content?.html) {
-    const extracted = extractImagesFromHtml(item.content.html);
-    if (extracted.length > 0) {
-      photos.push(...extracted);
-    }
-  }
-
-  if (photos.length > 0) jf2.photo = photos;
-  if (videos.length > 0) jf2.video = videos;
-  if (audios.length > 0) jf2.audio = audios;
-
-  // Interaction types
-  if (item.likeOf?.length > 0) jf2["like-of"] = item.likeOf;
-  if (item.repostOf?.length > 0) jf2["repost-of"] = item.repostOf;
-  if (item.bookmarkOf?.length > 0) jf2["bookmark-of"] = item.bookmarkOf;
-  if (item.inReplyTo?.length > 0) jf2["in-reply-to"] = item.inReplyTo;
-
-  // Source
-  if (item.source) jf2._source = item.source;
-
-  return jf2;
-}
-
 /**
  * Get an item by ID (MongoDB _id or uid)
  * @param {object} application - Indiekit application
@@ -357,425 +337,6 @@ export async function getItemsByUids(application, uids, userId) {
   return items.map((item) => transformToJf2(item, userId));
 }
 
-/**
- * Count read items in a channel
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {string} userId - User ID
- * @returns {Promise<number>} Number of read items
- */
-export async function countReadItems(application, channelId, userId) {
-  const collection = getCollection(application);
-  const objectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  return collection.countDocuments({
-    channelId: objectId,
-    readBy: userId,
-  });
-}
-
-/**
- * Mark items as read
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {Array} entryIds - Array of entry IDs to mark as read (can be ObjectId, uid, or URL)
- * @param {string} userId - User ID
- * @returns {Promise<number>} Number of items updated
- */
-// Maximum number of full read items to keep per channel before stripping content.
-// Items beyond this limit are converted to lightweight dedup skeletons (channelId,
-// uid, readBy) so the poller doesn't re-ingest them as new unread entries.
-const MAX_FULL_READ_ITEMS = 200;
-
-// Maximum age (in days) for stripped skeletons and unread items.
-// After this period, both are hard-deleted to prevent unbounded growth.
-const MAX_ITEM_AGE_DAYS = 30;
-
-/**
- * Cleanup old read items by stripping content but preserving dedup skeletons.
- * This prevents the vicious cycle where deleted read items get re-ingested as
- * unread by the poller because the dedup record (channelId + uid) was destroyed.
- *
- * AP items (feedId: null) are hard-deleted instead of stripped, since no poller
- * re-ingests them — they arrive via inbox push and don't need dedup skeletons.
- *
- * @param {object} collection - MongoDB collection
- * @param {ObjectId} channelObjectId - Channel ObjectId
- * @param {string} userId - User ID
- */
-async function cleanupOldReadItems(collection, channelObjectId, userId) {
-  const readCount = await collection.countDocuments({
-    channelId: channelObjectId,
-    readBy: userId,
-  });
-
-  if (readCount > MAX_FULL_READ_ITEMS) {
-    // Find old read items beyond the retention limit
-    const itemsToCleanup = await collection
-      .find({
-        channelId: channelObjectId,
-        readBy: userId,
-        _stripped: { $ne: true },
-      })
-      .sort({ published: -1, _id: -1 })
-      .skip(MAX_FULL_READ_ITEMS)
-      .project({ _id: 1, feedId: 1 })
-      .toArray();
-
-    if (itemsToCleanup.length === 0) return;
-
-    // Separate AP items (feedId: null) from RSS items (feedId: ObjectId)
-    const apItemIds = [];
-    const rssItemIds = [];
-    for (const item of itemsToCleanup) {
-      if (item.feedId) {
-        rssItemIds.push(item._id);
-      } else {
-        apItemIds.push(item._id);
-      }
-    }
-
-    // Hard-delete AP items — no poller to re-ingest, skeletons are useless
-    if (apItemIds.length > 0) {
-      const deleted = await collection.deleteMany({
-        _id: { $in: apItemIds },
-      });
-      console.info(
-        `[Microsub] Deleted ${deleted.deletedCount} old AP read items`,
-      );
-    }
-
-    // Strip RSS items to dedup skeletons — poller would re-ingest if deleted
-    if (rssItemIds.length > 0) {
-      const stripped = await collection.updateMany(
-        { _id: { $in: rssItemIds } },
-        {
-          $set: { _stripped: true },
-          $unset: {
-            name: "",
-            content: "",
-            summary: "",
-            author: "",
-            category: "",
-            photo: "",
-            video: "",
-            audio: "",
-            likeOf: "",
-            repostOf: "",
-            bookmarkOf: "",
-            inReplyTo: "",
-            source: "",
-          },
-        },
-      );
-      console.info(
-        `[Microsub] Stripped ${stripped.modifiedCount} old RSS read items (keeping ${MAX_FULL_READ_ITEMS} full)`,
-      );
-    }
-  }
-}
-
-/**
- * Cleanup all read items across all channels (startup cleanup).
- * RSS items are stripped to dedup skeletons; AP items are hard-deleted.
- * @param {object} application - Indiekit application
- * @returns {Promise<number>} Total number of items cleaned up
- */
-export async function cleanupAllReadItems(application) {
-  const collection = getCollection(application);
-  const channelsCollection = application.collections.get("microsub_channels");
-
-  const channels = await channelsCollection.find({}).toArray();
-  let totalCleaned = 0;
-
-  for (const channel of channels) {
-    const readByUsers = await collection.distinct("readBy", {
-      channelId: channel._id,
-      readBy: { $exists: true, $ne: [] },
-    });
-
-    for (const userId of readByUsers) {
-      if (!userId) continue;
-
-      const readCount = await collection.countDocuments({
-        channelId: channel._id,
-        readBy: userId,
-        _stripped: { $ne: true },
-      });
-
-      if (readCount > MAX_FULL_READ_ITEMS) {
-        const itemsToCleanup = await collection
-          .find({
-            channelId: channel._id,
-            readBy: userId,
-            _stripped: { $ne: true },
-          })
-          .sort({ published: -1, _id: -1 })
-          .skip(MAX_FULL_READ_ITEMS)
-          .project({ _id: 1, feedId: 1 })
-          .toArray();
-
-        if (itemsToCleanup.length > 0) {
-          const apItemIds = [];
-          const rssItemIds = [];
-          for (const item of itemsToCleanup) {
-            if (item.feedId) {
-              rssItemIds.push(item._id);
-            } else {
-              apItemIds.push(item._id);
-            }
-          }
-
-          // Hard-delete AP items
-          if (apItemIds.length > 0) {
-            const deleted = await collection.deleteMany({
-              _id: { $in: apItemIds },
-            });
-            totalCleaned += deleted.deletedCount;
-            console.info(
-              `[Microsub] Startup cleanup: deleted ${deleted.deletedCount} AP items from channel "${channel.name}"`,
-            );
-          }
-
-          // Strip RSS items to skeletons
-          if (rssItemIds.length > 0) {
-            const stripped = await collection.updateMany(
-              { _id: { $in: rssItemIds } },
-              {
-                $set: { _stripped: true },
-                $unset: {
-                  name: "",
-                  content: "",
-                  summary: "",
-                  author: "",
-                  category: "",
-                  photo: "",
-                  video: "",
-                  audio: "",
-                  likeOf: "",
-                  repostOf: "",
-                  bookmarkOf: "",
-                  inReplyTo: "",
-                  source: "",
-                },
-              },
-            );
-            totalCleaned += stripped.modifiedCount;
-            console.info(
-              `[Microsub] Startup cleanup: stripped ${stripped.modifiedCount} RSS items from channel "${channel.name}"`,
-            );
-          }
-        }
-      }
-    }
-  }
-
-  if (totalCleaned > 0) {
-    console.info(
-      `[Microsub] Startup cleanup complete: ${totalCleaned} total items cleaned`,
-    );
-  }
-
-  return totalCleaned;
-}
-
-/**
- * Delete stale items: stripped skeletons and unread items older than MAX_ITEM_AGE_DAYS.
- * Stripped skeletons have served their dedup purpose; stale unread items are unlikely
- * to be read. Both are hard-deleted to prevent unbounded collection growth.
- * @param {object} application - Indiekit application
- * @returns {Promise<number>} Total number of items deleted
- */
-export async function cleanupStaleItems(application) {
-  const collection = getCollection(application);
-  const cutoff = new Date();
-  cutoff.setDate(cutoff.getDate() - MAX_ITEM_AGE_DAYS);
-
-  // Delete stripped skeletons older than cutoff
-  const strippedResult = await collection.deleteMany({
-    _stripped: true,
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
-
-  // Delete unread items older than cutoff
-  const unreadResult = await collection.deleteMany({
-    readBy: { $in: [null, []] },
-    _stripped: { $ne: true },
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
-
-  // Also catch items with no readBy field at all
-  const noReadByResult = await collection.deleteMany({
-    readBy: { $exists: false },
-    _stripped: { $ne: true },
-    $or: [
-      { published: { $lt: cutoff } },
-      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
-    ],
-  });
-
-  const total =
-    strippedResult.deletedCount +
-    unreadResult.deletedCount +
-    noReadByResult.deletedCount;
-
-  if (total > 0) {
-    console.info(
-      `[Microsub] Stale cleanup: deleted ${strippedResult.deletedCount} stripped skeletons, ` +
-        `${unreadResult.deletedCount + noReadByResult.deletedCount} stale unread items ` +
-        `(cutoff: ${MAX_ITEM_AGE_DAYS} days)`,
-    );
-  }
-
-  return total;
-}
-
-export async function markItemsRead(application, channelId, entryIds, userId) {
-  const collection = getCollection(application);
-  const channelObjectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  console.info(
-    `[Microsub] markItemsRead called for channel ${channelId}, entries:`,
-    entryIds,
-    `userId: ${userId}`,
-  );
-
-  // Handle "last-read-entry" special value
-  if (entryIds.includes("last-read-entry")) {
-    // Mark all items in channel as read
-    const result = await collection.updateMany(
-      { channelId: channelObjectId },
-      { $addToSet: { readBy: userId } },
-    );
-    console.info(
-      `[Microsub] Marked all items as read: ${result.modifiedCount} updated`,
-    );
-
-    // Cleanup old read items, keeping only the most recent
-    await cleanupOldReadItems(collection, channelObjectId, userId);
-
-    return result.modifiedCount;
-  }
-
-  // Convert string IDs to ObjectIds where possible
-  const objectIds = entryIds
-    .map((id) => {
-      try {
-        return new ObjectId(id);
-      } catch {
-        return;
-      }
-    })
-    .filter(Boolean);
-
-  // Build query to match by _id, uid, or url (Microsub spec uses URLs as entry identifiers)
-  const result = await collection.updateMany(
-    {
-      channelId: channelObjectId,
-      $or: [
-        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
-        { uid: { $in: entryIds } },
-        { url: { $in: entryIds } },
-      ],
-    },
-    { $addToSet: { readBy: userId } },
-  );
-
-  console.info(
-    `[Microsub] markItemsRead result: ${result.modifiedCount} items updated`,
-  );
-
-  return result.modifiedCount;
-}
-
-/**
- * Mark all items from a specific feed as read in a channel
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {ObjectId|string} feedId - Feed ObjectId
- * @param {string} userId - User ID
- * @returns {Promise<number>} Number of items updated
- */
-export async function markFeedItemsRead(
-  application,
-  channelId,
-  feedId,
-  userId,
-) {
-  const collection = getCollection(application);
-  const channelObjectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-  const feedObjectId =
-    typeof feedId === "string" ? new ObjectId(feedId) : feedId;
-
-  const result = await collection.updateMany(
-    { channelId: channelObjectId, feedId: feedObjectId },
-    { $addToSet: { readBy: userId } },
-  );
-
-  console.info(
-    `[Microsub] markFeedItemsRead: marked ${result.modifiedCount} items from feed ${feedId} as read`,
-  );
-
-  // Cleanup old read items
-  await cleanupOldReadItems(collection, channelObjectId, userId);
-
-  return result.modifiedCount;
-}
-
-/**
- * Mark items as unread
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {Array} entryIds - Array of entry IDs to mark as unread (can be ObjectId, uid, or URL)
- * @param {string} userId - User ID
- * @returns {Promise<number>} Number of items updated
- */
-export async function markItemsUnread(
-  application,
-  channelId,
-  entryIds,
-  userId,
-) {
-  const collection = getCollection(application);
-  const channelObjectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  // Convert string IDs to ObjectIds where possible
-  const objectIds = entryIds
-    .map((id) => {
-      try {
-        return new ObjectId(id);
-      } catch {
-        return;
-      }
-    })
-    .filter(Boolean);
-
-  // Match by _id, uid, or url
-  const result = await collection.updateMany(
-    {
-      channelId: channelObjectId,
-      $or: [
-        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
-        { uid: { $in: entryIds } },
-        { url: { $in: entryIds } },
-      ],
-    },
-    { $pull: { readBy: userId } },
-  );
-
-  return result.modifiedCount;
-}
-
 /**
  * Remove items from channel
  * @param {object} application - Indiekit application
@@ -841,58 +402,6 @@ export async function deleteItemsForFeed(application, feedId) {
   return result.deletedCount;
 }
 
-import { UNREAD_RETENTION_DAYS } from "../utils/constants.js";
-
-/**
- * Get unread count for a channel
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {string} userId - User ID
- * @returns {Promise<number>} Unread count
- */
-export async function getUnreadCount(application, channelId, userId) {
-  const collection = getCollection(application);
-  const objectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  // Only count items from the last UNREAD_RETENTION_DAYS, exclude stripped skeletons
-  const cutoffDate = new Date();
-  cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
-
-  return collection.countDocuments({
-    channelId: objectId,
-    readBy: { $ne: userId },
-    published: { $gte: cutoffDate },
-    _stripped: { $ne: true },
-  });
-}
-
-/**
- * Search items by text
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {string} query - Search query
- * @param {number} [limit] - Max results
- * @returns {Promise<Array>} Array of matching items
- */
-export async function searchItems(application, channelId, query, limit = 20) {
-  const collection = getCollection(application);
-  const objectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  // Use MongoDB text index for efficient full-text search
-  const items = await collection
-    .find({
-      channelId: objectId,
-      $text: { $search: query },
-    })
-    .sort({ score: { $meta: "textScore" } })
-    .limit(limit)
-    .toArray();
-
-  return items.map((item) => transformToJf2(item));
-}
-
 /**
  * Delete items by author URL (for blocking)
  * @param {object} application - Indiekit application