@rmdes/indiekit-endpoint-microsub 1.0.63 → 1.0.65

package/lib/storage/filters.js

@@ -1,117 +1,22 @@
 /**
- * Filter storage operations (mute, block, channel filters)
+ * Item filtering: exclude-types and exclude-regex predicates applied during
+ * feed ingestion. Used by `lib/polling/processor.js` when deciding whether a
+ * newly-parsed item should be stored.
+ *
+ * Historical note: this module previously also held mute/block storage
+ * (microsub_muted / microsub_blocked operations). That code path was abandoned
+ * — those collections are managed directly by `lib/controllers/mute.js` and
+ * `lib/controllers/block.js` against MongoDB without an intermediate storage
+ * helper layer. Only the per-channel exclude filters remain here.
+ *
  * @module storage/filters
  */
 
-import { ObjectId } from "mongodb";
-
-/**
- * Get muted collection
- * @param {object} application - Indiekit application
- * @returns {object} MongoDB collection
- */
-function getMutedCollection(application) {
-  return application.collections.get("microsub_muted");
-}
-
-/**
- * Get blocked collection
- * @param {object} application - Indiekit application
- * @returns {object} MongoDB collection
- */
-function getBlockedCollection(application) {
-  return application.collections.get("microsub_blocked");
-}
-
-/**
- * Check if a URL is muted for a user/channel
- * @param {object} application - Indiekit application
- * @param {string} userId - User ID
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {string} url - URL to check
- * @returns {Promise<boolean>} Whether the URL is muted
- */
-export async function isMuted(application, userId, channelId, url) {
-  const collection = getMutedCollection(application);
-  const channelObjectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  // Check for channel-specific mute
-  const channelMute = await collection.findOne({
-    userId,
-    channelId: channelObjectId,
-    url,
-  });
-  if (channelMute) return true;
-
-  // Check for global mute (no channelId)
-  const globalMute = await collection.findOne({
-    userId,
-    channelId: { $exists: false },
-    url,
-  });
-  return !!globalMute;
-}
-
-/**
- * Check if a URL is blocked for a user
- * @param {object} application - Indiekit application
- * @param {string} userId - User ID
- * @param {string} url - URL to check
- * @returns {Promise<boolean>} Whether the URL is blocked
- */
-export async function isBlocked(application, userId, url) {
-  const collection = getBlockedCollection(application);
-  const blocked = await collection.findOne({ userId, url });
-  return !!blocked;
-}
-
-/**
- * Check if an item passes all filters
- * @param {object} application - Indiekit application
- * @param {string} userId - User ID
- * @param {object} channel - Channel document with settings
- * @param {object} item - Feed item to check
- * @returns {Promise<boolean>} Whether the item passes all filters
- */
-export async function passesAllFilters(application, userId, channel, item) {
-  // Check if author URL is blocked
-  if (
-    item.author?.url &&
-    (await isBlocked(application, userId, item.author.url))
-  ) {
-    return false;
-  }
-
-  // Check if source URL is muted
-  if (
-    item._source?.url &&
-    (await isMuted(application, userId, channel._id, item._source.url))
-  ) {
-    return false;
-  }
-
-  // Check channel settings filters
-  if (channel?.settings) {
-    // Check excludeTypes
-    if (!passesTypeFilter(item, channel.settings)) {
-      return false;
-    }
-
-    // Check excludeRegex
-    if (!passesRegexFilter(item, channel.settings)) {
-      return false;
-    }
-  }
-
-  return true;
-}
-
 /**
- * Check if an item passes the excludeTypes filter
+ * Check if an item passes the channel.settings.excludeTypes filter.
  * @param {object} item - Feed item
  * @param {object} settings - Channel settings
- * @returns {boolean} Whether the item passes
+ * @returns {boolean} True when the item should be kept
  */
 export function passesTypeFilter(item, settings) {
   if (!settings.excludeTypes || settings.excludeTypes.length === 0) {
@@ -123,10 +28,10 @@ export function passesTypeFilter(item, settings) {
 }
 
 /**
- * Check if an item passes the excludeRegex filter
+ * Check if an item passes the channel.settings.excludeRegex filter.
  * @param {object} item - Feed item
  * @param {object} settings - Channel settings
- * @returns {boolean} Whether the item passes
+ * @returns {boolean} True when the item should be kept
  */
 export function passesRegexFilter(item, settings) {
   if (!settings.excludeRegex) {
@@ -146,17 +51,26 @@ export function passesRegexFilter(item, settings) {
 
     return !regex.test(searchText);
   } catch {
-    // Invalid regex, skip filter
+    // Invalid regex — skip the filter rather than rejecting every item.
     return true;
   }
 }
 
 /**
- * Detect the interaction type of an item
+ * Classify an item by its interaction property. Internal helper for
+ * passesTypeFilter — only its symbolic return value (one of "like" | "repost"
+ * | "bookmark" | "reply" | "rsvp" | "checkin" | "post") is compared against
+ * the excludeTypes list.
+ *
+ * Note: a similar but stricter classifier exists in `lib/utils/jf2.js` for
+ * API-response shaping. The two cannot trivially be merged because this one
+ * treats kebab-case keys only and emits a "post" default that the jf2 one
+ * doesn't.
+ *
  * @param {object} item - Feed item
  * @returns {string} Interaction type
  */
-export function detectInteractionType(item) {
+function detectInteractionType(item) {
   if (item["like-of"] && item["like-of"].length > 0) {
     return "like";
   }
@@ -178,88 +92,3 @@ export function detectInteractionType(item) {
 
   return "post";
 }
-
-/**
- * Get all muted URLs for a user/channel
- * @param {object} application - Indiekit application
- * @param {string} userId - User ID
- * @param {ObjectId|string} [channelId] - Channel ObjectId (optional, for channel-specific)
- * @returns {Promise<Array>} Array of muted URLs
- */
-export async function getMutedUrls(application, userId, channelId) {
-  const collection = getMutedCollection(application);
-  const filter = { userId };
-
-  if (channelId) {
-    const channelObjectId =
-      typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-    filter.channelId = channelObjectId;
-  }
-
-  // eslint-disable-next-line unicorn/no-array-callback-reference -- filter is MongoDB query object
-  const muted = await collection.find(filter).toArray();
-  return muted.map((m) => m.url);
-}
-
-/**
- * Get all blocked URLs for a user
- * @param {object} application - Indiekit application
- * @param {string} userId - User ID
- * @returns {Promise<Array>} Array of blocked URLs
- */
-export async function getBlockedUrls(application, userId) {
-  const collection = getBlockedCollection(application);
-  const blocked = await collection.find({ userId }).toArray();
-  return blocked.map((b) => b.url);
-}
-
-/**
- * Update channel filter settings
- * @param {object} application - Indiekit application
- * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {object} filters - Filter settings to update
- * @param {Array} [filters.excludeTypes] - Post types to exclude
- * @param {string} [filters.excludeRegex] - Regex pattern to exclude
- * @returns {Promise<object>} Updated channel
- */
-export async function updateChannelFilters(application, channelId, filters) {
-  const collection = application.collections.get("microsub_channels");
-  const channelObjectId =
-    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
-
-  const updateFields = {};
-
-  if (filters.excludeTypes !== undefined) {
-    updateFields["settings.excludeTypes"] = filters.excludeTypes;
-  }
-
-  if (filters.excludeRegex !== undefined) {
-    updateFields["settings.excludeRegex"] = filters.excludeRegex;
-  }
-
-  const result = await collection.findOneAndUpdate(
-    { _id: channelObjectId },
-    { $set: updateFields },
-    { returnDocument: "after" },
-  );
-
-  return result;
-}
-
-/**
- * Create indexes for filter collections
- * @param {object} application - Indiekit application
- * @returns {Promise<void>}
- */
-export async function createFilterIndexes(application) {
-  const mutedCollection = getMutedCollection(application);
-  const blockedCollection = getBlockedCollection(application);
-
-  // Muted collection indexes
-  await mutedCollection.createIndex({ userId: 1, channelId: 1, url: 1 });
-  await mutedCollection.createIndex({ userId: 1 });
-
-  // Blocked collection indexes
-  await blockedCollection.createIndex({ userId: 1, url: 1 }, { unique: true });
-  await blockedCollection.createIndex({ userId: 1 });
-}

package/lib/storage/items-read-state.js

@@ -5,22 +5,17 @@
 
 import { ObjectId } from "mongodb";
 
-import { UNREAD_RETENTION_DAYS } from "../utils/constants.js";
+import {
+  MAX_FULL_READ_ITEMS,
+  UNREAD_RETENTION_DAYS,
+} from "../utils/constants.js";
 import { getCollection } from "./items.js";
 
-// Maximum number of full read items to keep per channel before stripping content.
-// Items beyond this limit are converted to lightweight dedup skeletons (channelId,
-// uid, readBy) so the poller doesn't re-ingest them as new unread entries.
-const MAX_FULL_READ_ITEMS = 200;
-
 /**
  * Cleanup old read items by stripping content but preserving dedup skeletons.
- * This prevents the vicious cycle where deleted read items get re-ingested as
+ * Prevents the vicious cycle where deleted read items get re-ingested as
  * unread by the poller because the dedup record (channelId + uid) was destroyed.
  *
- * AP items (feedId: null) are hard-deleted instead of stripped, since no poller
- * re-ingests them — they arrive via inbox push and don't need dedup skeletons.
- *
  * @param {object} collection - MongoDB collection
  * @param {ObjectId} channelObjectId - Channel ObjectId
  * @param {string} userId - User ID
@@ -32,7 +27,6 @@ async function cleanupOldReadItems(collection, channelObjectId, userId) {
   });
 
   if (readCount > MAX_FULL_READ_ITEMS) {
-    // Find old read items beyond the retention limit
     const itemsToCleanup = await collection
       .find({
         channelId: channelObjectId,
@@ -41,59 +35,36 @@ async function cleanupOldReadItems(collection, channelObjectId, userId) {
       })
       .sort({ published: -1, _id: -1 })
       .skip(MAX_FULL_READ_ITEMS)
-      .project({ _id: 1, feedId: 1 })
+      .project({ _id: 1 })
       .toArray();
 
     if (itemsToCleanup.length === 0) return;
 
-    // Separate AP items (feedId: null) from RSS items (feedId: ObjectId)
-    const apItemIds = [];
-    const rssItemIds = [];
-    for (const item of itemsToCleanup) {
-      if (item.feedId) {
-        rssItemIds.push(item._id);
-      } else {
-        apItemIds.push(item._id);
-      }
-    }
-
-    // Hard-delete AP items — no poller to re-ingest, skeletons are useless
-    if (apItemIds.length > 0) {
-      const deleted = await collection.deleteMany({
-        _id: { $in: apItemIds },
-      });
-      console.info(
-        `[Microsub] Deleted ${deleted.deletedCount} old AP read items`,
-      );
-    }
-
-    // Strip RSS items to dedup skeletons — poller would re-ingest if deleted
-    if (rssItemIds.length > 0) {
-      const stripped = await collection.updateMany(
-        { _id: { $in: rssItemIds } },
-        {
-          $set: { _stripped: true },
-          $unset: {
-            name: "",
-            content: "",
-            summary: "",
-            author: "",
-            category: "",
-            photo: "",
-            video: "",
-            audio: "",
-            likeOf: "",
-            repostOf: "",
-            bookmarkOf: "",
-            inReplyTo: "",
-            source: "",
-          },
+    const ids = itemsToCleanup.map((item) => item._id);
+    const stripped = await collection.updateMany(
+      { _id: { $in: ids } },
+      {
+        $set: { _stripped: true },
+        $unset: {
+          name: "",
+          content: "",
+          summary: "",
+          author: "",
+          category: "",
+          photo: "",
+          video: "",
+          audio: "",
+          likeOf: "",
+          repostOf: "",
+          bookmarkOf: "",
+          inReplyTo: "",
+          source: "",
         },
-      );
-      console.info(
-        `[Microsub] Stripped ${stripped.modifiedCount} old RSS read items (keeping ${MAX_FULL_READ_ITEMS} full)`,
-      );
-    }
+      },
+    );
+    console.info(
+      `[Microsub] Stripped ${stripped.modifiedCount} old read items (keeping ${MAX_FULL_READ_ITEMS} full)`,
+    );
   }
 }
 

package/lib/storage/items-retention.js

@@ -3,11 +3,9 @@
  * @module storage/items-retention
  */
 
+import { MAX_FULL_READ_ITEMS } from "../utils/constants.js";
 import { getCollection } from "./items.js";
 
-// Maximum number of full read items to keep per channel before stripping content.
-const MAX_FULL_READ_ITEMS = 200;
-
 // Global retention defaults. Each can be overridden per channel via
 // channel.settings.{maxItems,maxItemsPerFeed,maxUnreadAgeDays}. The "notifications"
 // channel is exempt from these caps entirely — webmentions are high-signal and

package/lib/storage/items.js

@@ -323,20 +323,6 @@ export async function getItemById(application, id, userId) {
   return transformToJf2(item, userId);
 }
 
-/**
- * Get items by UIDs
- * @param {object} application - Indiekit application
- * @param {Array} uids - Array of item UIDs
- * @param {string} [userId] - User ID for read state
- * @returns {Promise<Array>} Array of jf2 items
- */
-export async function getItemsByUids(application, uids, userId) {
-  const collection = getCollection(application);
-
-  const items = await collection.find({ uid: { $in: uids } }).toArray();
-  return items.map((item) => transformToJf2(item, userId));
-}
-
 /**
  * Remove items from channel
  * @param {object} application - Indiekit application

package/lib/utils/blogroll-notify.js

@@ -36,7 +36,7 @@ export async function notifyBlogroll(application, action, data) {
       status: "deleted",
     });
     if (deleted) {
-      console.log(
+      console.info(
         `[Microsub→Blogroll] Skipping follow for ${data.url} — previously deleted by user`,
       );
       return;
@@ -76,7 +76,7 @@ export async function notifyBlogroll(application, action, data) {
       { upsert: true },
     );
 
-    console.log(`[Microsub→Blogroll] Added/updated feed ${data.url}`);
+    console.info(`[Microsub→Blogroll] Added/updated feed ${data.url}`);
   } else if (action === "unfollow") {
     // Soft-delete the blog entry if it came from microsub
     const result = await collection.updateOne(
@@ -96,7 +96,7 @@ export async function notifyBlogroll(application, action, data) {
     );
 
     if (result.modifiedCount > 0) {
-      console.log(`[Microsub→Blogroll] Soft-deleted feed ${data.url}`);
+      console.info(`[Microsub→Blogroll] Soft-deleted feed ${data.url}`);
     }
   }
 }

package/lib/utils/constants.js

@@ -5,3 +5,10 @@
 
 /** Retention period for unread count queries (only count recent items) */
 export const UNREAD_RETENTION_DAYS = 30;
+
+/**
+ * Maximum number of full read items to keep per channel/user before stripping
+ * content. Items beyond this limit are converted to lightweight dedup skeletons
+ * (channelId, uid, readBy) so the poller doesn't re-ingest them as new unread.
+ */
+export const MAX_FULL_READ_ITEMS = 200;

package/lib/utils/jf2.js

@@ -29,115 +29,6 @@ export function generateChannelUid() {
   return result;
 }
 
-/**
- * Create a jf2 Item from normalized feed data
- * @param {object} data - Normalized item data
- * @param {object} source - Feed source metadata
- * @returns {object} jf2 Item object
- */
-export function createJf2Item(data, source) {
-  return {
-    type: "entry",
-    uid: data.uid,
-    url: data.url,
-    name: data.name || undefined,
-    content: data.content || undefined,
-    summary: data.summary || undefined,
-    published: data.published,
-    updated: data.updated || undefined,
-    author: data.author || undefined,
-    category: data.category || [],
-    photo: data.photo || [],
-    video: data.video || [],
-    audio: data.audio || [],
-    // Interaction types
-    "like-of": data.likeOf || [],
-    "repost-of": data.repostOf || [],
-    "bookmark-of": data.bookmarkOf || [],
-    "in-reply-to": data.inReplyTo || [],
-    // Internal properties (prefixed with _)
-    _id: data._id,
-    _is_read: data._is_read || false,
-    _source: source,
-  };
-}
-
-/**
- * Create a jf2 Card (author/person)
- * @param {object} data - Author data
- * @returns {object} jf2 Card object
- */
-export function createJf2Card(data) {
-  if (!data) return;
-
-  return {
-    type: "card",
-    name: data.name || undefined,
-    url: data.url || undefined,
-    photo: data.photo || undefined,
-  };
-}
-
-/**
- * Create a jf2 Content object
- * @param {string} text - Plain text content
- * @param {string} html - HTML content
- * @returns {object|undefined} jf2 Content object
- */
-export function createJf2Content(text, html) {
-  if (!text && !html) return;
-
-  return {
-    text: text || stripHtml(html),
-    html: html || undefined,
-  };
-}
-
-/**
- * Strip HTML tags from string
- * @param {string} html - HTML string
- * @returns {string} Plain text
- */
-export function stripHtml(html) {
-  if (!html) return "";
-  return html.replaceAll(/<[^>]*>/g, "").trim();
-}
-
-/**
- * Create a jf2 Feed response
- * @param {object} options - Feed options
- * @param {Array} options.items - Array of jf2 items
- * @param {object} options.paging - Pagination cursors
- * @returns {object} jf2 Feed object
- */
-export function createJf2Feed({ items, paging }) {
-  const feed = {
-    items: items || [],
-  };
-
-  if (paging) {
-    feed.paging = {};
-    if (paging.before) feed.paging.before = paging.before;
-    if (paging.after) feed.paging.after = paging.after;
-  }
-
-  return feed;
-}
-
-/**
- * Create a Channel response object
- * @param {object} channel - Channel data
- * @param {number} unreadCount - Number of unread items
- * @returns {object} Channel object for API response
- */
-export function createChannelResponse(channel, unreadCount = 0) {
-  return {
-    uid: channel.uid,
-    name: channel.name,
-    unread: unreadCount > 0 ? unreadCount : false,
-  };
-}
-
 /**
  * Create a Feed response object
  * @param {object} feed - Feed data

package/lib/utils/pagination.js

@@ -11,7 +11,7 @@ import { ObjectId } from "mongodb";
  * @param {string} id - Item ID
  * @returns {string} Base64-encoded cursor
  */
-export function encodeCursor(timestamp, id) {
+function encodeCursor(timestamp, id) {
   const data = {
     t: timestamp instanceof Date ? timestamp.toISOString() : timestamp,
     i: id.toString(),
@@ -24,7 +24,7 @@ export function encodeCursor(timestamp, id) {
  * @param {string} cursor - Base64-encoded cursor
  * @returns {object|null} Decoded cursor with timestamp and id
  */
-export function decodeCursor(cursor) {
+function decodeCursor(cursor) {
   if (!cursor) return;
 
   try {
@@ -133,15 +133,8 @@ export function generatePagingCursors(items, limit, hasMore, before) {
   return paging;
 }
 
-/**
- * Default pagination limit
- */
-export const DEFAULT_LIMIT = 20;
-
-/**
- * Maximum pagination limit
- */
-export const MAX_LIMIT = 100;
+const DEFAULT_LIMIT = 20;
+const MAX_LIMIT = 100;
 
 /**
  * Parse and validate limit parameter

package/lib/utils/sanitize.js

@@ -1,6 +1,5 @@
 /**
- * Shared HTML sanitization configuration
- * Used by both RSS/Atom normalizer and ActivityPub outbox fetcher
+ * Shared HTML sanitization configuration used by the feed normalizers.
  * @module utils/sanitize
  */
 

package/lib/utils/validation.js

@@ -23,16 +23,6 @@ export const VALID_ACTIONS = [
   "events",
 ];
 
-/**
- * Valid channel methods
- */
-export const VALID_CHANNEL_METHODS = ["delete", "order"];
-
-/**
- * Valid timeline methods
- */
-export const VALID_TIMELINE_METHODS = ["mark_read", "mark_read_source", "mark_unread", "remove"];
-
 /**
  * Valid exclude types for channel filtering
  */