@rmdes/indiekit-endpoint-microsub 1.0.42 → 1.0.44

package/index.js

@@ -9,7 +9,11 @@ import { readerController } from "./lib/controllers/reader.js";
 import { handleMediaProxy } from "./lib/media/proxy.js";
 import { startScheduler, stopScheduler } from "./lib/polling/scheduler.js";
 import { ensureActivityPubChannel } from "./lib/storage/channels.js";
-import { cleanupAllReadItems, createIndexes } from "./lib/storage/items.js";
+import {
+  cleanupAllReadItems,
+  cleanupStaleItems,
+  createIndexes,
+} from "./lib/storage/items.js";
 import { webmentionReceiver } from "./lib/webmention/receiver.js";
 import { websubHandler } from "./lib/websub/handler.js";
 
@@ -210,6 +214,11 @@ export default class MicrosubEndpoint {
       cleanupAllReadItems(indiekit).catch((error) => {
         console.warn("[Microsub] Startup cleanup failed:", error.message);
       });
+
+      // Delete stale items (stripped skeletons + unread older than 30 days)
+      cleanupStaleItems(indiekit).catch((error) => {
+        console.warn("[Microsub] Stale cleanup failed:", error.message);
+      });
     } else {
       console.warn(
         "[Microsub] Database not available at init, scheduler not started",

package/lib/controllers/follow.js

@@ -67,13 +67,24 @@ export async function follow(request, response) {
     throw new IndiekitError("Channel not found", { status: 404 });
   }
 
-  // Create feed subscription
-  const feed = await createFeed(application, {
-    channelId: channelDocument._id,
-    url,
-    title: undefined, // Will be populated on first fetch
-    photo: undefined,
-  });
+  // Create feed subscription (throws DUPLICATE_FEED if already exists elsewhere)
+  let feed;
+  try {
+    feed = await createFeed(application, {
+      channelId: channelDocument._id,
+      url,
+      title: undefined, // Will be populated on first fetch
+      photo: undefined,
+    });
+  } catch (error) {
+    if (error.code === "DUPLICATE_FEED") {
+      throw new IndiekitError(
+        `Feed already exists in channel "${error.channelName}"`,
+        { status: 409 },
+      );
+    }
+    throw error;
+  }
 
   // Trigger immediate fetch in background (don't await)
   // This will also discover and subscribe to WebSub hubs

package/lib/controllers/reader.js

@@ -319,20 +319,43 @@ export async function addFeed(request, response) {
     return response.status(404).render("404");
   }
 
-  // Create feed subscription
-  const feed = await createFeed(application, {
-    channelId: channelDocument._id,
-    url,
-    title: undefined,
-    photo: undefined,
-  });
+  try {
+    // Create feed subscription (throws DUPLICATE_FEED if already exists)
+    const feed = await createFeed(application, {
+      channelId: channelDocument._id,
+      url,
+      title: undefined,
+      photo: undefined,
+    });
 
-  // Trigger immediate fetch in background
-  refreshFeedNow(application, feed._id).catch((error) => {
-    console.error(`[Microsub] Error fetching new feed ${url}:`, error.message);
-  });
+    // Trigger immediate fetch in background
+    refreshFeedNow(application, feed._id).catch((error) => {
+      console.error(`[Microsub] Error fetching new feed ${url}:`, error.message);
+    });
 
-  response.redirect(`${request.baseUrl}/channels/${uid}/feeds`);
+    response.redirect(`${request.baseUrl}/channels/${uid}/feeds`);
+  } catch (error) {
+    if (error.code === "DUPLICATE_FEED") {
+      // Re-render feeds page with error message
+      const feedList = await getFeedsForChannel(application, channelDocument._id);
+      return response.render("feeds", {
+        title: request.__("microsub.feeds.title"),
+        channel: channelDocument,
+        feeds: feedList,
+        baseUrl: request.baseUrl,
+        readerBaseUrl: request.baseUrl,
+        activeView: "channels",
+        error: `This feed already exists in channel "${error.channelName}"`,
+        breadcrumbs: [
+          { text: "Reader", href: request.baseUrl },
+          { text: "Channels", href: `${request.baseUrl}/channels` },
+          { text: channelDocument.name, href: `${request.baseUrl}/channels/${uid}` },
+          { text: "Feeds" },
+        ],
+      });
+    }
+    throw error;
+  }
 }
 
 /**
@@ -782,20 +805,40 @@ export async function subscribe(request, response) {
     }
   }
 
-  // Create feed subscription
-  const feed = await createFeed(application, {
-    channelId: channelDocument._id,
-    url,
-    title: undefined,
-    photo: undefined,
-  });
+  // Create feed subscription (throws DUPLICATE_FEED if already exists elsewhere)
+  try {
+    const feed = await createFeed(application, {
+      channelId: channelDocument._id,
+      url,
+      title: undefined,
+      photo: undefined,
+    });
 
-  // Trigger immediate fetch in background
-  refreshFeedNow(application, feed._id).catch((error) => {
-    console.error(`[Microsub] Error fetching new feed ${url}:`, error.message);
-  });
+    // Trigger immediate fetch in background
+    refreshFeedNow(application, feed._id).catch((error) => {
+      console.error(`[Microsub] Error fetching new feed ${url}:`, error.message);
+    });
 
-  response.redirect(`${request.baseUrl}/channels/${channelUid}/feeds`);
+    response.redirect(`${request.baseUrl}/channels/${channelUid}/feeds`);
+  } catch (error) {
+    if (error.code === "DUPLICATE_FEED") {
+      const channelList = await getChannels(application, userId);
+      return response.render("search", {
+        title: request.__("microsub.search.title"),
+        channels: channelList,
+        query: url,
+        validationError: `This feed already exists in channel "${error.channelName}"`,
+        baseUrl: request.baseUrl,
+        readerBaseUrl: request.baseUrl,
+        activeView: "channels",
+        breadcrumbs: [
+          { text: "Reader", href: request.baseUrl },
+          { text: "Search" },
+        ],
+      });
+    }
+    throw error;
+  }
 }
 
 /**

package/lib/feeds/fetcher.js

@@ -171,12 +171,14 @@ export async function fetchAndParseFeed(url, options = {}) {
       // Fetch and parse the discovered feed
       const feedResult = await fetchFeed(fallbackFeed.url, options);
       if (!feedResult.notModified) {
+        const fallbackType = detectFeedType(feedResult.content, feedResult.contentType);
         const parsed = await parseFeed(feedResult.content, fallbackFeed.url, {
           contentType: feedResult.contentType,
         });
         return {
           ...feedResult,
           ...parsed,
+          feedType: fallbackType,
           hub: feedResult.hub || parsed._hub,
           discoveredFrom: url,
         };
@@ -194,6 +196,7 @@ export async function fetchAndParseFeed(url, options = {}) {
   return {
     ...result,
     ...parsed,
+    feedType: feedType,
     hub: result.hub || parsed._hub,
   };
 }

package/lib/polling/processor.js

@@ -132,13 +132,16 @@ export async function processFeed(application, feed) {
       lastModified: parsed.lastModified,
     };
 
-    // Update feed title/photo if discovered
+    // Update feed title/photo/feedType if discovered
     if (parsed.name && !feed.title) {
       updateData.title = parsed.name;
     }
     if (parsed.photo && !feed.photo) {
       updateData.photo = parsed.photo;
     }
+    if (parsed.feedType && !feed.feedType) {
+      updateData.feedType = parsed.feedType;
+    }
 
     await updateFeedAfterFetch(
       application,

package/lib/storage/feeds.js

@@ -16,6 +16,73 @@ function getCollection(application) {
   return application.collections.get("microsub_feeds");
 }
 
+/**
+ * Normalize a feed URL for duplicate comparison.
+ * Strips trailing slashes, normalizes protocol to https, lowercases hostname.
+ * @param {string} url - Feed URL
+ * @returns {string} Normalized URL
+ */
+export function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    // Normalize protocol to https
+    parsed.protocol = "https:";
+    // Lowercase hostname
+    parsed.hostname = parsed.hostname.toLowerCase();
+    // Remove trailing slash from path (but keep "/" for root)
+    if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+      parsed.pathname = parsed.pathname.slice(0, -1);
+    }
+    return parsed.href;
+  } catch {
+    return url;
+  }
+}
+
+/**
+ * Find an existing feed across ALL channels by normalized URL
+ * @param {object} application - Indiekit application
+ * @param {string} url - Feed URL to check
+ * @returns {Promise<object|null>} Existing feed with channel info, or null
+ */
+export async function findFeedAcrossChannels(application, url) {
+  const collection = getCollection(application);
+  const normalized = normalizeUrl(url);
+
+  // Get all feeds and check normalized URLs
+  // We check a few common URL variants directly for efficiency
+  const variants = new Set();
+  variants.add(url);
+  variants.add(normalized);
+  // Also try with/without trailing slash
+  if (url.endsWith("/")) {
+    variants.add(url.slice(0, -1));
+  } else {
+    variants.add(url + "/");
+  }
+  // Try http/https variants
+  if (url.startsWith("https://")) {
+    variants.add(url.replace("https://", "http://"));
+  } else if (url.startsWith("http://")) {
+    variants.add(url.replace("http://", "https://"));
+  }
+
+  const existing = await collection.findOne({
+    url: { $in: [...variants] },
+  });
+
+  if (!existing) return null;
+
+  // Look up the channel name for a useful error message
+  const channelsCollection = application.collections.get("microsub_channels");
+  const channel = await channelsCollection.findOne({ _id: existing.channelId });
+
+  return {
+    feed: existing,
+    channelName: channel?.name || "unknown channel",
+  };
+}
+
 /**
  * Create a new feed subscription
  * @param {object} application - Indiekit application
@@ -32,12 +99,24 @@ export async function createFeed(
 ) {
   const collection = getCollection(application);
 
-  // Check if feed already exists in channel
+  // Check if feed already exists in this channel (exact match)
   const existing = await collection.findOne({ channelId, url });
   if (existing) {
     return existing;
   }
 
+  // Check for duplicate across ALL channels (normalized URL)
+  const duplicate = await findFeedAcrossChannels(application, url);
+  if (duplicate) {
+    const error = new Error(
+      `Feed already exists in channel "${duplicate.channelName}"`,
+    );
+    error.code = "DUPLICATE_FEED";
+    error.existingFeed = duplicate.feed;
+    error.channelName = duplicate.channelName;
+    throw error;
+  }
+
   const feed = {
     channelId,
     url,

package/lib/storage/items.js

@@ -387,6 +387,10 @@ export async function countReadItems(application, channelId, userId) {
 // uid, readBy) so the poller doesn't re-ingest them as new unread entries.
 const MAX_FULL_READ_ITEMS = 200;
 
+// Maximum age (in days) for stripped skeletons and unread items.
+// After this period, both are hard-deleted to prevent unbounded growth.
+const MAX_ITEM_AGE_DAYS = 30;
+
 /**
  * Cleanup old read items by stripping content but preserving dedup skeletons.
  * This prevents the vicious cycle where deleted read items get re-ingested as
@@ -575,6 +579,63 @@ export async function cleanupAllReadItems(application) {
   return totalCleaned;
 }
 
+/**
+ * Delete stale items: stripped skeletons and unread items older than MAX_ITEM_AGE_DAYS.
+ * Stripped skeletons have served their dedup purpose; stale unread items are unlikely
+ * to be read. Both are hard-deleted to prevent unbounded collection growth.
+ * @param {object} application - Indiekit application
+ * @returns {Promise<number>} Total number of items deleted
+ */
+export async function cleanupStaleItems(application) {
+  const collection = getCollection(application);
+  const cutoff = new Date();
+  cutoff.setDate(cutoff.getDate() - MAX_ITEM_AGE_DAYS);
+
+  // Delete stripped skeletons older than cutoff
+  const strippedResult = await collection.deleteMany({
+    _stripped: true,
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  // Delete unread items older than cutoff
+  const unreadResult = await collection.deleteMany({
+    readBy: { $in: [null, []] },
+    _stripped: { $ne: true },
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  // Also catch items with no readBy field at all
+  const noReadByResult = await collection.deleteMany({
+    readBy: { $exists: false },
+    _stripped: { $ne: true },
+    $or: [
+      { published: { $lt: cutoff } },
+      { published: { $exists: false }, createdAt: { $lt: cutoff.toISOString() } },
+    ],
+  });
+
+  const total =
+    strippedResult.deletedCount +
+    unreadResult.deletedCount +
+    noReadByResult.deletedCount;
+
+  if (total > 0) {
+    console.info(
+      `[Microsub] Stale cleanup: deleted ${strippedResult.deletedCount} stripped skeletons, ` +
+        `${unreadResult.deletedCount + noReadByResult.deletedCount} stale unread items ` +
+        `(cutoff: ${MAX_ITEM_AGE_DAYS} days)`,
+    );
+  }
+
+  return total;
+}
+
 export async function markItemsRead(application, channelId, entryIds, userId) {
   const collection = getCollection(application);
   const channelObjectId =

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-microsub",
-  "version": "1.0.42",
+  "version": "1.0.44",
   "description": "Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the Microsub protocol.",
   "keywords": [
     "indiekit",

package/views/feeds.njk

@@ -10,6 +10,12 @@
 
     <h2>{{ __("microsub.feeds.title") }}</h2>
 
+    {% if error %}
+    <div class="notice notice--error" role="alert">
+      {{ error }}
+    </div>
+    {% endif %}
+
     {% if feeds.length > 0 %}
     <div class="feeds__list">
       {% for feed in feeds %}
@@ -27,6 +33,9 @@
           <div class="feeds__details">
             <span class="feeds__name">
               {{ feed.title or feed.url }}
+              {% if feed.feedType %}
+              <span class="badge badge--offset badge--small" title="Feed format">{{ feed.feedType | upper }}</span>
+              {% endif %}
               {% if feed.status == 'error' %}
               <span class="badge badge--red">Error</span>
               {% elif feed.status == 'active' %}