@rmdes/indiekit-endpoint-microsub 1.0.56 → 1.0.58

package/lib/controllers/reader/search.js

@@ -0,0 +1,157 @@
+/**
+ * Feed discovery UI
+ * @module controllers/reader/search
+ */
+
+import { discoverAndValidateFeeds } from "../../feeds/discovery.js";
+import { validateFeedUrl } from "../../feeds/validator.js";
+import { refreshFeedNow } from "../../polling/scheduler.js";
+import { getChannels, getChannel } from "../../storage/channels.js";
+import { createFeed } from "../../storage/feeds.js";
+import { getUserId } from "../../utils/auth.js";
+
+/**
+ * Search/discover feeds page
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function searchPage(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+
+  const channelList = await getChannels(application, userId);
+
+  response.render("search", {
+    title: request.__("microsub.search.title"),
+    channels: channelList,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "channels",
+    breadcrumbs: [
+      { text: "Reader", href: request.baseUrl },
+      { text: "Search" },
+    ],
+  });
+}
+
+/**
+ * Search for feeds from URL - enhanced with validation
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function searchFeeds(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { query } = request.body;
+
+  const channelList = await getChannels(application, userId);
+
+  let results = [];
+  let discoveryError = null;
+
+  if (query) {
+    try {
+      // Use enhanced discovery with validation
+      results = await discoverAndValidateFeeds(query);
+    } catch (error) {
+      discoveryError = error.message;
+    }
+  }
+
+  response.render("search", {
+    title: request.__("microsub.search.title"),
+    channels: channelList,
+    query,
+    results,
+    discoveryError,
+    searched: true,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "channels",
+    breadcrumbs: [
+      { text: "Reader", href: request.baseUrl },
+      { text: "Search" },
+    ],
+  });
+}
+
+/**
+ * Subscribe to a feed from search results - with validation
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function subscribe(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { url, channel: channelUid, skipValidation } = request.body;
+
+  const channelDocument = await getChannel(application, channelUid, userId);
+  if (!channelDocument) {
+    return response.status(404).render("404");
+  }
+
+  // Validate feed unless explicitly skipped (for power users)
+  if (!skipValidation) {
+    const validation = await validateFeedUrl(url);
+
+    if (!validation.valid) {
+      const channelList = await getChannels(application, userId);
+      return response.render("search", {
+        title: request.__("microsub.search.title"),
+        channels: channelList,
+        query: url,
+        validationError: validation.error,
+        baseUrl: request.baseUrl,
+        readerBaseUrl: request.baseUrl,
+        activeView: "channels",
+        breadcrumbs: [
+          { text: "Reader", href: request.baseUrl },
+          { text: "Search" },
+        ],
+      });
+    }
+
+    // Warn about comments feeds but allow subscription
+    if (validation.isCommentsFeed) {
+      console.warn(`[Microsub] Subscribing to comments feed: ${url}`);
+    }
+  }
+
+  // Create feed subscription (throws DUPLICATE_FEED if already exists elsewhere)
+  try {
+    const feed = await createFeed(application, {
+      channelId: channelDocument._id,
+      url,
+      title: undefined,
+      photo: undefined,
+    });
+
+    // Trigger immediate fetch in background
+    refreshFeedNow(application, feed._id).catch((error) => {
+      console.error(`[Microsub] Error fetching new feed ${url}:`, error.message);
+    });
+
+    response.redirect(`${request.baseUrl}/channels/${channelUid}/feeds`);
+  } catch (error) {
+    if (error.code === "DUPLICATE_FEED") {
+      const channelList = await getChannels(application, userId);
+      return response.render("search", {
+        title: request.__("microsub.search.title"),
+        channels: channelList,
+        query: url,
+        validationError: `This feed already exists in channel "${error.channelName}"`,
+        baseUrl: request.baseUrl,
+        readerBaseUrl: request.baseUrl,
+        activeView: "channels",
+        breadcrumbs: [
+          { text: "Reader", href: request.baseUrl },
+          { text: "Search" },
+        ],
+      });
+    }
+    throw error;
+  }
+}

package/lib/controllers/reader/timeline.js

@@ -0,0 +1,251 @@
+/**
+ * Timeline views + read state
+ * @module controllers/reader/timeline
+ */
+
+import { getChannel, getChannelById, getChannelsWithColors } from "../../storage/channels.js";
+import {
+  getTimelineItems,
+  getAllTimelineItems,
+  getItemById,
+} from "../../storage/items.js";
+import { markItemsRead } from "../../storage/items-read-state.js";
+import { getUserId } from "../../utils/auth.js";
+import { proxyItemImages } from "../../media/proxy.js";
+
+/**
+ * Timeline view - all channels chronologically
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function timeline(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { before, after } = request.query;
+
+  // Get channels with colors for filtering UI and item decoration
+  const channelList = await getChannelsWithColors(application, userId);
+
+  // Build channel lookup map (ObjectId string -> { name, color, uid })
+  const channelMap = new Map();
+  for (const ch of channelList) {
+    channelMap.set(ch._id.toString(), { name: ch.name, color: ch.color, uid: ch.uid });
+  }
+
+  // Parse excluded channel IDs from query params
+  const excludeParam = request.query.exclude;
+  const excludeIds = excludeParam
+    ? (Array.isArray(excludeParam) ? excludeParam : [excludeParam])
+    : [];
+
+  // Exclude the notifications channel by default
+  const notificationsChannel = channelList.find((ch) => ch.uid === "notifications");
+  const excludeChannelIds = [...excludeIds];
+  if (notificationsChannel && !excludeChannelIds.includes(notificationsChannel._id.toString())) {
+    excludeChannelIds.push(notificationsChannel._id.toString());
+  }
+
+  const result = await getAllTimelineItems(application, {
+    before,
+    after,
+    userId,
+    excludeChannelIds,
+  });
+
+  // Proxy images
+  const proxyBaseUrl = application.url;
+  if (proxyBaseUrl && result.items) {
+    result.items = result.items.map((item) => proxyItemImages(item, proxyBaseUrl));
+  }
+
+  // Decorate items with channel name and color
+  for (const item of result.items) {
+    if (item._channelId) {
+      const info = channelMap.get(item._channelId);
+      if (info) {
+        item._channelName = info.name;
+        item._channelColor = info.color;
+        item._channelUid = info.uid;
+      }
+    }
+  }
+
+  // Set view preference cookie
+  if (request.session) request.session.microsubView = "timeline";
+
+  response.render("timeline", {
+    title: "Timeline",
+    channels: channelList,
+    items: result.items,
+    paging: result.paging,
+    excludeIds,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "timeline",
+    breadcrumbs: [
+      { text: "Reader", href: request.baseUrl },
+      { text: "Timeline" },
+    ],
+  });
+}
+
+/**
+ * Return rendered HTML fragments for timeline infinite scroll
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function timelineHtml(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { before, after } = request.query;
+
+  const channelList = await getChannelsWithColors(application, userId);
+  const channelMap = new Map();
+  for (const ch of channelList) {
+    channelMap.set(ch._id.toString(), { name: ch.name, color: ch.color, uid: ch.uid });
+  }
+
+  const excludeParam = request.query.exclude;
+  const excludeIds = excludeParam
+    ? (Array.isArray(excludeParam) ? excludeParam : [excludeParam])
+    : [];
+
+  const notificationsChannel = channelList.find((ch) => ch.uid === "notifications");
+  const excludeChannelIds = [...excludeIds];
+  if (notificationsChannel && !excludeChannelIds.includes(notificationsChannel._id.toString())) {
+    excludeChannelIds.push(notificationsChannel._id.toString());
+  }
+
+  const result = await getAllTimelineItems(application, {
+    before,
+    after,
+    userId,
+    excludeChannelIds,
+  });
+
+  const proxyBaseUrl = application.url;
+  if (proxyBaseUrl && result.items) {
+    result.items = result.items.map((item) => proxyItemImages(item, proxyBaseUrl));
+  }
+
+  for (const item of result.items) {
+    if (item._channelId) {
+      const info = channelMap.get(item._channelId);
+      if (info) {
+        item._channelName = info.name;
+        item._channelColor = info.color;
+        item._channelUid = info.uid;
+      }
+    }
+  }
+
+  const fragmentHtml = await new Promise((resolve, reject) => {
+    response.render("partials/items-fragment-timeline", {
+      items: result.items,
+      baseUrl: request.baseUrl,
+    }, (error, html) => error ? reject(error) : resolve(html));
+  });
+
+  response.json({
+    html: fragmentHtml,
+    paging: result.paging,
+    count: result.items.length,
+  });
+}
+
+/**
+ * Mark all items in channel as read
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function markAllRead(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { channel: channelUid } = request.body;
+
+  const channelDocument = await getChannel(application, channelUid, userId);
+  if (!channelDocument) {
+    return response.status(404).render("404");
+  }
+
+  // Mark all items as read using the special "last-read-entry" value
+  await markItemsRead(
+    application,
+    channelDocument._id,
+    ["last-read-entry"],
+    userId,
+  );
+
+  response.redirect(`${request.baseUrl}/channels/${channelUid}`);
+}
+
+/**
+ * Mark specific items as read (no-JS form fallback for mark-view-as-read)
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function markViewRead(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { channel: channelUid } = request.body;
+  let { entry } = request.body;
+
+  const channelDocument = await getChannel(application, channelUid, userId);
+  if (!channelDocument) {
+    return response.status(404).render("404");
+  }
+
+  const entryIds = Array.isArray(entry) ? entry : entry ? [entry] : [];
+  if (entryIds.length > 0) {
+    await markItemsRead(application, channelDocument._id, entryIds, userId);
+  }
+
+  response.redirect(`${request.baseUrl}/channels/${channelUid}`);
+}
+
+/**
+ * View single item
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function item(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { id } = request.params;
+
+  const itemDocument = await getItemById(application, id, userId);
+  if (!itemDocument) {
+    return response.status(404).render("404");
+  }
+
+  // Get the channel for this item (needed for mark-read)
+  // Note: transformToJf2() renames channelId → _channelId (string)
+  let channel = null;
+  if (itemDocument._channelId) {
+    channel = await getChannelById(application, itemDocument._channelId);
+  }
+
+  const itemBreadcrumbs = [
+    { text: "Reader", href: request.baseUrl },
+  ];
+  if (channel) {
+    itemBreadcrumbs.push(
+      { text: "Channels", href: `${request.baseUrl}/channels` },
+      { text: channel.name, href: `${request.baseUrl}/channels/${channel.uid}` },
+    );
+  }
+  itemBreadcrumbs.push({ text: itemDocument.name || "Item" });
+
+  response.render("item", {
+    title: itemDocument.name || "Item",
+    item: itemDocument,
+    channel,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "channels",
+    breadcrumbs: itemBreadcrumbs,
+  });
+}

package/lib/controllers/timeline.js

@@ -9,11 +9,13 @@ import { proxyItemImages } from "../media/proxy.js";
 import { getChannel, getChannelById } from "../storage/channels.js";
 import {
   getTimelineItems,
+  removeItems,
+} from "../storage/items.js";
+import {
   markFeedItemsRead,
   markItemsRead,
   markItemsUnread,
-  removeItems,
-} from "../storage/items.js";
+} from "../storage/items-read-state.js";
 import { getUserId } from "../utils/auth.js";
 import {
   validateChannel,

package/lib/feeds/atom.js

@@ -7,7 +7,7 @@ import { Readable } from "node:stream";
 
 import FeedParser from "feedparser";
 
-import { normalizeItem, normalizeFeedMeta } from "./normalizer.js";
+import { normalizeItem, normalizeFeedMeta } from "./normalizer-rss.js";
 
 /**
  * Parse Atom feed content

package/lib/feeds/fetcher.js

@@ -3,7 +3,6 @@
  * @module feeds/fetcher
  */
 
-import { getCache, setCache } from "../cache/redis.js";
 import { isPrivateUrl } from "../media/proxy.js";
 
 const DEFAULT_TIMEOUT = 30_000; // 30 seconds
@@ -16,32 +15,16 @@ const DEFAULT_USER_AGENT = "Indiekit Microsub/1.0 (+https://getindiekit.com)";
  * @param {string} [options.etag] - Previous ETag for conditional request
  * @param {string} [options.lastModified] - Previous Last-Modified for conditional request
  * @param {number} [options.timeout] - Request timeout in ms
- * @param {object} [options.redis] - Redis client for caching
  * @returns {Promise<object>} Fetch result with content and headers
  */
 export async function fetchFeed(url, options = {}) {
-  const { etag, lastModified, timeout = DEFAULT_TIMEOUT, redis } = options;
+  const { etag, lastModified, timeout = DEFAULT_TIMEOUT } = options;
 
   // SSRF protection — block private/internal IPs (including DNS rebinding)
   if (await isPrivateUrl(url)) {
     throw new Error(`Feed URL blocked (private/internal address): ${url}`);
   }
 
-  // Check cache first
-  if (redis) {
-    const cached = await getCache(redis, `feed:${url}`);
-    if (cached) {
-      return {
-        content: cached.content,
-        contentType: cached.contentType,
-        etag: cached.etag,
-        lastModified: cached.lastModified,
-        fromCache: true,
-        status: 200,
-      };
-    }
-  }
-
   const headers = {
     Accept:
       "application/atom+xml, application/rss+xml, application/json, application/feed+json, text/xml, text/html;q=0.9, */*;q=0.8",
@@ -105,18 +88,6 @@ export async function fetchFeed(url, options = {}) {
       result.self = extractSelfFromLinkHeader(linkHeader);
     }
 
-    // Cache the result
-    if (redis) {
-      const cacheData = {
-        content,
-        contentType,
-        etag: responseEtag,
-        lastModified: responseLastModified,
-      };
-      // Cache for 5 minutes by default
-      await setCache(redis, `feed:${url}`, cacheData, 300);
-    }
-
     return result;
   } catch (error) {
     clearTimeout(timeoutId);

package/lib/feeds/hfeed.js

@@ -5,7 +5,7 @@
 
 import { mf2 } from "microformats-parser";
 
-import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer.js";
+import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer-hfeed.js";
 
 /**
  * Parse h-feed content from HTML

package/lib/feeds/jsonfeed.js

@@ -3,7 +3,7 @@
  * @module feeds/jsonfeed
  */
 
-import { normalizeJsonFeedItem, normalizeJsonFeedMeta } from "./normalizer.js";
+import { normalizeJsonFeedItem, normalizeJsonFeedMeta } from "./normalizer-jsonfeed.js";
 
 /**
  * Parse JSON Feed content