@rmdes/indiekit-endpoint-microsub 1.0.55 → 1.0.57

package/lib/controllers/reader/timeline.js

@@ -0,0 +1,250 @@
+/**
+ * Timeline views + read state
+ * @module controllers/reader/timeline
+ */
+
+import { getChannel, getChannelById, getChannelsWithColors } from "../../storage/channels.js";
+import {
+  getTimelineItems,
+  getAllTimelineItems,
+  getItemById,
+} from "../../storage/items.js";
+import { markItemsRead } from "../../storage/items-read-state.js";
+import { getUserId } from "../../utils/auth.js";
+import { proxyItemImages } from "../../media/proxy.js";
+
+/**
+ * Timeline view - all channels chronologically
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function timeline(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { before, after } = request.query;
+
+  // Get channels with colors for filtering UI and item decoration
+  const channelList = await getChannelsWithColors(application, userId);
+
+  // Build channel lookup map (ObjectId string -> { name, color, uid })
+  const channelMap = new Map();
+  for (const ch of channelList) {
+    channelMap.set(ch._id.toString(), { name: ch.name, color: ch.color, uid: ch.uid });
+  }
+
+  // Parse excluded channel IDs from query params
+  const excludeParam = request.query.exclude;
+  const excludeIds = excludeParam
+    ? (Array.isArray(excludeParam) ? excludeParam : [excludeParam])
+    : [];
+
+  // Exclude the notifications channel by default
+  const notificationsChannel = channelList.find((ch) => ch.uid === "notifications");
+  const excludeChannelIds = [...excludeIds];
+  if (notificationsChannel && !excludeChannelIds.includes(notificationsChannel._id.toString())) {
+    excludeChannelIds.push(notificationsChannel._id.toString());
+  }
+
+  const result = await getAllTimelineItems(application, {
+    before,
+    after,
+    userId,
+    excludeChannelIds,
+  });
+
+  // Proxy images
+  const proxyBaseUrl = application.url;
+  if (proxyBaseUrl && result.items) {
+    result.items = result.items.map((item) => proxyItemImages(item, proxyBaseUrl));
+  }
+
+  // Decorate items with channel name and color
+  for (const item of result.items) {
+    if (item._channelId) {
+      const info = channelMap.get(item._channelId);
+      if (info) {
+        item._channelName = info.name;
+        item._channelColor = info.color;
+        item._channelUid = info.uid;
+      }
+    }
+  }
+
+  // Set view preference cookie
+  if (request.session) request.session.microsubView = "timeline";
+
+  response.render("timeline", {
+    title: "Timeline",
+    channels: channelList,
+    items: result.items,
+    paging: result.paging,
+    excludeIds,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "timeline",
+    breadcrumbs: [
+      { text: "Reader", href: request.baseUrl },
+      { text: "Timeline" },
+    ],
+  });
+}
+
+/**
+ * Return rendered HTML fragments for timeline infinite scroll
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function timelineHtml(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { before, after } = request.query;
+
+  const channelList = await getChannelsWithColors(application, userId);
+  const channelMap = new Map();
+  for (const ch of channelList) {
+    channelMap.set(ch._id.toString(), { name: ch.name, color: ch.color, uid: ch.uid });
+  }
+
+  const excludeParam = request.query.exclude;
+  const excludeIds = excludeParam
+    ? (Array.isArray(excludeParam) ? excludeParam : [excludeParam])
+    : [];
+
+  const notificationsChannel = channelList.find((ch) => ch.uid === "notifications");
+  const excludeChannelIds = [...excludeIds];
+  if (notificationsChannel && !excludeChannelIds.includes(notificationsChannel._id.toString())) {
+    excludeChannelIds.push(notificationsChannel._id.toString());
+  }
+
+  const result = await getAllTimelineItems(application, {
+    before,
+    after,
+    userId,
+    excludeChannelIds,
+  });
+
+  const proxyBaseUrl = application.url;
+  if (proxyBaseUrl && result.items) {
+    result.items = result.items.map((item) => proxyItemImages(item, proxyBaseUrl));
+  }
+
+  for (const item of result.items) {
+    if (item._channelId) {
+      const info = channelMap.get(item._channelId);
+      if (info) {
+        item._channelName = info.name;
+        item._channelColor = info.color;
+        item._channelUid = info.uid;
+      }
+    }
+  }
+
+  const fragmentHtml = await new Promise((resolve, reject) => {
+    response.render("partials/items-fragment-timeline", {
+      items: result.items,
+      baseUrl: request.baseUrl,
+    }, (error, html) => error ? reject(error) : resolve(html));
+  });
+
+  response.json({
+    html: fragmentHtml,
+    paging: result.paging,
+    count: result.items.length,
+  });
+}
+
+/**
+ * Mark all items in channel as read
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function markAllRead(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { channel: channelUid } = request.body;
+
+  const channelDocument = await getChannel(application, channelUid, userId);
+  if (!channelDocument) {
+    return response.status(404).render("404");
+  }
+
+  // Mark all items as read using the special "last-read-entry" value
+  await markItemsRead(
+    application,
+    channelDocument._id,
+    ["last-read-entry"],
+    userId,
+  );
+
+  response.redirect(`${request.baseUrl}/channels/${channelUid}`);
+}
+
+/**
+ * Mark specific items as read (no-JS form fallback for mark-view-as-read)
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function markViewRead(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { channel: channelUid } = request.body;
+  let { entry } = request.body;
+
+  const channelDocument = await getChannel(application, channelUid, userId);
+  if (!channelDocument) {
+    return response.status(404).render("404");
+  }
+
+  const entryIds = Array.isArray(entry) ? entry : entry ? [entry] : [];
+  if (entryIds.length > 0) {
+    await markItemsRead(application, channelDocument._id, entryIds, userId);
+  }
+
+  response.redirect(`${request.baseUrl}/channels/${channelUid}`);
+}
+
+/**
+ * View single item
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function item(request, response) {
+  const { application } = request.app.locals;
+  const userId = getUserId(request);
+  const { id } = request.params;
+
+  const itemDocument = await getItemById(application, id, userId);
+  if (!itemDocument) {
+    return response.status(404).render("404");
+  }
+
+  // Get the channel for this item (needed for mark-read)
+  let channel = null;
+  if (itemDocument.channelId) {
+    channel = await getChannelById(application, itemDocument.channelId);
+  }
+
+  const itemBreadcrumbs = [
+    { text: "Reader", href: request.baseUrl },
+  ];
+  if (channel) {
+    itemBreadcrumbs.push(
+      { text: "Channels", href: `${request.baseUrl}/channels` },
+      { text: channel.name, href: `${request.baseUrl}/channels/${channel.uid}` },
+    );
+  }
+  itemBreadcrumbs.push({ text: itemDocument.name || "Item" });
+
+  response.render("item", {
+    title: itemDocument.name || "Item",
+    item: itemDocument,
+    channel,
+    baseUrl: request.baseUrl,
+    readerBaseUrl: request.baseUrl,
+    activeView: "channels",
+    breadcrumbs: itemBreadcrumbs,
+  });
+}

package/lib/controllers/search.js

@@ -6,6 +6,7 @@
 import { IndiekitError } from "@indiekit/error";
 
 import { discoverFeeds } from "../feeds/hfeed.js";
+import { isPrivateUrl } from "../media/proxy.js";
 import { searchWithFallback } from "../search/query.js";
 import { getChannel } from "../storage/channels.js";
 import { getUserId } from "../utils/auth.js";
@@ -35,6 +36,11 @@ export async function discover(request, response) {
     return response.json({ results: [] });
   }
 
+  // SSRF protection
+  if (await isPrivateUrl(url.href)) {
+    throw new IndiekitError("URL blocked (private/internal address)", { status: 400 });
+  }
+
   try {
     // Fetch the URL content
     const fetchResponse = await fetch(url.href, {

package/lib/controllers/timeline.js

@@ -9,16 +9,18 @@ import { proxyItemImages } from "../media/proxy.js";
 import { getChannel, getChannelById } from "../storage/channels.js";
 import {
   getTimelineItems,
+  removeItems,
+} from "../storage/items.js";
+import {
   markFeedItemsRead,
   markItemsRead,
   markItemsUnread,
-  removeItems,
-} from "../storage/items.js";
+} from "../storage/items-read-state.js";
 import { getUserId } from "../utils/auth.js";
 import {
   validateChannel,
   validateEntries,
-  parseArrayParameter as parseArrayParametereter,
+  parseArrayParameter,
 } from "../utils/validation.js";
 
 /**
@@ -90,7 +92,7 @@ export async function action(request, response) {
   }
 
   // Get entry IDs from request
-  const entries = parseArrayParametereter(request.body, "entry");
+  const entries = parseArrayParameter(request.body, "entry");
 
   switch (method) {
     case "mark_read": {

package/lib/feeds/atom.js

@@ -7,7 +7,7 @@ import { Readable } from "node:stream";
 
 import FeedParser from "feedparser";
 
-import { normalizeItem, normalizeFeedMeta } from "./normalizer.js";
+import { normalizeItem, normalizeFeedMeta } from "./normalizer-rss.js";
 
 /**
  * Parse Atom feed content

package/lib/feeds/capabilities.js

@@ -4,6 +4,8 @@
  * @module feeds/capabilities
  */
 
+import { isPrivateUrl } from "../media/proxy.js";
+
 /**
  * Known Fediverse domain patterns
  */
@@ -136,6 +138,9 @@ async function discoverEndpoints(url) {
     micropub: null,
   };
 
+  // SSRF protection
+  if (await isPrivateUrl(url)) return endpoints;
+
   const controller = new AbortController();
   const timeout = setTimeout(() => controller.abort(), 15_000);
 

package/lib/feeds/fetcher.js

@@ -3,7 +3,7 @@
  * @module feeds/fetcher
  */
 
-import { getCache, setCache } from "../cache/redis.js";
+import { isPrivateUrl } from "../media/proxy.js";
 
 const DEFAULT_TIMEOUT = 30_000; // 30 seconds
 const DEFAULT_USER_AGENT = "Indiekit Microsub/1.0 (+https://getindiekit.com)";
@@ -15,25 +15,14 @@ const DEFAULT_USER_AGENT = "Indiekit Microsub/1.0 (+https://getindiekit.com)";
  * @param {string} [options.etag] - Previous ETag for conditional request
  * @param {string} [options.lastModified] - Previous Last-Modified for conditional request
  * @param {number} [options.timeout] - Request timeout in ms
- * @param {object} [options.redis] - Redis client for caching
  * @returns {Promise<object>} Fetch result with content and headers
  */
 export async function fetchFeed(url, options = {}) {
-  const { etag, lastModified, timeout = DEFAULT_TIMEOUT, redis } = options;
+  const { etag, lastModified, timeout = DEFAULT_TIMEOUT } = options;
 
-  // Check cache first
-  if (redis) {
-    const cached = await getCache(redis, `feed:${url}`);
-    if (cached) {
-      return {
-        content: cached.content,
-        contentType: cached.contentType,
-        etag: cached.etag,
-        lastModified: cached.lastModified,
-        fromCache: true,
-        status: 200,
-      };
-    }
+  // SSRF protection — block private/internal IPs (including DNS rebinding)
+  if (await isPrivateUrl(url)) {
+    throw new Error(`Feed URL blocked (private/internal address): ${url}`);
   }
 
   const headers = {
@@ -99,18 +88,6 @@ export async function fetchFeed(url, options = {}) {
       result.self = extractSelfFromLinkHeader(linkHeader);
     }
 
-    // Cache the result
-    if (redis) {
-      const cacheData = {
-        content,
-        contentType,
-        etag: responseEtag,
-        lastModified: responseLastModified,
-      };
-      // Cache for 5 minutes by default
-      await setCache(redis, `feed:${url}`, cacheData, 300);
-    }
-
     return result;
   } catch (error) {
     clearTimeout(timeoutId);

package/lib/feeds/hfeed.js

@@ -5,7 +5,7 @@
 
 import { mf2 } from "microformats-parser";
 
-import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer.js";
+import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer-hfeed.js";
 
 /**
  * Parse h-feed content from HTML

package/lib/feeds/jsonfeed.js

@@ -3,7 +3,7 @@
  * @module feeds/jsonfeed
  */
 
-import { normalizeJsonFeedItem, normalizeJsonFeedMeta } from "./normalizer.js";
+import { normalizeJsonFeedItem, normalizeJsonFeedMeta } from "./normalizer-jsonfeed.js";
 
 /**
  * Parse JSON Feed content

package/lib/feeds/normalizer-hfeed.js

@@ -0,0 +1,209 @@
+/**
+ * h-feed (Microformats2) normalization
+ * @module feeds/normalizer-hfeed
+ */
+
+import {
+  generateItemUid,
+  extractImagesFromHtml,
+  extractPhotoUrl,
+  normalizeUrlArray,
+  getFirst,
+  getContentText,
+  sanitizeHtml,
+  SANITIZE_OPTIONS,
+} from "./normalizer.js";
+
+/**
+ * Normalize h-card author
+ * @param {object|string} hcard - h-card or author name string
+ * @returns {object} Normalized author object
+ */
+function normalizeHcard(hcard) {
+  if (typeof hcard === "string") {
+    return { type: "card", name: hcard };
+  }
+
+  if (!hcard || !hcard.properties) {
+    return;
+  }
+
+  const properties = hcard.properties;
+
+  return {
+    type: "card",
+    name: getFirst(properties.name),
+    url: getFirst(properties.url),
+    photo: extractPhotoUrl(getFirst(properties.photo)),
+  };
+}
+
+/**
+ * Normalize h-feed entry
+ * @param {object} entry - Microformats h-entry
+ * @param {string} feedUrl - Feed URL
+ * @returns {object} Normalized jf2 item
+ */
+export function normalizeHfeedItem(entry, feedUrl) {
+  const properties = entry.properties || {};
+  const url = getFirst(properties.url) || getFirst(properties.uid);
+  const uid = generateItemUid(feedUrl, getFirst(properties.uid) || url);
+
+  const normalized = {
+    type: "entry",
+    uid,
+    url,
+    _source: {
+      url: feedUrl,
+      feedUrl,
+      feedType: "hfeed",
+      originalId: getFirst(properties.uid),
+    },
+  };
+
+  // Name/title
+  if (properties.name) {
+    const name = getFirst(properties.name);
+    // Only include name if it's not just the content
+    if (
+      name &&
+      (!properties.content || name !== getContentText(properties.content))
+    ) {
+      normalized.name = name;
+    }
+  }
+
+  // Published
+  if (properties.published) {
+    const published = getFirst(properties.published);
+    normalized.published = new Date(published).toISOString();
+  }
+
+  // Updated
+  if (properties.updated) {
+    const updated = getFirst(properties.updated);
+    normalized.updated = new Date(updated).toISOString();
+  }
+
+  // Content
+  if (properties.content) {
+    const content = getFirst(properties.content);
+    if (typeof content === "object") {
+      normalized.content = {
+        html: content.html
+          ? sanitizeHtml(content.html, SANITIZE_OPTIONS)
+          : undefined,
+        text: content.value || undefined,
+      };
+    } else if (typeof content === "string") {
+      normalized.content = { text: content };
+    }
+  }
+
+  // Summary
+  if (properties.summary) {
+    normalized.summary = getFirst(properties.summary);
+  }
+
+  // Author
+  if (properties.author) {
+    const author = getFirst(properties.author);
+    normalized.author = normalizeHcard(author);
+  }
+
+  // Categories
+  if (properties.category) {
+    normalized.category = properties.category;
+  }
+
+  // Photos
+  if (properties.photo) {
+    normalized.photo = properties.photo.map((p) =>
+      typeof p === "object" ? p.value || p.url : p,
+    );
+  }
+
+  // Videos
+  if (properties.video) {
+    normalized.video = properties.video.map((v) =>
+      typeof v === "object" ? v.value || v.url : v,
+    );
+  }
+
+  // Audio
+  if (properties.audio) {
+    normalized.audio = properties.audio.map((a) =>
+      typeof a === "object" ? a.value || a.url : a,
+    );
+  }
+
+  // Interaction types - normalize to string URLs
+  if (properties["like-of"]) {
+    normalized["like-of"] = normalizeUrlArray(properties["like-of"]);
+  }
+  if (properties["repost-of"]) {
+    normalized["repost-of"] = normalizeUrlArray(properties["repost-of"]);
+  }
+  if (properties["bookmark-of"]) {
+    normalized["bookmark-of"] = normalizeUrlArray(properties["bookmark-of"]);
+  }
+  if (properties["in-reply-to"]) {
+    normalized["in-reply-to"] = normalizeUrlArray(properties["in-reply-to"]);
+  }
+
+  // RSVP
+  if (properties.rsvp) {
+    normalized.rsvp = getFirst(properties.rsvp);
+  }
+
+  // Syndication
+  if (properties.syndication) {
+    normalized.syndication = properties.syndication;
+  }
+
+  // Extract images from HTML content as fallback
+  if (!normalized.photo && normalized.content?.html) {
+    const extracted = extractImagesFromHtml(normalized.content.html);
+    if (extracted.length > 0) {
+      normalized.photo = extracted;
+    }
+  }
+
+  return normalized;
+}
+
+/**
+ * Normalize h-feed metadata
+ * @param {object} hfeed - h-feed microformat object
+ * @param {string} feedUrl - Feed URL
+ * @returns {object} Normalized feed metadata
+ */
+export function normalizeHfeedMeta(hfeed, feedUrl) {
+  const properties = hfeed.properties || {};
+
+  const normalized = {
+    name: getFirst(properties.name) || feedUrl,
+  };
+
+  if (properties.summary) {
+    normalized.summary = getFirst(properties.summary);
+  }
+
+  if (properties.url) {
+    normalized.url = getFirst(properties.url);
+  }
+
+  if (properties.photo) {
+    normalized.photo = getFirst(properties.photo);
+    if (typeof normalized.photo === "object") {
+      normalized.photo = normalized.photo.value || normalized.photo.url;
+    }
+  }
+
+  if (properties.author) {
+    const author = getFirst(properties.author);
+    normalized.author = normalizeHcard(author);
+  }
+
+  return normalized;
+}