@rmdes/indiekit-endpoint-microsub 1.0.55 → 1.0.57

package/lib/media/proxy.js

@@ -4,6 +4,7 @@
  */
 
 import crypto from "node:crypto";
+import dns from "node:dns/promises";
 
 import { getCache, setCache } from "../cache/redis.js";
 
@@ -20,39 +21,59 @@ const BLOCKED_IP_PREFIXES = [
 ];
 
 /**
- * Check if a hostname resolves to a private/internal address
+ * Check if an IP address is in a private/internal range
+ * @param {string} ip - IP address to check
+ * @returns {boolean} True if private
+ */
+function isPrivateIp(ip) {
+  if (ip === "::1" || ip === "127.0.0.1") return true;
+
+  for (const prefix of BLOCKED_IP_PREFIXES) {
+    if (ip.startsWith(prefix)) return true;
+  }
+
+  // 172.16.0.0/12
+  const match172 = ip.match(/^172\.(\d+)\./);
+  if (match172) {
+    const second = Number.parseInt(match172[1], 10);
+    if (second >= 16 && second <= 31) return true;
+  }
+
+  return false;
+}
+
+/**
+ * Check if a URL targets a private/internal address.
+ * Performs both string-based hostname checks AND DNS resolution
+ * to prevent DNS rebinding attacks.
+ *
  * @param {string} urlString - URL to check
- * @returns {boolean} True if the URL targets a private/internal address
+ * @returns {Promise<boolean>} True if the URL targets a private/internal address
  */
-export function isPrivateUrl(urlString) {
+export async function isPrivateUrl(urlString) {
   try {
     const parsed = new URL(urlString);
     const hostname = parsed.hostname;
 
+    // Block non-HTTP protocols
+    if (!["http:", "https:"].includes(parsed.protocol)) return true;
+
     // Block known private hostnames
-    if (BLOCKED_HOSTNAMES.has(hostname)) {
-      return true;
-    }
+    if (BLOCKED_HOSTNAMES.has(hostname)) return true;
 
     // Block IPv6 loopback
-    if (hostname === "::1" || hostname === "[::1]") {
-      return true;
-    }
+    if (hostname === "::1" || hostname === "[::1]") return true;
 
-    // Block private IPv4 ranges
-    for (const prefix of BLOCKED_IP_PREFIXES) {
-      if (hostname.startsWith(prefix)) {
-        return true;
-      }
-    }
+    // Block private IPv4 ranges (string check for literal IPs)
+    if (isPrivateIp(hostname)) return true;
 
-    // Block 172.16.0.0/12 (172.16.x.x - 172.31.x.x)
-    const match172 = hostname.match(/^172\.(\d+)\./);
-    if (match172) {
-      const second = Number.parseInt(match172[1], 10);
-      if (second >= 16 && second <= 31) {
-        return true;
-      }
+    // DNS resolution check — catches domains resolving to private IPs
+    try {
+      const { address } = await dns.lookup(hostname);
+      if (isPrivateIp(address)) return true;
+    } catch {
+      // DNS resolution failure — block as precaution
+      return true;
     }
 
     return false;
@@ -68,8 +89,8 @@ const ALLOWED_TYPES = new Set([
   "image/png",
   "image/gif",
   "image/webp",
-  "image/svg+xml",
   "image/avif",
+  // image/svg+xml intentionally excluded — SVGs can contain embedded JavaScript
 ]);
 
 /**
@@ -81,6 +102,34 @@ export function hashUrl(url) {
   return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
 }
 
+/**
+ * Generate HMAC signature for a media proxy URL
+ * @param {string} url - Original image URL
+ * @returns {string} HMAC hex signature (16 chars)
+ */
+export function signProxyUrl(url) {
+  const secret = process.env.SECRET || "microsub-default-secret";
+  return crypto
+    .createHmac("sha256", secret)
+    .update(url)
+    .digest("hex")
+    .slice(0, 16);
+}
+
+/**
+ * Verify HMAC signature for a media proxy URL
+ * @param {string} url - Original image URL
+ * @param {string} sig - Submitted signature
+ * @returns {boolean} Whether signature is valid
+ */
+export function verifyProxySignature(url, sig) {
+  if (!sig) return false;
+  const expected = signProxyUrl(url);
+  // Constant-time comparison
+  if (sig.length !== expected.length) return false;
+  return crypto.timingSafeEqual(Buffer.from(sig), Buffer.from(expected));
+}
+
 /**
  * Get the proxied URL for an image
  * @param {string} baseUrl - Base URL of the Microsub endpoint
@@ -103,7 +152,8 @@ export function getProxiedUrl(baseUrl, originalUrl) {
   }
 
   const hash = hashUrl(originalUrl);
-  return `${baseUrl}/microsub/media/${hash}?url=${encodeURIComponent(originalUrl)}`;
+  const sig = signProxyUrl(originalUrl);
+  return `${baseUrl}/microsub/media/${hash}?url=${encodeURIComponent(originalUrl)}&sig=${sig}`;
 }
 
 /**
@@ -155,7 +205,7 @@ export function proxyItemImages(item, baseUrl) {
  */
 export async function fetchImage(redis, url) {
   // Block private/internal URLs (defense-in-depth)
-  if (isPrivateUrl(url)) {
+  if (await isPrivateUrl(url)) {
     console.error(`[Microsub] Media proxy blocked private URL: ${url}`);
     return;
   }
@@ -239,12 +289,17 @@ export async function fetchImage(redis, url) {
  * @returns {Promise<void>}
  */
 export async function handleMediaProxy(request, response) {
-  const { url } = request.query;
+  const { url, sig } = request.query;
 
   if (!url) {
     return response.status(400).send("Missing url parameter");
   }
 
+  // Verify HMAC signature (prevents abuse as open proxy)
+  if (!verifyProxySignature(url, sig)) {
+    return response.status(403).send("Invalid proxy signature");
+  }
+
   // Validate URL
   try {
     const parsed = new URL(url);
@@ -256,7 +311,7 @@ export async function handleMediaProxy(request, response) {
   }
 
   // Block requests to private/internal networks (SSRF protection)
-  if (isPrivateUrl(url)) {
+  if (await isPrivateUrl(url)) {
     return response.status(403).send("URL not allowed");
   }
 

package/lib/polling/processor.js

@@ -3,10 +3,13 @@
  * @module polling/processor
  */
 
+const FEED_PROCESS_TIMEOUT = 60_000; // 60 seconds max per feed
+const MAX_ITEMS_PER_CYCLE = 100; // Max items to process per feed per cycle
+
 import { getRedisClient, publishEvent } from "../cache/redis.js";
 import { detectCapabilities } from "../feeds/capabilities.js";
 import { fetchAndParseFeed } from "../feeds/fetcher.js";
-import { getChannel } from "../storage/channels.js";
+import { getChannelById } from "../storage/channels.js";
 import {
   updateFeed,
   updateFeedAfterFetch,
@@ -15,6 +18,7 @@ import {
 } from "../storage/feeds.js";
 import { passesRegexFilter, passesTypeFilter } from "../storage/filters.js";
 import { addItem } from "../storage/items.js";
+import { classifyUrl } from "../utils/source-type.js";
 import {
   subscribe as websubSubscribe,
   getCallbackUrl,
@@ -40,7 +44,7 @@ export async function processFeed(application, feed) {
 
   try {
     // Get Redis client for caching
-    const redis = getRedisClient(application);
+    const redis = await getRedisClient(application);
 
     // Fetch and parse the feed
     const parsed = await fetchAndParseFeed(feed.url, {
@@ -69,11 +73,14 @@ export async function processFeed(application, feed) {
     }
 
     // Get channel for filtering
-    const channel = await getChannel(application, feed.channelId);
+    const channel = await getChannelById(application, feed.channelId);
 
-    // Process items
+    // Process items (limited to MAX_ITEMS_PER_CYCLE per feed per cycle)
     let newItemCount = 0;
+    let processedCount = 0;
     for (const item of parsed.items) {
+      if (processedCount >= MAX_ITEMS_PER_CYCLE) break;
+      processedCount++;
       // Apply channel filters
       if (channel?.settings && !passesFilters(item, channel.settings)) {
         continue;
@@ -90,7 +97,7 @@ export async function processFeed(application, feed) {
       if (feed.capabilities?.source_type) {
         item._source.source_type = feed.capabilities.source_type;
       } else {
-        item._source.source_type = inferSourceType(feed.url);
+        item._source.source_type = classifyUrl(feed.url).type;
       }
 
       // Store the item
@@ -236,21 +243,6 @@ export async function processFeed(application, feed) {
   return result;
 }
 
-/**
- * Infer source type from feed URL when capabilities haven't been detected yet
- * @param {string} url - Feed URL
- * @returns {string} Source type
- */
-function inferSourceType(url) {
-  if (!url) return "web";
-  const lower = url.toLowerCase();
-  if (lower.includes("bsky.app") || lower.includes("bluesky")) return "bluesky";
-  if (lower.includes("mastodon.") || lower.includes("mstdn.") ||
-      lower.includes("fosstodon.") || lower.includes("pleroma.") ||
-      lower.includes("misskey.") || lower.includes("pixelfed.")) return "mastodon";
-  return "web";
-}
-
 /**
  * Check if an item passes channel filters
  * @param {object} item - Feed item
@@ -276,7 +268,24 @@ export async function processFeedBatch(application, feeds, options = {}) {
   for (let index = 0; index < feeds.length; index += concurrency) {
     const batch = feeds.slice(index, index + concurrency);
     const batchResults = await Promise.all(
-      batch.map((feed) => processFeed(application, feed)),
+      batch.map((feed) =>
+        Promise.race([
+          processFeed(application, feed),
+          new Promise((resolve) =>
+            setTimeout(
+              () =>
+                resolve({
+                  feedId: feed._id,
+                  url: feed.url,
+                  success: false,
+                  itemsAdded: 0,
+                  error: "Feed processing timeout",
+                }),
+              FEED_PROCESS_TIMEOUT,
+            ),
+          ),
+        ]),
+      ),
     );
     results.push(...batchResults);
   }

package/lib/polling/scheduler.js

@@ -7,6 +7,8 @@ import { getFeedsToFetch } from "../storage/feeds.js";
 
 import { processFeedBatch } from "./processor.js";
 
+// TODO: Refactor scheduler to a class that accepts `application` as constructor
+// argument. Module-level singletons prevent unit testing and multiple instances.
 let schedulerInterval;
 let indiekitInstance;
 let isRunning = false;

package/lib/realtime/broker.js

@@ -62,7 +62,12 @@ export function removeClient(response) {
       (c) => c.userId === client.userId,
     );
     if (!hasOtherClients) {
-      // Could clean up Redis subscription here if needed
+      // Clean up Redis subscriber connection for this user
+      const subscriber = userSubscribers.get(client.userId);
+      if (subscriber) {
+        subscriber.quit().catch(() => {});
+        userSubscribers.delete(client.userId);
+      }
     }
   }
 }

package/lib/storage/channels.js

@@ -112,8 +112,35 @@ export async function createChannel(application, { name, userId }) {
   return channel;
 }
 
-// Retention period for unread count (only count recent items)
-const UNREAD_RETENTION_DAYS = 30;
+import { UNREAD_RETENTION_DAYS } from "../utils/constants.js";
+
+/**
+ * Get unread counts for multiple channels in a single aggregation query.
+ * Replaces the N+1 countDocuments pattern.
+ * @param {object} itemsCollection - MongoDB items collection
+ * @param {Array<import("mongodb").ObjectId>} channelIds - Channel IDs
+ * @param {string} userId - User ID
+ * @returns {Promise<Map<string, number>>} Map of channelId string → unread count
+ */
+async function getUnreadCounts(itemsCollection, channelIds, userId) {
+  const cutoffDate = new Date();
+  cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
+
+  const pipeline = [
+    {
+      $match: {
+        channelId: { $in: channelIds },
+        readBy: { $ne: userId },
+        published: { $gte: cutoffDate },
+        _stripped: { $ne: true },
+      },
+    },
+    { $group: { _id: "$channelId", count: { $sum: 1 } } },
+  ];
+
+  const results = await itemsCollection.aggregate(pipeline).toArray();
+  return new Map(results.map((r) => [r._id.toString(), r.count]));
+}
 
 /**
  * Get all channels for a user
@@ -133,27 +160,18 @@ export async function getChannels(application, userId) {
     .sort({ order: 1 })
     .toArray();
 
-  // Calculate cutoff date for unread counts (only count recent items)
-  const cutoffDate = new Date();
-  cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
-
-  // Get unread counts for each channel (only recent items)
-  const channelsWithCounts = await Promise.all(
-    channels.map(async (channel) => {
-      const unreadCount = await itemsCollection.countDocuments({
-        channelId: channel._id,
-        readBy: { $ne: userId },
-        published: { $gte: cutoffDate },
-        _stripped: { $ne: true },
-      });
-
-      return {
-        uid: channel.uid,
-        name: channel.name,
-        unread: unreadCount > 0 ? unreadCount : false,
-      };
-    }),
-  );
+  // Single aggregation query for all channel unread counts
+  const channelIds = channels.map((c) => c._id);
+  const unreadMap = await getUnreadCounts(itemsCollection, channelIds, userId);
+
+  const channelsWithCounts = channels.map((channel) => {
+    const unreadCount = unreadMap.get(channel._id.toString()) || 0;
+    return {
+      uid: channel.uid,
+      name: channel.name,
+      unread: unreadCount > 0 ? unreadCount : false,
+    };
+  });
 
   // Always include notifications channel first
   const notificationsChannel = channelsWithCounts.find(
@@ -189,25 +207,18 @@ export async function getChannelsWithColors(application, userId) {
     .sort({ order: 1 })
     .toArray();
 
-  const cutoffDate = new Date();
-  cutoffDate.setDate(cutoffDate.getDate() - UNREAD_RETENTION_DAYS);
-
-  const enriched = await Promise.all(
-    channels.map(async (channel, index) => {
-      const unreadCount = await itemsCollection.countDocuments({
-        channelId: channel._id,
-        readBy: { $ne: userId },
-        published: { $gte: cutoffDate },
-        _stripped: { $ne: true },
-      });
-
-      return {
-        ...channel,
-        color: channel.color || getChannelColor(index),
-        unread: unreadCount > 0 ? unreadCount : false,
-      };
-    }),
-  );
+  // Single aggregation query for all channel unread counts
+  const channelIds = channels.map((c) => c._id);
+  const unreadMap = await getUnreadCounts(itemsCollection, channelIds, userId);
+
+  const enriched = channels.map((channel, index) => {
+    const unreadCount = unreadMap.get(channel._id.toString()) || 0;
+    return {
+      ...channel,
+      color: channel.color || getChannelColor(index),
+      unread: unreadCount > 0 ? unreadCount : false,
+    };
+  });
 
   // Notifications first, then by order
   const notifications = enriched.find((c) => c.uid === "notifications");

package/lib/storage/feeds.js

@@ -249,7 +249,7 @@ export async function deleteFeedsForChannel(application, channelId) {
  * @param {object} application - Indiekit application
  * @returns {Promise<Array>} Array of feeds to fetch
  */
-export async function getFeedsToFetch(application) {
+export async function getFeedsToFetch(application, limit = 25) {
   const collection = getCollection(application);
   const now = new Date();
 
@@ -257,6 +257,8 @@ export async function getFeedsToFetch(application) {
     .find({
       $or: [{ nextFetchAt: undefined }, { nextFetchAt: { $lte: now } }],
     })
+    .sort({ nextFetchAt: 1 })
+    .limit(limit)
     .toArray();
 }