@rmdes/indiekit-endpoint-microsub 1.0.0-beta.9 → 1.0.1

package/lib/feeds/fetcher.js

@@ -150,7 +150,7 @@ function extractSelfFromLinkHeader(linkHeader) {
  * @returns {Promise<object>} Parsed feed
  */
 export async function fetchAndParseFeed(url, options = {}) {
-  const { parseFeed } = await import("./parser.js");
+  const { parseFeed, detectFeedType } = await import("./parser.js");
 
   const result = await fetchFeed(url, options);
 
@@ -161,6 +161,32 @@ export async function fetchAndParseFeed(url, options = {}) {
     };
   }
 
+  // Check if we got a parseable feed
+  const feedType = detectFeedType(result.content, result.contentType);
+
+  // If we got ActivityPub or unknown, try common feed paths
+  if (feedType === "activitypub" || feedType === "unknown") {
+    const fallbackFeed = await tryCommonFeedPaths(url, options);
+    if (fallbackFeed) {
+      // Fetch and parse the discovered feed
+      const feedResult = await fetchFeed(fallbackFeed.url, options);
+      if (!feedResult.notModified) {
+        const parsed = await parseFeed(feedResult.content, fallbackFeed.url, {
+          contentType: feedResult.contentType,
+        });
+        return {
+          ...feedResult,
+          ...parsed,
+          hub: feedResult.hub || parsed._hub,
+          discoveredFrom: url,
+        };
+      }
+    }
+    throw new Error(
+      `Unable to find a feed at ${url}. Try the direct feed URL.`,
+    );
+  }
+
   const parsed = await parseFeed(result.content, url, {
     contentType: result.contentType,
   });
@@ -172,6 +198,48 @@ export async function fetchAndParseFeed(url, options = {}) {
   };
 }
 
+/**
+ * Common feed paths to try when discovery fails
+ */
+const COMMON_FEED_PATHS = ["/feed/", "/feed", "/rss", "/rss.xml", "/atom.xml"];
+
+/**
+ * Try to fetch a feed from common paths
+ * @param {string} baseUrl - Base URL of the site
+ * @param {object} options - Fetch options
+ * @returns {Promise<object|undefined>} Feed result or undefined
+ */
+async function tryCommonFeedPaths(baseUrl, options = {}) {
+  const base = new URL(baseUrl);
+
+  for (const feedPath of COMMON_FEED_PATHS) {
+    const feedUrl = new URL(feedPath, base).href;
+    try {
+      const result = await fetchFeed(feedUrl, { ...options, timeout: 10_000 });
+      const contentType = result.contentType?.toLowerCase() || "";
+
+      // Check if we got a feed
+      if (
+        contentType.includes("xml") ||
+        contentType.includes("rss") ||
+        contentType.includes("atom") ||
+        (contentType.includes("json") &&
+          result.content?.includes("jsonfeed.org"))
+      ) {
+        return {
+          url: feedUrl,
+          type: contentType.includes("json") ? "jsonfeed" : "xml",
+          rel: "alternate",
+        };
+      }
+    } catch {
+      // Try next path
+    }
+  }
+
+  return;
+}
+
 /**
  * Discover feeds from a URL
  * @param {string} url - Page URL
@@ -187,19 +255,62 @@ export async function discoverFeedsFromUrl(url, options = {}) {
   if (
     contentType.includes("xml") ||
     contentType.includes("rss") ||
-    contentType.includes("atom") ||
-    contentType.includes("json")
+    contentType.includes("atom")
   ) {
     return [
       {
         url,
-        type: contentType.includes("json") ? "jsonfeed" : "xml",
+        type: "xml",
         rel: "self",
       },
     ];
   }
 
-  // Otherwise, discover feeds from HTML
-  const feeds = await discoverFeeds(result.content, url);
-  return feeds;
+  // Check for JSON Feed specifically
+  if (
+    contentType.includes("json") &&
+    result.content?.includes("jsonfeed.org")
+  ) {
+    return [
+      {
+        url,
+        type: "jsonfeed",
+        rel: "self",
+      },
+    ];
+  }
+
+  // Check if we got ActivityPub JSON or other non-feed JSON
+  // This happens with WordPress sites using ActivityPub plugin
+  if (
+    contentType.includes("json") ||
+    (result.content?.trim().startsWith("{") &&
+      result.content?.includes("@context"))
+  ) {
+    // Try common feed paths as fallback
+    const fallbackFeed = await tryCommonFeedPaths(url, options);
+    if (fallbackFeed) {
+      return [fallbackFeed];
+    }
+  }
+
+  // If content looks like HTML, discover feeds from it
+  if (
+    contentType.includes("html") ||
+    result.content?.includes("<!DOCTYPE html") ||
+    result.content?.includes("<html")
+  ) {
+    const feeds = await discoverFeeds(result.content, url);
+    if (feeds.length > 0) {
+      return feeds;
+    }
+  }
+
+  // Last resort: try common feed paths
+  const fallbackFeed = await tryCommonFeedPaths(url, options);
+  if (fallbackFeed) {
+    return [fallbackFeed];
+  }
+
+  return [];
 }

package/lib/feeds/normalizer.js

@@ -7,6 +7,66 @@ import crypto from "node:crypto";
 
 import sanitizeHtml from "sanitize-html";
 
+/**
+ * Parse a date string with fallback for non-standard formats
+ * @param {string|Date} dateInput - Date string or Date object
+ * @returns {Date|undefined} Parsed Date or undefined if invalid
+ */
+function parseDate(dateInput) {
+  if (!dateInput) {
+    return;
+  }
+
+  // Already a valid Date
+  if (dateInput instanceof Date && !Number.isNaN(dateInput.getTime())) {
+    return dateInput;
+  }
+
+  const dateString = String(dateInput).trim();
+
+  // Try standard parsing first
+  let date = new Date(dateString);
+  if (!Number.isNaN(date.getTime())) {
+    return date;
+  }
+
+  // Handle "YYYY-MM-DD HH:MM" format (missing seconds and timezone)
+  // e.g., "2026-01-28 08:40"
+  const shortDateTime = dateString.match(
+    /^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2})$/,
+  );
+  if (shortDateTime) {
+    date = new Date(`${shortDateTime[1]}T${shortDateTime[2]}:00Z`);
+    if (!Number.isNaN(date.getTime())) {
+      return date;
+    }
+  }
+
+  // Handle "YYYY-MM-DD HH:MM:SS" without timezone
+  const dateTimeNoTz = dateString.match(
+    /^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2}:\d{2})$/,
+  );
+  if (dateTimeNoTz) {
+    date = new Date(`${dateTimeNoTz[1]}T${dateTimeNoTz[2]}Z`);
+    if (!Number.isNaN(date.getTime())) {
+      return date;
+    }
+  }
+
+  // If all else fails, return undefined
+  return;
+}
+
+/**
+ * Safely convert date to ISO string
+ * @param {string|Date} dateInput - Date input
+ * @returns {string|undefined} ISO string or undefined
+ */
+function toISOStringSafe(dateInput) {
+  const date = parseDate(dateInput);
+  return date ? date.toISOString() : undefined;
+}
+
 /**
  * Sanitize HTML options
  */
@@ -91,8 +151,8 @@ export function normalizeItem(item, feedUrl, feedType) {
     uid,
     url,
     name: item.title || undefined,
-    published: item.pubdate ? new Date(item.pubdate).toISOString() : undefined,
-    updated: item.date ? new Date(item.date).toISOString() : undefined,
+    published: toISOStringSafe(item.pubdate),
+    updated: toISOStringSafe(item.date),
     _source: {
       feedUrl,
       feedType,

package/lib/feeds/parser.js

@@ -51,13 +51,18 @@ export function detectFeedType(content, contentType = "") {
   // Fall back to content inspection
   const trimmed = content.trim();
 
-  // JSON Feed
+  // JSON content
   if (trimmed.startsWith("{")) {
     try {
       const json = JSON.parse(trimmed);
+      // JSON Feed
       if (json.version && json.version.includes("jsonfeed.org")) {
         return "jsonfeed";
       }
+      // ActivityPub - return special type to indicate we need feed discovery
+      if (json["@context"] || json.type === "Group" || json.inbox) {
+        return "activitypub";
+      }
     } catch {
       // Not JSON
     }
@@ -112,6 +117,12 @@ export async function parseFeed(content, feedUrl, options = {}) {
       return parseHfeed(content, feedUrl);
     }
 
+    case "activitypub": {
+      throw new Error(
+        `URL returns ActivityPub JSON instead of a feed. Try the direct feed URL (e.g., ${feedUrl}feed/)`,
+      );
+    }
+
     default: {
       throw new Error(`Unable to detect feed type for ${feedUrl}`);
     }

package/lib/media/proxy.js

@@ -0,0 +1,219 @@
+/**
+ * Media proxy with caching
+ * @module media/proxy
+ */
+
+import crypto from "node:crypto";
+
+import { getCache, setCache } from "../cache/redis.js";
+
+const MAX_SIZE = 2 * 1024 * 1024; // 2MB max image size
+const CACHE_TTL = 4 * 60 * 60; // 4 hours
+const ALLOWED_TYPES = new Set([
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/svg+xml",
+]);
+
+/**
+ * Generate a hash for a URL to use as cache key
+ * @param {string} url - Original image URL
+ * @returns {string} URL-safe hash
+ */
+export function hashUrl(url) {
+  return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
+}
+
+/**
+ * Get the proxied URL for an image
+ * @param {string} baseUrl - Base URL of the Microsub endpoint
+ * @param {string} originalUrl - Original image URL
+ * @returns {string} Proxied URL
+ */
+export function getProxiedUrl(baseUrl, originalUrl) {
+  if (!originalUrl || !baseUrl) {
+    return originalUrl;
+  }
+
+  // Skip data URLs
+  if (originalUrl.startsWith("data:")) {
+    return originalUrl;
+  }
+
+  // Skip already-proxied URLs
+  if (originalUrl.includes("/microsub/media/")) {
+    return originalUrl;
+  }
+
+  const hash = hashUrl(originalUrl);
+  return `${baseUrl}/microsub/media/${hash}?url=${encodeURIComponent(originalUrl)}`;
+}
+
+/**
+ * Rewrite image URLs in an item to use the proxy
+ * @param {object} item - JF2 item
+ * @param {string} baseUrl - Base URL for proxy
+ * @returns {object} Item with proxied URLs
+ */
+export function proxyItemImages(item, baseUrl) {
+  if (!baseUrl || !item) {
+    return item;
+  }
+
+  const proxied = { ...item };
+
+  // Proxy photo URLs
+  if (proxied.photo) {
+    if (Array.isArray(proxied.photo)) {
+      proxied.photo = proxied.photo.map((p) => {
+        if (typeof p === "string") {
+          return getProxiedUrl(baseUrl, p);
+        }
+        if (p?.value) {
+          return { ...p, value: getProxiedUrl(baseUrl, p.value) };
+        }
+        return p;
+      });
+    } else if (typeof proxied.photo === "string") {
+      proxied.photo = getProxiedUrl(baseUrl, proxied.photo);
+    }
+  }
+
+  // Proxy author photo
+  if (proxied.author?.photo) {
+    proxied.author = {
+      ...proxied.author,
+      photo: getProxiedUrl(baseUrl, proxied.author.photo),
+    };
+  }
+
+  return proxied;
+}
+
+/**
+ * Fetch and cache an image
+ * @param {object} redis - Redis client
+ * @param {string} url - Image URL to fetch
+ * @returns {Promise<object|null>} Cached image data or null
+ */
+export async function fetchImage(redis, url) {
+  const cacheKey = `media:${hashUrl(url)}`;
+
+  // Try cache first
+  if (redis) {
+    const cached = await getCache(redis, cacheKey);
+    if (cached) {
+      return cached;
+    }
+  }
+
+  try {
+    // Fetch the image
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)",
+        Accept: "image/*",
+      },
+      signal: AbortSignal.timeout(10_000), // 10 second timeout
+    });
+
+    if (!response.ok) {
+      console.error(
+        `[Microsub] Media proxy fetch failed: ${response.status} for ${url}`,
+      );
+      return;
+    }
+
+    // Check content type
+    const contentType = response.headers.get("content-type")?.split(";")[0];
+    if (!ALLOWED_TYPES.has(contentType)) {
+      console.error(
+        `[Microsub] Media proxy rejected type: ${contentType} for ${url}`,
+      );
+      return;
+    }
+
+    // Check content length
+    const contentLength = Number.parseInt(
+      response.headers.get("content-length") || "0",
+      10,
+    );
+    if (contentLength > MAX_SIZE) {
+      console.error(
+        `[Microsub] Media proxy rejected size: ${contentLength} for ${url}`,
+      );
+      return;
+    }
+
+    // Read the body
+    const buffer = await response.arrayBuffer();
+    if (buffer.byteLength > MAX_SIZE) {
+      return;
+    }
+
+    const imageData = {
+      contentType,
+      data: Buffer.from(buffer).toString("base64"),
+      size: buffer.byteLength,
+    };
+
+    // Cache in Redis
+    if (redis) {
+      await setCache(redis, cacheKey, imageData, CACHE_TTL);
+    }
+
+    return imageData;
+  } catch (error) {
+    console.error(`[Microsub] Media proxy error: ${error.message} for ${url}`);
+    return;
+  }
+}
+
+/**
+ * Express route handler for media proxy
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function handleMediaProxy(request, response) {
+  const { url } = request.query;
+
+  if (!url) {
+    return response.status(400).send("Missing url parameter");
+  }
+
+  // Validate URL
+  try {
+    const parsed = new URL(url);
+    if (!["http:", "https:"].includes(parsed.protocol)) {
+      return response.status(400).send("Invalid URL protocol");
+    }
+  } catch {
+    return response.status(400).send("Invalid URL");
+  }
+
+  // Get Redis client from application
+  const { application } = request.app.locals;
+  const redis = application.redis;
+
+  // Fetch or get from cache
+  const imageData = await fetchImage(redis, url);
+
+  if (!imageData) {
+    // Redirect to original URL as fallback
+    return response.redirect(url);
+  }
+
+  // Set cache headers
+  response.set({
+    "Content-Type": imageData.contentType,
+    "Content-Length": imageData.size,
+    "Cache-Control": "public, max-age=14400", // 4 hours
+    "X-Proxied-From": url,
+  });
+
+  // Send the image
+  response.send(Buffer.from(imageData.data, "base64"));
+}

package/lib/polling/processor.js

@@ -9,6 +9,10 @@ import { getChannel } from "../storage/channels.js";
 import { updateFeedAfterFetch, updateFeedWebsub } from "../storage/feeds.js";
 import { passesRegexFilter, passesTypeFilter } from "../storage/filters.js";
 import { addItem } from "../storage/items.js";
+import {
+  subscribe as websubSubscribe,
+  getCallbackUrl,
+} from "../websub/subscriber.js";
 
 import { calculateNewTier } from "./tier.js";
 
@@ -123,13 +127,37 @@ export async function processFeed(application, feed) {
       updateData,
     );
 
-    // Handle WebSub hub discovery
+    // Handle WebSub hub discovery and auto-subscription
     if (parsed.hub && (!feed.websub || feed.websub.hub !== parsed.hub)) {
       await updateFeedWebsub(application, feed._id, {
         hub: parsed.hub,
         topic: parsed.self || feed.url,
       });
-      // TODO: Subscribe to hub
+
+      // Auto-subscribe to WebSub hub if we have a callback URL
+      const baseUrl = application.url;
+      if (baseUrl) {
+        const callbackUrl = getCallbackUrl(baseUrl, feed._id.toString());
+        const updatedFeed = {
+          ...feed,
+          websub: { hub: parsed.hub, topic: parsed.self || feed.url },
+        };
+
+        websubSubscribe(application, updatedFeed, callbackUrl)
+          .then((subscribed) => {
+            if (subscribed) {
+              console.info(
+                `[Microsub] WebSub subscription initiated for ${feed.url}`,
+              );
+            }
+          })
+          .catch((error) => {
+            console.error(
+              `[Microsub] WebSub subscription error for ${feed.url}:`,
+              error.message,
+            );
+          });
+      }
     }
 
     result.success = true;

package/lib/storage/channels.js

@@ -7,6 +7,9 @@ import { ObjectId } from "mongodb";
 
 import { generateChannelUid } from "../utils/jf2.js";
 
+import { deleteFeedsForChannel } from "./feeds.js";
+import { deleteItemsForChannel } from "./items.js";
+
 /**
  * Get channels collection from application
  * @param {object} application - Indiekit application
@@ -184,7 +187,7 @@ export async function updateChannel(application, uid, updates, userId) {
 }
 
 /**
- * Delete a channel
+ * Delete a channel and all its feeds and items
  * @param {object} application - Indiekit application
  * @param {string} uid - Channel UID
  * @param {string} [userId] - User ID
@@ -200,7 +203,20 @@ export async function deleteChannel(application, uid, userId) {
     return false;
   }
 
-  const result = await collection.deleteOne(query);
+  // Find the channel first to get its ObjectId
+  const channel = await collection.findOne(query);
+  if (!channel) {
+    return false;
+  }
+
+  // Cascade delete: items first, then feeds, then channel
+  const itemsDeleted = await deleteItemsForChannel(application, channel._id);
+  const feedsDeleted = await deleteFeedsForChannel(application, channel._id);
+  console.info(
+    `[Microsub] Deleted channel ${uid}: ${feedsDeleted} feeds, ${itemsDeleted} items`,
+  );
+
+  const result = await collection.deleteOne({ _id: channel._id });
   return result.deletedCount > 0;
 }
 

package/lib/storage/feeds.js

@@ -5,6 +5,8 @@
 
 import { ObjectId } from "mongodb";
 
+import { deleteItemsForFeed } from "./items.js";
+
 /**
  * Get feeds collection from application
  * @param {object} application - Indiekit application
@@ -122,7 +124,7 @@ export async function updateFeed(application, id, updates) {
 }
 
 /**
- * Delete a feed subscription
+ * Delete a feed subscription and all its items
  * @param {object} application - Indiekit application
  * @param {ObjectId|string} channelId - Channel ObjectId
  * @param {string} url - Feed URL
@@ -133,7 +135,18 @@ export async function deleteFeed(application, channelId, url) {
   const objectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
-  const result = await collection.deleteOne({ channelId: objectId, url });
+  // Find the feed first to get its ID for cascade delete
+  const feed = await collection.findOne({ channelId: objectId, url });
+  if (!feed) {
+    return false;
+  }
+
+  // Delete all items from this feed
+  const itemsDeleted = await deleteItemsForFeed(application, feed._id);
+  console.info(`[Microsub] Deleted ${itemsDeleted} items from feed ${url}`);
+
+  // Delete the feed itself
+  const result = await collection.deleteOne({ _id: feed._id });
   return result.deletedCount > 0;
 }