@rmdes/indiekit-endpoint-microsub 1.0.0-beta.8 → 1.0.0

package/lib/controllers/timeline.js

@@ -5,6 +5,7 @@
 
 import { IndiekitError } from "@indiekit/error";
 
+import { proxyItemImages } from "../media/proxy.js";
 import { getChannel } from "../storage/channels.js";
 import {
   getTimelineItems,
@@ -12,6 +13,7 @@ import {
   markItemsUnread,
   removeItems,
 } from "../storage/items.js";
+import { getUserId } from "../utils/auth.js";
 import {
   validateChannel,
   validateEntries,
@@ -26,7 +28,7 @@ import {
  */
 export async function get(request, response) {
   const { application } = request.app.locals;
-  const userId = request.session?.userId;
+  const userId = getUserId(request);
   const { channel, before, after, limit } = request.query;
 
   validateChannel(channel);
@@ -46,6 +48,14 @@ export async function get(request, response) {
     userId,
   });
 
+  // Proxy images if application URL is available
+  const baseUrl = application.url;
+  if (baseUrl && timeline.items) {
+    timeline.items = timeline.items.map((item) =>
+      proxyItemImages(item, baseUrl),
+    );
+  }
+
   response.json(timeline);
 }
 
@@ -57,7 +67,7 @@ export async function get(request, response) {
  */
 export async function action(request, response) {
   const { application } = request.app.locals;
-  const userId = request.session?.userId;
+  const userId = getUserId(request);
   const { method, channel } = request.body;
 
   validateChannel(channel);

package/lib/media/proxy.js

@@ -0,0 +1,213 @@
+/**
+ * Media proxy with caching
+ * @module media/proxy
+ */
+
+import crypto from "node:crypto";
+
+import { getCache, setCache } from "../cache/redis.js";
+
+const MAX_SIZE = 2 * 1024 * 1024; // 2MB max image size
+const CACHE_TTL = 4 * 60 * 60; // 4 hours
+const ALLOWED_TYPES = [
+  "image/jpeg",
+  "image/png",
+  "image/gif",
+  "image/webp",
+  "image/svg+xml",
+];
+
+/**
+ * Generate a hash for a URL to use as cache key
+ * @param {string} url - Original image URL
+ * @returns {string} URL-safe hash
+ */
+export function hashUrl(url) {
+  return crypto.createHash("sha256").update(url).digest("hex").slice(0, 16);
+}
+
+/**
+ * Get the proxied URL for an image
+ * @param {string} baseUrl - Base URL of the Microsub endpoint
+ * @param {string} originalUrl - Original image URL
+ * @returns {string} Proxied URL
+ */
+export function getProxiedUrl(baseUrl, originalUrl) {
+  if (!originalUrl || !baseUrl) {
+    return originalUrl;
+  }
+
+  // Skip data URLs
+  if (originalUrl.startsWith("data:")) {
+    return originalUrl;
+  }
+
+  // Skip already-proxied URLs
+  if (originalUrl.includes("/microsub/media/")) {
+    return originalUrl;
+  }
+
+  const hash = hashUrl(originalUrl);
+  return `${baseUrl}/microsub/media/${hash}?url=${encodeURIComponent(originalUrl)}`;
+}
+
+/**
+ * Rewrite image URLs in an item to use the proxy
+ * @param {object} item - JF2 item
+ * @param {string} baseUrl - Base URL for proxy
+ * @returns {object} Item with proxied URLs
+ */
+export function proxyItemImages(item, baseUrl) {
+  if (!baseUrl || !item) {
+    return item;
+  }
+
+  const proxied = { ...item };
+
+  // Proxy photo URLs
+  if (proxied.photo) {
+    if (Array.isArray(proxied.photo)) {
+      proxied.photo = proxied.photo.map((p) => {
+        if (typeof p === "string") {
+          return getProxiedUrl(baseUrl, p);
+        }
+        if (p?.value) {
+          return { ...p, value: getProxiedUrl(baseUrl, p.value) };
+        }
+        return p;
+      });
+    } else if (typeof proxied.photo === "string") {
+      proxied.photo = getProxiedUrl(baseUrl, proxied.photo);
+    }
+  }
+
+  // Proxy author photo
+  if (proxied.author?.photo) {
+    proxied.author = {
+      ...proxied.author,
+      photo: getProxiedUrl(baseUrl, proxied.author.photo),
+    };
+  }
+
+  return proxied;
+}
+
+/**
+ * Fetch and cache an image
+ * @param {object} redis - Redis client
+ * @param {string} url - Image URL to fetch
+ * @returns {Promise<object|null>} Cached image data or null
+ */
+export async function fetchImage(redis, url) {
+  const cacheKey = `media:${hashUrl(url)}`;
+
+  // Try cache first
+  if (redis) {
+    const cached = await getCache(redis, cacheKey);
+    if (cached) {
+      return cached;
+    }
+  }
+
+  try {
+    // Fetch the image
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)",
+        Accept: "image/*",
+      },
+      signal: AbortSignal.timeout(10_000), // 10 second timeout
+    });
+
+    if (!response.ok) {
+      console.error(`[Microsub] Media proxy fetch failed: ${response.status} for ${url}`);
+      return null;
+    }
+
+    // Check content type
+    const contentType = response.headers.get("content-type")?.split(";")[0];
+    if (!ALLOWED_TYPES.includes(contentType)) {
+      console.error(`[Microsub] Media proxy rejected type: ${contentType} for ${url}`);
+      return null;
+    }
+
+    // Check content length
+    const contentLength = Number.parseInt(
+      response.headers.get("content-length") || "0",
+      10,
+    );
+    if (contentLength > MAX_SIZE) {
+      console.error(`[Microsub] Media proxy rejected size: ${contentLength} for ${url}`);
+      return null;
+    }
+
+    // Read the body
+    const buffer = await response.arrayBuffer();
+    if (buffer.byteLength > MAX_SIZE) {
+      return null;
+    }
+
+    const imageData = {
+      contentType,
+      data: Buffer.from(buffer).toString("base64"),
+      size: buffer.byteLength,
+    };
+
+    // Cache in Redis
+    if (redis) {
+      await setCache(redis, cacheKey, imageData, CACHE_TTL);
+    }
+
+    return imageData;
+  } catch (error) {
+    console.error(`[Microsub] Media proxy error: ${error.message} for ${url}`);
+    return null;
+  }
+}
+
+/**
+ * Express route handler for media proxy
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ * @returns {Promise<void>}
+ */
+export async function handleMediaProxy(request, response) {
+  const { url } = request.query;
+
+  if (!url) {
+    return response.status(400).send("Missing url parameter");
+  }
+
+  // Validate URL
+  try {
+    const parsed = new URL(url);
+    if (!["http:", "https:"].includes(parsed.protocol)) {
+      return response.status(400).send("Invalid URL protocol");
+    }
+  } catch {
+    return response.status(400).send("Invalid URL");
+  }
+
+  // Get Redis client from application
+  const { application } = request.app.locals;
+  const redis = application.redis;
+
+  // Fetch or get from cache
+  const imageData = await fetchImage(redis, url);
+
+  if (!imageData) {
+    // Redirect to original URL as fallback
+    return response.redirect(url);
+  }
+
+  // Set cache headers
+  response.set({
+    "Content-Type": imageData.contentType,
+    "Content-Length": imageData.size,
+    "Cache-Control": "public, max-age=14400", // 4 hours
+    "X-Proxied-From": url,
+  });
+
+  // Send the image
+  response.send(Buffer.from(imageData.data, "base64"));
+}

package/lib/polling/processor.js

@@ -9,6 +9,10 @@ import { getChannel } from "../storage/channels.js";
 import { updateFeedAfterFetch, updateFeedWebsub } from "../storage/feeds.js";
 import { passesRegexFilter, passesTypeFilter } from "../storage/filters.js";
 import { addItem } from "../storage/items.js";
+import {
+  subscribe as websubSubscribe,
+  getCallbackUrl,
+} from "../websub/subscriber.js";
 
 import { calculateNewTier } from "./tier.js";
 
@@ -123,13 +127,37 @@ export async function processFeed(application, feed) {
       updateData,
     );
 
-    // Handle WebSub hub discovery
+    // Handle WebSub hub discovery and auto-subscription
     if (parsed.hub && (!feed.websub || feed.websub.hub !== parsed.hub)) {
       await updateFeedWebsub(application, feed._id, {
         hub: parsed.hub,
         topic: parsed.self || feed.url,
       });
-      // TODO: Subscribe to hub
+
+      // Auto-subscribe to WebSub hub if we have a callback URL
+      const baseUrl = application.url;
+      if (baseUrl) {
+        const callbackUrl = getCallbackUrl(baseUrl, feed._id.toString());
+        const updatedFeed = {
+          ...feed,
+          websub: { hub: parsed.hub, topic: parsed.self || feed.url },
+        };
+
+        websubSubscribe(application, updatedFeed, callbackUrl)
+          .then((subscribed) => {
+            if (subscribed) {
+              console.info(
+                `[Microsub] WebSub subscription initiated for ${feed.url}`,
+              );
+            }
+          })
+          .catch((error) => {
+            console.error(
+              `[Microsub] WebSub subscription error for ${feed.url}:`,
+              error.message,
+            );
+          });
+      }
     }
 
     result.success = true;

package/lib/storage/channels.js

@@ -5,6 +5,8 @@
 
 import { ObjectId } from "mongodb";
 
+import { deleteFeedsForChannel } from "./feeds.js";
+import { deleteItemsForChannel } from "./items.js";
 import { generateChannelUid } from "../utils/jf2.js";
 
 /**
@@ -184,7 +186,7 @@ export async function updateChannel(application, uid, updates, userId) {
 }
 
 /**
- * Delete a channel
+ * Delete a channel and all its feeds and items
  * @param {object} application - Indiekit application
  * @param {string} uid - Channel UID
  * @param {string} [userId] - User ID
@@ -200,7 +202,20 @@ export async function deleteChannel(application, uid, userId) {
     return false;
   }
 
-  const result = await collection.deleteOne(query);
+  // Find the channel first to get its ObjectId
+  const channel = await collection.findOne(query);
+  if (!channel) {
+    return false;
+  }
+
+  // Cascade delete: items first, then feeds, then channel
+  const itemsDeleted = await deleteItemsForChannel(application, channel._id);
+  const feedsDeleted = await deleteFeedsForChannel(application, channel._id);
+  console.info(
+    `[Microsub] Deleted channel ${uid}: ${feedsDeleted} feeds, ${itemsDeleted} items`,
+  );
+
+  const result = await collection.deleteOne({ _id: channel._id });
   return result.deletedCount > 0;
 }
 

package/lib/storage/feeds.js

@@ -5,6 +5,8 @@
 
 import { ObjectId } from "mongodb";
 
+import { deleteItemsForFeed } from "./items.js";
+
 /**
  * Get feeds collection from application
  * @param {object} application - Indiekit application
@@ -122,7 +124,7 @@ export async function updateFeed(application, id, updates) {
 }
 
 /**
- * Delete a feed subscription
+ * Delete a feed subscription and all its items
  * @param {object} application - Indiekit application
  * @param {ObjectId|string} channelId - Channel ObjectId
  * @param {string} url - Feed URL
@@ -133,7 +135,18 @@ export async function deleteFeed(application, channelId, url) {
   const objectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
-  const result = await collection.deleteOne({ channelId: objectId, url });
+  // Find the feed first to get its ID for cascade delete
+  const feed = await collection.findOne({ channelId: objectId, url });
+  if (!feed) {
+    return false;
+  }
+
+  // Delete all items from this feed
+  const itemsDeleted = await deleteItemsForFeed(application, feed._id);
+  console.info(`[Microsub] Deleted ${itemsDeleted} items from feed ${url}`);
+
+  // Delete the feed itself
+  const result = await collection.deleteOne({ _id: feed._id });
   return result.deletedCount > 0;
 }
 

package/lib/storage/items.js

@@ -195,7 +195,7 @@ export async function getItemsByUids(application, uids, userId) {
  * Mark items as read
  * @param {object} application - Indiekit application
  * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {Array} entryIds - Array of entry IDs to mark as read
+ * @param {Array} entryIds - Array of entry IDs to mark as read (can be ObjectId, uid, or URL)
  * @param {string} userId - User ID
  * @returns {Promise<number>} Number of items updated
  */
@@ -204,6 +204,12 @@ export async function markItemsRead(application, channelId, entryIds, userId) {
   const channelObjectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
+  console.info(
+    `[Microsub] markItemsRead called for channel ${channelId}, entries:`,
+    entryIds,
+    `userId: ${userId}`,
+  );
+
   // Handle "last-read-entry" special value
   if (entryIds.includes("last-read-entry")) {
     // Mark all items in channel as read
@@ -211,26 +217,39 @@ export async function markItemsRead(application, channelId, entryIds, userId) {
       { channelId: channelObjectId },
       { $addToSet: { readBy: userId } },
     );
+    console.info(
+      `[Microsub] Marked all items as read: ${result.modifiedCount} updated`,
+    );
     return result.modifiedCount;
   }
 
   // Convert string IDs to ObjectIds where possible
-  const objectIds = entryIds.map((id) => {
-    try {
-      return new ObjectId(id);
-    } catch {
-      return id;
-    }
-  });
+  const objectIds = entryIds
+    .map((id) => {
+      try {
+        return new ObjectId(id);
+      } catch {
+        return undefined;
+      }
+    })
+    .filter(Boolean);
 
+  // Build query to match by _id, uid, or url (Microsub spec uses URLs as entry identifiers)
   const result = await collection.updateMany(
     {
       channelId: channelObjectId,
-      $or: [{ _id: { $in: objectIds } }, { uid: { $in: entryIds } }],
+      $or: [
+        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
+        { uid: { $in: entryIds } },
+        { url: { $in: entryIds } },
+      ],
     },
     { $addToSet: { readBy: userId } },
   );
 
+  console.info(
+    `[Microsub] markItemsRead result: ${result.modifiedCount} items updated`,
+  );
   return result.modifiedCount;
 }
 
@@ -238,7 +257,7 @@ export async function markItemsRead(application, channelId, entryIds, userId) {
  * Mark items as unread
  * @param {object} application - Indiekit application
  * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {Array} entryIds - Array of entry IDs to mark as unread
+ * @param {Array} entryIds - Array of entry IDs to mark as unread (can be ObjectId, uid, or URL)
  * @param {string} userId - User ID
  * @returns {Promise<number>} Number of items updated
  */
@@ -252,18 +271,26 @@ export async function markItemsUnread(
   const channelObjectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
-  const objectIds = entryIds.map((id) => {
-    try {
-      return new ObjectId(id);
-    } catch {
-      return id;
-    }
-  });
+  // Convert string IDs to ObjectIds where possible
+  const objectIds = entryIds
+    .map((id) => {
+      try {
+        return new ObjectId(id);
+      } catch {
+        return undefined;
+      }
+    })
+    .filter(Boolean);
 
+  // Match by _id, uid, or url
   const result = await collection.updateMany(
     {
       channelId: channelObjectId,
-      $or: [{ _id: { $in: objectIds } }, { uid: { $in: entryIds } }],
+      $or: [
+        ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
+        { uid: { $in: entryIds } },
+        { url: { $in: entryIds } },
+      ],
     },
     { $pull: { readBy: userId } },
   );
@@ -275,7 +302,7 @@ export async function markItemsUnread(
  * Remove items from channel
  * @param {object} application - Indiekit application
  * @param {ObjectId|string} channelId - Channel ObjectId
- * @param {Array} entryIds - Array of entry IDs to remove
+ * @param {Array} entryIds - Array of entry IDs to remove (can be ObjectId, uid, or URL)
  * @returns {Promise<number>} Number of items removed
  */
 export async function removeItems(application, channelId, entryIds) {
@@ -283,17 +310,25 @@ export async function removeItems(application, channelId, entryIds) {
   const channelObjectId =
     typeof channelId === "string" ? new ObjectId(channelId) : channelId;
 
-  const objectIds = entryIds.map((id) => {
-    try {
-      return new ObjectId(id);
-    } catch {
-      return id;
-    }
-  });
+  // Convert string IDs to ObjectIds where possible
+  const objectIds = entryIds
+    .map((id) => {
+      try {
+        return new ObjectId(id);
+      } catch {
+        return undefined;
+      }
+    })
+    .filter(Boolean);
 
+  // Match by _id, uid, or url
   const result = await collection.deleteMany({
     channelId: channelObjectId,
-    $or: [{ _id: { $in: objectIds } }, { uid: { $in: entryIds } }],
+    $or: [
+      ...(objectIds.length > 0 ? [{ _id: { $in: objectIds } }] : []),
+      { uid: { $in: entryIds } },
+      { url: { $in: entryIds } },
+    ],
   });
 
   return result.deletedCount;
@@ -411,11 +446,34 @@ export async function deleteItemsByAuthorUrl(application, userId, authorUrl) {
 export async function createIndexes(application) {
   const collection = getCollection(application);
 
+  // Primary query indexes
   await collection.createIndex({ channelId: 1, published: -1 });
   await collection.createIndex({ channelId: 1, uid: 1 }, { unique: true });
   await collection.createIndex({ feedId: 1 });
+
+  // URL matching index for mark_read operations
+  await collection.createIndex({ channelId: 1, url: 1 });
+
+  // Full-text search index with weights
+  // Higher weight = more importance in relevance scoring
   await collection.createIndex(
-    { name: "text", "content.text": "text", summary: "text" },
-    { name: "text_search" },
+    {
+      name: "text",
+      "content.text": "text",
+      "content.html": "text",
+      summary: "text",
+      "author.name": "text",
+    },
+    {
+      name: "text_search",
+      weights: {
+        name: 10, // Titles most important
+        summary: 5, // Summaries second
+        "content.text": 3, // Content third
+        "content.html": 2, // HTML content lower
+        "author.name": 1, // Author names lowest
+      },
+      default_language: "english",
+    },
   );
 }

package/lib/utils/auth.js

@@ -0,0 +1,37 @@
+/**
+ * Authentication utilities for Microsub
+ * @module utils/auth
+ */
+
+/**
+ * Get the user ID from request context
+ *
+ * In Indiekit, the userId can come from:
+ * 1. request.session.userId (if explicitly set)
+ * 2. request.session.me (from token introspection)
+ * 3. application.publication.me (single-user fallback)
+ *
+ * @param {object} request - Express request
+ * @returns {string|undefined} User ID
+ */
+export function getUserId(request) {
+  // Check session for explicit userId
+  if (request.session?.userId) {
+    return request.session.userId;
+  }
+
+  // Check session for me URL from token introspection
+  if (request.session?.me) {
+    return request.session.me;
+  }
+
+  // Fall back to publication me URL (single-user mode)
+  const { application } = request.app.locals;
+  if (application?.publication?.me) {
+    return application.publication.me;
+  }
+
+  // Final fallback: use "default" as user ID for single-user instances
+  // This ensures read state is tracked even without explicit user identity
+  return "default";
+}

package/lib/webmention/receiver.js

@@ -3,6 +3,7 @@
  * @module webmention/receiver
  */
 
+import { getUserId } from "../utils/auth.js";
 import { processWebmention } from "./processor.js";
 
 /**
@@ -33,7 +34,7 @@ export async function receive(request, response) {
   }
 
   const { application } = request.app.locals;
-  const userId = request.session?.userId;
+  const userId = getUserId(request);
 
   // Return 202 Accepted immediately (processing asynchronously)
   response.status(202).json({

package/locales/en.json

@@ -46,7 +46,8 @@
       "cancel": "Cancel",
       "replyTo": "Replying to",
       "likeOf": "Liking",
-      "repostOf": "Reposting"
+      "repostOf": "Reposting",
+      "bookmarkOf": "Bookmarking"
     },
     "settings": {
       "title": "{{channel}} settings",
@@ -55,6 +56,10 @@
       "excludeRegex": "Exclude pattern",
       "excludeRegexHelp": "Regular expression to filter out matching content",
       "save": "Save settings",
+      "dangerZone": "Danger zone",
+      "deleteWarning": "Deleting this channel will permanently remove all feeds and items. This action cannot be undone.",
+      "deleteConfirm": "Are you sure you want to delete this channel and all its content?",
+      "delete": "Delete channel",
       "types": {
         "like": "Likes",
         "repost": "Reposts",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-microsub",
-  "version": "1.0.0-beta.8",
+  "version": "1.0.0",
   "description": "Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the Microsub protocol.",
   "keywords": [
     "indiekit",
@@ -21,6 +21,9 @@
   },
   "type": "module",
   "main": "index.js",
+  "scripts": {
+    "test": "node --test test/unit/*.js"
+  },
   "files": [
     "lib",
     "locales",