@rmdes/indiekit-endpoint-microsub 1.0.0-beta.1

package/lib/realtime/broker.js

@@ -0,0 +1,241 @@
+/**
+ * Server-Sent Events broker
+ * Manages SSE connections and event distribution
+ * @module realtime/broker
+ */
+
+import { subscribeToChannel } from "../cache/redis.js";
+
+/**
+ * SSE Client connection
+ * @typedef {object} SseClient
+ * @property {object} response - Express response object
+ * @property {string} userId - User ID
+ * @property {Set<string>} channels - Subscribed channel IDs
+ */
+
+/** @type {Map<object, SseClient>} */
+const clients = new Map();
+
+/** @type {Map<string, object>} Map of userId to Redis subscriber */
+const userSubscribers = new Map();
+
+const PING_INTERVAL = 10_000; // 10 seconds
+
+/**
+ * Add a client to the broker
+ * @param {object} response - Express response object
+ * @param {string} userId - User ID
+ * @param {object} application - Indiekit application
+ * @returns {object} Client object
+ */
+export function addClient(response, userId, application) {
+  const client = {
+    response,
+    userId,
+    channels: new Set(),
+    pingInterval: setInterval(() => {
+      sendEvent(response, "ping", { timestamp: new Date().toISOString() });
+    }, PING_INTERVAL),
+  };
+
+  clients.set(response, client);
+
+  // Set up Redis subscription for this user if not already done
+  setupUserSubscription(userId, application);
+
+  return client;
+}
+
+/**
+ * Remove a client from the broker
+ * @param {object} response - Express response object
+ */
+export function removeClient(response) {
+  const client = clients.get(response);
+  if (client) {
+    clearInterval(client.pingInterval);
+    clients.delete(response);
+
+    // Check if any other clients for this user
+    const hasOtherClients = [...clients.values()].some(
+      (c) => c.userId === client.userId,
+    );
+    if (!hasOtherClients) {
+      // Could clean up Redis subscription here if needed
+    }
+  }
+}
+
+/**
+ * Subscribe a client to a channel
+ * @param {object} response - Express response object
+ * @param {string} channelId - Channel ID
+ */
+export function subscribeClient(response, channelId) {
+  const client = clients.get(response);
+  if (client) {
+    client.channels.add(channelId);
+  }
+}
+
+/**
+ * Unsubscribe a client from a channel
+ * @param {object} response - Express response object
+ * @param {string} channelId - Channel ID
+ */
+export function unsubscribeClient(response, channelId) {
+  const client = clients.get(response);
+  if (client) {
+    client.channels.delete(channelId);
+  }
+}
+
+/**
+ * Send an event to a specific client
+ * @param {object} response - Express response object
+ * @param {string} event - Event name
+ * @param {object} data - Event data
+ */
+export function sendEvent(response, event, data) {
+  try {
+    response.write(`event: ${event}\n`);
+    response.write(`data: ${JSON.stringify(data)}\n\n`);
+  } catch {
+    // Client disconnected
+    removeClient(response);
+  }
+}
+
+/**
+ * Broadcast an event to all clients subscribed to a channel
+ * @param {string} channelId - Channel ID
+ * @param {string} event - Event name
+ * @param {object} data - Event data
+ */
+export function broadcastToChannel(channelId, event, data) {
+  for (const client of clients.values()) {
+    if (client.channels.has(channelId)) {
+      sendEvent(client.response, event, data);
+    }
+  }
+}
+
+/**
+ * Broadcast an event to all clients for a user
+ * @param {string} userId - User ID
+ * @param {string} event - Event name
+ * @param {object} data - Event data
+ */
+export function broadcastToUser(userId, event, data) {
+  for (const client of clients.values()) {
+    if (client.userId === userId) {
+      sendEvent(client.response, event, data);
+    }
+  }
+}
+
+/**
+ * Broadcast an event to all connected clients
+ * @param {string} event - Event name
+ * @param {object} data - Event data
+ */
+export function broadcastToAll(event, data) {
+  for (const client of clients.values()) {
+    sendEvent(client.response, event, data);
+  }
+}
+
+/**
+ * Set up Redis subscription for a user
+ * @param {string} userId - User ID
+ * @param {object} application - Indiekit application
+ */
+async function setupUserSubscription(userId, application) {
+  if (userSubscribers.has(userId)) {
+    return; // Already subscribed
+  }
+
+  const redis = application.redis;
+  if (!redis) {
+    return; // No Redis, skip real-time
+  }
+
+  // Create a duplicate connection for pub/sub
+  const subscriber = redis.duplicate();
+  userSubscribers.set(userId, subscriber);
+
+  try {
+    await subscribeToChannel(subscriber, `microsub:user:${userId}`, (data) => {
+      handleRedisEvent(userId, data);
+    });
+  } catch {
+    // Subscription failed, remove from map
+    userSubscribers.delete(userId);
+  }
+}
+
+/**
+ * Handle event received from Redis
+ * @param {string} userId - User ID
+ * @param {object} data - Event data
+ */
+function handleRedisEvent(userId, data) {
+  const { type, channelId, ...eventData } = data;
+
+  switch (type) {
+    case "new-item": {
+      broadcastToUser(userId, "new-item", { channelId, ...eventData });
+      break;
+    }
+    case "channel-update": {
+      broadcastToUser(userId, "channel-update", { channelId, ...eventData });
+      break;
+    }
+    case "unread-count": {
+      broadcastToUser(userId, "unread-count", { channelId, ...eventData });
+      break;
+    }
+    default: {
+      // Unknown event type, broadcast as generic event
+      broadcastToUser(userId, type, data);
+    }
+  }
+}
+
+/**
+ * Get broker statistics
+ * @returns {object} Statistics
+ */
+export function getStats() {
+  const userCounts = new Map();
+  for (const client of clients.values()) {
+    const count = userCounts.get(client.userId) || 0;
+    userCounts.set(client.userId, count + 1);
+  }
+
+  return {
+    totalClients: clients.size,
+    uniqueUsers: userCounts.size,
+    userSubscribers: userSubscribers.size,
+  };
+}
+
+/**
+ * Clean up all connections
+ */
+export function cleanup() {
+  for (const client of clients.values()) {
+    clearInterval(client.pingInterval);
+  }
+  clients.clear();
+
+  for (const subscriber of userSubscribers.values()) {
+    try {
+      subscriber.quit();
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+  userSubscribers.clear();
+}

package/lib/search/indexer.js

@@ -0,0 +1,90 @@
+/**
+ * Search indexer for MongoDB text search
+ * @module search/indexer
+ */
+
+/**
+ * Create text indexes for microsub items
+ * @param {object} application - Indiekit application
+ * @returns {Promise<void>}
+ */
+export async function createSearchIndexes(application) {
+  const itemsCollection = application.collections.get("microsub_items");
+
+  // Create compound text index for full-text search
+  await itemsCollection.createIndex(
+    {
+      name: "text",
+      "content.text": "text",
+      "content.html": "text",
+      summary: "text",
+      "author.name": "text",
+    },
+    {
+      name: "text_search",
+      weights: {
+        name: 10,
+        "content.text": 5,
+        summary: 3,
+        "author.name": 2,
+      },
+      default_language: "english",
+      background: true,
+    },
+  );
+
+  // Create index for channel + published for efficient timeline queries
+  await itemsCollection.createIndex(
+    { channelId: 1, published: -1 },
+    { name: "channel_timeline" },
+  );
+
+  // Create index for deduplication
+  await itemsCollection.createIndex(
+    { channelId: 1, uid: 1 },
+    { name: "channel_uid", unique: true },
+  );
+
+  // Create index for feed-based queries
+  await itemsCollection.createIndex({ feedId: 1 }, { name: "feed_items" });
+}
+
+/**
+ * Rebuild search indexes (drops and recreates)
+ * @param {object} application - Indiekit application
+ * @returns {Promise<void>}
+ */
+export async function rebuildSearchIndexes(application) {
+  const itemsCollection = application.collections.get("microsub_items");
+
+  // Drop existing text index
+  try {
+    await itemsCollection.dropIndex("text_search");
+  } catch {
+    // Index may not exist
+  }
+
+  // Recreate indexes
+  await createSearchIndexes(application);
+}
+
+/**
+ * Get search index stats
+ * @param {object} application - Indiekit application
+ * @returns {Promise<object>} Index statistics
+ */
+export async function getSearchIndexStats(application) {
+  const itemsCollection = application.collections.get("microsub_items");
+
+  const indexes = await itemsCollection.indexes();
+  const stats = await itemsCollection.stats();
+
+  return {
+    indexes: indexes.map((index) => ({
+      name: index.name,
+      key: index.key,
+    })),
+    totalDocuments: stats.count,
+    size: stats.size,
+  };
+}

package/lib/search/query.js

@@ -0,0 +1,197 @@
+/**
+ * Search query module for full-text search
+ * @module search/query
+ */
+
+import { ObjectId } from "mongodb";
+
+/**
+ * Search items using MongoDB text search
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} query - Search query string
+ * @param {object} options - Search options
+ * @param {number} [options.limit] - Max results (default 20)
+ * @param {number} [options.skip] - Skip results for pagination
+ * @param {boolean} [options.sortByScore] - Sort by relevance (default true)
+ * @returns {Promise<Array>} Array of matching items
+ */
+export async function searchItemsFullText(
+  application,
+  channelId,
+  query,
+  options = {},
+) {
+  const collection = application.collections.get("microsub_items");
+  const { limit = 20, skip = 0, sortByScore = true } = options;
+
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  // Build the search query
+  const searchQuery = {
+    channelId: channelObjectId,
+    $text: { $search: query },
+  };
+
+  // Build aggregation pipeline for scoring
+  const pipeline = [
+    { $match: searchQuery },
+    { $addFields: { score: { $meta: "textScore" } } },
+  ];
+
+  if (sortByScore) {
+    pipeline.push(
+      { $sort: { score: -1, published: -1 } },
+      { $skip: skip },
+      { $limit: limit },
+    );
+  } else {
+    pipeline.push(
+      { $sort: { published: -1 } },
+      { $skip: skip },
+      { $limit: limit },
+    );
+  }
+
+  const items = await collection.aggregate(pipeline).toArray();
+
+  return items.map((item) => transformToSearchResult(item));
+}
+
+/**
+ * Search items using regex fallback (for partial matching)
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} query - Search query string
+ * @param {object} options - Search options
+ * @returns {Promise<Array>} Array of matching items
+ */
+export async function searchItemsRegex(
+  application,
+  channelId,
+  query,
+  options = {},
+) {
+  const collection = application.collections.get("microsub_items");
+  const { limit = 20 } = options;
+
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  // Escape regex special characters
+  const escapedQuery = query.replaceAll(/[$()*+.?[\\\]^{|}]/g, String.raw`\$&`);
+  const regex = new RegExp(escapedQuery, "i");
+
+  const items = await collection
+    .find({
+      channelId: channelObjectId,
+      $or: [
+        { name: regex },
+        { "content.text": regex },
+        { "content.html": regex },
+        { summary: regex },
+        { "author.name": regex },
+      ],
+    })
+    .toSorted({ published: -1 })
+    .limit(limit)
+    .toArray();
+
+  return items.map((item) => transformToSearchResult(item));
+}
+
+/**
+ * Search with automatic fallback
+ * Uses full-text search first, falls back to regex if no results
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} query - Search query string
+ * @param {object} options - Search options
+ * @returns {Promise<Array>} Array of matching items
+ */
+export async function searchWithFallback(
+  application,
+  channelId,
+  query,
+  options = {},
+) {
+  // Try full-text search first
+  try {
+    const results = await searchItemsFullText(
+      application,
+      channelId,
+      query,
+      options,
+    );
+    if (results.length > 0) {
+      return results;
+    }
+  } catch {
+    // Text index might not exist, fall through to regex
+  }
+
+  // Fall back to regex search
+  return searchItemsRegex(application, channelId, query, options);
+}
+
+/**
+ * Transform database item to search result format
+ * @param {object} item - Database item
+ * @returns {object} Search result
+ */
+function transformToSearchResult(item) {
+  const result = {
+    type: item.type || "entry",
+    uid: item.uid,
+    url: item.url,
+    published: item.published?.toISOString(),
+    _id: item._id.toString(),
+  };
+
+  if (item.name) result.name = item.name;
+  if (item.content) result.content = item.content;
+  if (item.summary) result.summary = item.summary;
+  if (item.author) result.author = item.author;
+  if (item.photo?.length > 0) result.photo = item.photo;
+  if (item.score) result._score = item.score;
+
+  return result;
+}
+
+/**
+ * Get search suggestions (autocomplete)
+ * @param {object} application - Indiekit application
+ * @param {ObjectId|string} channelId - Channel ObjectId
+ * @param {string} prefix - Search prefix
+ * @param {number} limit - Max suggestions
+ * @returns {Promise<Array>} Array of suggestions
+ */
+export async function getSearchSuggestions(
+  application,
+  channelId,
+  prefix,
+  limit = 5,
+) {
+  const collection = application.collections.get("microsub_items");
+
+  const channelObjectId =
+    typeof channelId === "string" ? new ObjectId(channelId) : channelId;
+
+  const escapedPrefix = prefix.replaceAll(
+    /[$()*+.?[\\\]^{|}]/g,
+    String.raw`\$&`,
+  );
+  const regex = new RegExp(`^${escapedPrefix}`, "i");
+
+  // Get unique names/titles that match prefix
+  const results = await collection
+    .aggregate([
+      { $match: { channelId: channelObjectId, name: regex } },
+      { $group: { _id: "$name" } },
+      { $limit: limit },
+    ])
+    .toArray();
+
+  return results.map((r) => r._id).filter(Boolean);
+}