npm - @rmdes/indiekit-endpoint-microsub - Versions diffs - 1.0.0-beta.1 - Mend

@rmdes/indiekit-endpoint-microsub 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (52) hide show

package/README.md +111 -0
package/index.js +140 -0
package/lib/cache/redis.js +133 -0
package/lib/controllers/block.js +85 -0
package/lib/controllers/channels.js +135 -0
package/lib/controllers/events.js +56 -0
package/lib/controllers/follow.js +108 -0
package/lib/controllers/microsub.js +138 -0
package/lib/controllers/mute.js +124 -0
package/lib/controllers/preview.js +67 -0
package/lib/controllers/reader.js +218 -0
package/lib/controllers/search.js +142 -0
package/lib/controllers/timeline.js +117 -0
package/lib/feeds/atom.js +61 -0
package/lib/feeds/fetcher.js +205 -0
package/lib/feeds/hfeed.js +177 -0
package/lib/feeds/jsonfeed.js +43 -0
package/lib/feeds/normalizer.js +586 -0
package/lib/feeds/parser.js +124 -0
package/lib/feeds/rss.js +61 -0
package/lib/polling/processor.js +201 -0
package/lib/polling/scheduler.js +128 -0
package/lib/polling/tier.js +139 -0
package/lib/realtime/broker.js +241 -0
package/lib/search/indexer.js +90 -0
package/lib/search/query.js +197 -0
package/lib/storage/channels.js +281 -0
package/lib/storage/feeds.js +286 -0
package/lib/storage/filters.js +265 -0
package/lib/storage/items.js +419 -0
package/lib/storage/read-state.js +109 -0
package/lib/utils/jf2.js +170 -0
package/lib/utils/pagination.js +157 -0
package/lib/utils/validation.js +217 -0
package/lib/webmention/processor.js +214 -0
package/lib/webmention/receiver.js +54 -0
package/lib/webmention/verifier.js +308 -0
package/lib/websub/discovery.js +129 -0
package/lib/websub/handler.js +163 -0
package/lib/websub/subscriber.js +181 -0
package/locales/en.json +80 -0
package/package.json +54 -0
package/views/channel-new.njk +33 -0
package/views/channel.njk +41 -0
package/views/compose.njk +61 -0
package/views/item.njk +85 -0
package/views/partials/actions.njk +15 -0
package/views/partials/author.njk +17 -0
package/views/partials/item-card.njk +65 -0
package/views/partials/timeline.njk +10 -0
package/views/reader.njk +37 -0
package/views/settings.njk +81 -0

package/lib/feeds/parser.js ADDED Viewed

@@ -0,0 +1,124 @@
+/**
+ * Feed parser dispatcher
+ * @module feeds/parser
+ */
+import { parseAtom } from "./atom.js";
+import { parseHfeed } from "./hfeed.js";
+import { parseJsonFeed } from "./jsonfeed.js";
+import { parseRss } from "./rss.js";
+/**
+ * Detect feed type from content
+ * @param {string} content - Feed content
+ * @param {string} contentType - HTTP Content-Type header
+ * @returns {string} Feed type: 'rss' | 'atom' | 'jsonfeed' | 'hfeed' | 'unknown'
+ */
+export function detectFeedType(content, contentType = "") {
+  const ct = contentType.toLowerCase();
+  // Check Content-Type header first
+  if (ct.includes("application/json") || ct.includes("application/feed+json")) {
+    return "jsonfeed";
+  }
+  if (ct.includes("application/atom+xml")) {
+    return "atom";
+  }
+  if (
+    ct.includes("application/rss+xml") ||
+    ct.includes("application/xml") ||
+    ct.includes("text/xml")
+  ) {
+    // Need to check content to distinguish RSS from Atom
+    const trimmed = content.trim();
+    if (
+      trimmed.includes("<feed") &&
+      trimmed.includes('xmlns="http://www.w3.org/2005/Atom"')
+    ) {
+      return "atom";
+    }
+    if (trimmed.includes("<rss") || trimmed.includes("<rdf:RDF")) {
+      return "rss";
+    }
+  }
+  if (ct.includes("text/html")) {
+    return "hfeed";
+  }
+  // Fall back to content inspection
+  const trimmed = content.trim();
+  // JSON Feed
+  if (trimmed.startsWith("{")) {
+    try {
+      const json = JSON.parse(trimmed);
+      if (json.version && json.version.includes("jsonfeed.org")) {
+        return "jsonfeed";
+      }
+    } catch {
+      // Not JSON
+    }
+  }
+  // XML feeds
+  if (trimmed.startsWith("<?xml") || trimmed.startsWith("<")) {
+    if (
+      trimmed.includes("<feed") &&
+      trimmed.includes('xmlns="http://www.w3.org/2005/Atom"')
+    ) {
+      return "atom";
+    }
+    if (trimmed.includes("<rss") || trimmed.includes("<rdf:RDF")) {
+      return "rss";
+    }
+  }
+  // HTML with potential h-feed
+  if (trimmed.includes("<!DOCTYPE html") || trimmed.includes("<html")) {
+    return "hfeed";
+  }
+  return "unknown";
+}
+/**
+ * Parse feed content into normalized items
+ * @param {string} content - Feed content
+ * @param {string} feedUrl - URL of the feed
+ * @param {object} options - Parse options
+ * @param {string} [options.contentType] - HTTP Content-Type header
+ * @returns {Promise<object>} Parsed feed with metadata and items
+ */
+export async function parseFeed(content, feedUrl, options = {}) {
+  const feedType = detectFeedType(content, options.contentType);
+  switch (feedType) {
+    case "rss": {
+      return parseRss(content, feedUrl);
+    }
+    case "atom": {
+      return parseAtom(content, feedUrl);
+    }
+    case "jsonfeed": {
+      return parseJsonFeed(content, feedUrl);
+    }
+    case "hfeed": {
+      return parseHfeed(content, feedUrl);
+    }
+    default: {
+      throw new Error(`Unable to detect feed type for ${feedUrl}`);
+    }
+  }
+}
+export { parseAtom } from "./atom.js";
+export { parseHfeed } from "./hfeed.js";
+export { parseJsonFeed } from "./jsonfeed.js";
+export { parseRss } from "./rss.js";

package/lib/feeds/rss.js ADDED Viewed

@@ -0,0 +1,61 @@
+/**
+ * RSS 1.0/2.0 feed parser
+ * @module feeds/rss
+ */
+import { Readable } from "node:stream";
+import FeedParser from "feedparser";
+import { normalizeItem, normalizeFeedMeta } from "./normalizer.js";
+/**
+ * Parse RSS feed content
+ * @param {string} content - RSS XML content
+ * @param {string} feedUrl - URL of the feed
+ * @returns {Promise<object>} Parsed feed with metadata and items
+ */
+export async function parseRss(content, feedUrl) {
+  return new Promise((resolve, reject) => {
+    const feedparser = new FeedParser({ feedurl: feedUrl });
+    const items = [];
+    let meta;
+    feedparser.on("error", (error) => {
+      reject(new Error(`RSS parse error: ${error.message}`));
+    });
+    feedparser.on("meta", (feedMeta) => {
+      meta = feedMeta;
+    });
+    feedparser.on("readable", function () {
+      let item;
+      while ((item = this.read())) {
+        items.push(item);
+      }
+    });
+    feedparser.on("end", () => {
+      try {
+        const normalizedMeta = normalizeFeedMeta(meta, feedUrl);
+        const normalizedItems = items.map((item) =>
+          normalizeItem(item, feedUrl, "rss"),
+        );
+        resolve({
+          type: "feed",
+          url: feedUrl,
+          ...normalizedMeta,
+          items: normalizedItems,
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+    // Create readable stream from string and pipe to feedparser
+    const stream = Readable.from([content]);
+    stream.pipe(feedparser);
+  });
+}

package/lib/polling/processor.js ADDED Viewed

@@ -0,0 +1,201 @@
+/**
+ * Feed processing pipeline
+ * @module polling/processor
+ */
+import { getRedisClient, publishEvent } from "../cache/redis.js";
+import { fetchAndParseFeed } from "../feeds/fetcher.js";
+import { getChannel } from "../storage/channels.js";
+import { updateFeedAfterFetch, updateFeedWebsub } from "../storage/feeds.js";
+import { passesRegexFilter, passesTypeFilter } from "../storage/filters.js";
+import { addItem } from "../storage/items.js";
+import { calculateNewTier } from "./tier.js";
+/**
+ * Process a single feed
+ * @param {object} application - Indiekit application
+ * @param {object} feed - Feed document from database
+ * @returns {Promise<object>} Processing result
+ */
+export async function processFeed(application, feed) {
+  const startTime = Date.now();
+  const result = {
+    feedId: feed._id,
+    url: feed.url,
+    success: false,
+    itemsAdded: 0,
+    error: undefined,
+  };
+  try {
+    // Get Redis client for caching
+    const redis = getRedisClient(application);
+    // Fetch and parse the feed
+    const parsed = await fetchAndParseFeed(feed.url, {
+      etag: feed.etag,
+      lastModified: feed.lastModified,
+      redis,
+    });
+    // Handle 304 Not Modified
+    if (parsed.notModified) {
+      const tierResult = calculateNewTier({
+        currentTier: feed.tier,
+        hasNewItems: false,
+        consecutiveUnchanged: feed.unmodified || 0,
+      });
+      await updateFeedAfterFetch(application, feed._id, false, {
+        tier: tierResult.tier,
+        unmodified: tierResult.consecutiveUnchanged,
+        nextFetchAt: tierResult.nextFetchAt,
+      });
+      result.success = true;
+      result.notModified = true;
+      return result;
+    }
+    // Get channel for filtering
+    const channel = await getChannel(application, feed.channelId);
+    // Process items
+    let newItemCount = 0;
+    for (const item of parsed.items) {
+      // Apply channel filters
+      if (channel?.settings && !passesFilters(item, channel.settings)) {
+        continue;
+      }
+      // Store the item
+      const stored = await addItem(application, {
+        channelId: feed.channelId,
+        feedId: feed._id,
+        uid: item.uid,
+        item,
+      });
+      if (stored) {
+        newItemCount++;
+        // Publish real-time event
+        if (redis) {
+          await publishEvent(redis, `microsub:${feed.channelId}`, {
+            type: "new-item",
+            channelId: feed.channelId.toString(),
+            item: stored,
+          });
+        }
+      }
+    }
+    result.itemsAdded = newItemCount;
+    // Update tier based on whether we found new items
+    const tierResult = calculateNewTier({
+      currentTier: feed.tier,
+      hasNewItems: newItemCount > 0,
+      consecutiveUnchanged: newItemCount > 0 ? 0 : feed.unmodified || 0,
+    });
+    // Update feed metadata
+    const updateData = {
+      tier: tierResult.tier,
+      unmodified: tierResult.consecutiveUnchanged,
+      nextFetchAt: tierResult.nextFetchAt,
+      etag: parsed.etag,
+      lastModified: parsed.lastModified,
+    };
+    // Update feed title/photo if discovered
+    if (parsed.name && !feed.title) {
+      updateData.title = parsed.name;
+    }
+    if (parsed.photo && !feed.photo) {
+      updateData.photo = parsed.photo;
+    }
+    await updateFeedAfterFetch(
+      application,
+      feed._id,
+      newItemCount > 0,
+      updateData,
+    );
+    // Handle WebSub hub discovery
+    if (parsed.hub && (!feed.websub || feed.websub.hub !== parsed.hub)) {
+      await updateFeedWebsub(application, feed._id, {
+        hub: parsed.hub,
+        topic: parsed.self || feed.url,
+      });
+      // TODO: Subscribe to hub
+    }
+    result.success = true;
+    result.tier = tierResult.tier;
+  } catch (error) {
+    result.error = error.message;
+    // Still update the feed to prevent retry storms
+    try {
+      const tierResult = calculateNewTier({
+        currentTier: feed.tier,
+        hasNewItems: false,
+        consecutiveUnchanged: (feed.unmodified || 0) + 1,
+      });
+      await updateFeedAfterFetch(application, feed._id, false, {
+        tier: Math.min(tierResult.tier + 1, 10), // Increase tier on error
+        unmodified: tierResult.consecutiveUnchanged,
+        nextFetchAt: tierResult.nextFetchAt,
+        lastError: error.message,
+        lastErrorAt: new Date(),
+      });
+    } catch {
+      // Ignore update errors
+    }
+  }
+  result.duration = Date.now() - startTime;
+  return result;
+}
+/**
+ * Check if an item passes channel filters
+ * @param {object} item - Feed item
+ * @param {object} settings - Channel settings
+ * @returns {boolean} Whether the item passes filters
+ */
+function passesFilters(item, settings) {
+  return passesTypeFilter(item, settings) && passesRegexFilter(item, settings);
+}
+/**
+ * Process multiple feeds in batch
+ * @param {object} application - Indiekit application
+ * @param {Array} feeds - Array of feed documents
+ * @param {object} options - Processing options
+ * @returns {Promise<object>} Batch processing result
+ */
+export async function processFeedBatch(application, feeds, options = {}) {
+  const { concurrency = 5 } = options;
+  const results = [];
+  // Process in batches with limited concurrency
+  for (let index = 0; index < feeds.length; index += concurrency) {
+    const batch = feeds.slice(index, index + concurrency);
+    const batchResults = await Promise.all(
+      batch.map((feed) => processFeed(application, feed)),
+    );
+    results.push(...batchResults);
+  }
+  return {
+    total: feeds.length,
+    successful: results.filter((r) => r.success).length,
+    failed: results.filter((r) => !r.success).length,
+    itemsAdded: results.reduce((sum, r) => sum + r.itemsAdded, 0),
+    results,
+  };
+}

package/lib/polling/scheduler.js ADDED Viewed

@@ -0,0 +1,128 @@
+/**
+ * Feed polling scheduler
+ * @module polling/scheduler
+ */
+import { getFeedsToFetch } from "../storage/feeds.js";
+import { processFeedBatch } from "./processor.js";
+let schedulerInterval;
+let indiekitInstance;
+let isRunning = false;
+const POLL_INTERVAL = 60 * 1000; // Run scheduler every minute
+const BATCH_CONCURRENCY = 5; // Process 5 feeds at a time
+/**
+ * Start the feed polling scheduler
+ * @param {object} indiekit - Indiekit instance
+ */
+export function startScheduler(indiekit) {
+  if (schedulerInterval) {
+    return; // Already running
+  }
+  indiekitInstance = indiekit;
+  // Run every minute
+  schedulerInterval = setInterval(async () => {
+    await runSchedulerCycle();
+  }, POLL_INTERVAL);
+  // Run immediately on start
+  runSchedulerCycle();
+  console.log("[Microsub] Feed polling scheduler started");
+}
+/**
+ * Stop the feed polling scheduler
+ */
+export function stopScheduler() {
+  if (schedulerInterval) {
+    clearInterval(schedulerInterval);
+    schedulerInterval = undefined;
+  }
+  indiekitInstance = undefined;
+  console.log("[Microsub] Feed polling scheduler stopped");
+}
+/**
+ * Run a single scheduler cycle
+ */
+async function runSchedulerCycle() {
+  if (!indiekitInstance) {
+    return;
+  }
+  // Prevent overlapping runs
+  if (isRunning) {
+    return;
+  }
+  isRunning = true;
+  try {
+    const application = indiekitInstance;
+    const feeds = await getFeedsToFetch(application);
+    if (feeds.length === 0) {
+      isRunning = false;
+      return;
+    }
+    console.log(`[Microsub] Processing ${feeds.length} feeds due for refresh`);
+    const result = await processFeedBatch(application, feeds, {
+      concurrency: BATCH_CONCURRENCY,
+    });
+    console.log(
+      `[Microsub] Processed ${result.total} feeds: ${result.successful} successful, ` +
+        `${result.failed} failed, ${result.itemsAdded} new items`,
+    );
+    // Log any errors
+    for (const feedResult of result.results) {
+      if (feedResult.error) {
+        console.error(
+          `[Microsub] Error processing ${feedResult.url}: ${feedResult.error}`,
+        );
+      }
+    }
+  } catch (error) {
+    console.error("[Microsub] Error in scheduler cycle:", error.message);
+  } finally {
+    isRunning = false;
+  }
+}
+/**
+ * Manually trigger a feed refresh
+ * @param {object} application - Indiekit application
+ * @param {string} feedId - Feed ID to refresh
+ * @returns {Promise<object>} Processing result
+ */
+export async function refreshFeedNow(application, feedId) {
+  const { getFeedById } = await import("../storage/feeds.js");
+  const { processFeed } = await import("./processor.js");
+  const feed = await getFeedById(application, feedId);
+  if (!feed) {
+    throw new Error("Feed not found");
+  }
+  return processFeed(application, feed);
+}
+/**
+ * Get scheduler status
+ * @returns {object} Scheduler status
+ */
+export function getSchedulerStatus() {
+  return {
+    running: !!schedulerInterval,
+    processing: isRunning,
+  };
+}

package/lib/polling/tier.js ADDED Viewed

@@ -0,0 +1,139 @@
+/**
+ * Adaptive tier-based polling algorithm
+ * Based on Ekster's approach: https://github.com/pstuifzand/ekster
+ *
+ * Tier determines poll interval: interval = 2^tier minutes
+ * - Tier 0: Every minute (active/new feeds)
+ * - Tier 1: Every 2 minutes
+ * - Tier 2: Every 4 minutes
+ * - Tier 3: Every 8 minutes
+ * - Tier 4: Every 16 minutes
+ * - Tier 5: Every 32 minutes
+ * - Tier 6: Every 64 minutes (~1 hour)
+ * - Tier 7: Every 128 minutes (~2 hours)
+ * - Tier 8: Every 256 minutes (~4 hours)
+ * - Tier 9: Every 512 minutes (~8 hours)
+ * - Tier 10: Every 1024 minutes (~17 hours)
+ *
+ * @module polling/tier
+ */
+const MIN_TIER = 0;
+const MAX_TIER = 10;
+const DEFAULT_TIER = 1;
+/**
+ * Get polling interval for a tier in milliseconds
+ * @param {number} tier - Polling tier (0-10)
+ * @returns {number} Interval in milliseconds
+ */
+export function getIntervalForTier(tier) {
+  const clampedTier = Math.max(MIN_TIER, Math.min(MAX_TIER, tier));
+  const minutes = Math.pow(2, clampedTier);
+  return minutes * 60 * 1000;
+}
+/**
+ * Get next fetch time based on tier
+ * @param {number} tier - Polling tier
+ * @returns {Date} Next fetch time
+ */
+export function getNextFetchTime(tier) {
+  const interval = getIntervalForTier(tier);
+  return new Date(Date.now() + interval);
+}
+/**
+ * Calculate new tier after a fetch
+ * @param {object} options - Options
+ * @param {number} options.currentTier - Current tier
+ * @param {boolean} options.hasNewItems - Whether new items were found
+ * @param {number} options.consecutiveUnchanged - Consecutive fetches with no changes
+ * @returns {object} New tier and metadata
+ */
+export function calculateNewTier(options) {
+  const {
+    currentTier = DEFAULT_TIER,
+    hasNewItems,
+    consecutiveUnchanged = 0,
+  } = options;
+  let newTier = currentTier;
+  let newConsecutiveUnchanged = consecutiveUnchanged;
+  if (hasNewItems) {
+    // Reset unchanged counter
+    newConsecutiveUnchanged = 0;
+    // Decrease tier (more frequent) if we found new items
+    if (currentTier > MIN_TIER) {
+      newTier = currentTier - 1;
+    }
+  } else {
+    // Increment unchanged counter
+    newConsecutiveUnchanged = consecutiveUnchanged + 1;
+    // Increase tier (less frequent) after consecutive unchanged fetches
+    // The threshold increases with tier to prevent thrashing
+    const threshold = Math.max(2, currentTier);
+    if (newConsecutiveUnchanged >= threshold && currentTier < MAX_TIER) {
+      newTier = currentTier + 1;
+      // Reset counter after tier change
+      newConsecutiveUnchanged = 0;
+    }
+  }
+  return {
+    tier: newTier,
+    consecutiveUnchanged: newConsecutiveUnchanged,
+    nextFetchAt: getNextFetchTime(newTier),
+  };
+}
+/**
+ * Get initial tier for a new feed subscription
+ * @returns {object} Initial tier settings
+ */
+export function getInitialTier() {
+  return {
+    tier: MIN_TIER, // Start at tier 0 for immediate first fetch
+    consecutiveUnchanged: 0,
+    nextFetchAt: new Date(), // Fetch immediately
+  };
+}
+/**
+ * Determine if a feed is due for fetching
+ * @param {object} feed - Feed document
+ * @returns {boolean} Whether the feed should be fetched
+ */
+export function isDueForFetch(feed) {
+  if (!feed.nextFetchAt) {
+    return true;
+  }
+  return new Date(feed.nextFetchAt) <= new Date();
+}
+/**
+ * Get human-readable description of polling interval
+ * @param {number} tier - Polling tier
+ * @returns {string} Description
+ */
+export function getTierDescription(tier) {
+  const minutes = Math.pow(2, tier);
+  if (minutes < 60) {
+    return `every ${minutes} minute${minutes === 1 ? "" : "s"}`;
+  }
+  const hours = minutes / 60;
+  if (hours < 24) {
+    return `every ${hours.toFixed(1)} hour${hours === 1 ? "" : "s"}`;
+  }
+  const days = hours / 24;
+  return `every ${days.toFixed(1)} day${days === 1 ? "" : "s"}`;
+}
+export { MIN_TIER, MAX_TIER, DEFAULT_TIER };