@rmdes/indiekit-endpoint-microsub 1.0.0-beta.1

package/lib/controllers/search.js

@@ -0,0 +1,142 @@
+/**
+ * Search controller
+ * @module controllers/search
+ */
+
+import { IndiekitError } from "@indiekit/error";
+
+import { discoverFeeds } from "../feeds/hfeed.js";
+import { searchWithFallback } from "../search/query.js";
+import { getChannel } from "../storage/channels.js";
+import { validateChannel, validateUrl } from "../utils/validation.js";
+
+/**
+ * Discover feeds from a URL
+ * GET ?action=search&query=<url>
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function discover(request, response) {
+  const { query } = request.query;
+
+  if (!query) {
+    throw new IndiekitError("Missing required parameter: query", {
+      status: 400,
+    });
+  }
+
+  // Check if query is a URL
+  let url;
+  try {
+    url = new URL(query);
+  } catch {
+    // Not a URL, return empty results
+    return response.json({ results: [] });
+  }
+
+  try {
+    // Fetch the URL content
+    const fetchResponse = await fetch(url.href, {
+      headers: {
+        Accept: "text/html, application/xhtml+xml, */*",
+        "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)",
+      },
+    });
+
+    if (!fetchResponse.ok) {
+      throw new IndiekitError(`Failed to fetch URL: ${fetchResponse.status}`, {
+        status: 502,
+      });
+    }
+
+    const content = await fetchResponse.text();
+    const feeds = await discoverFeeds(content, url.href);
+
+    // Transform to Microsub search result format
+    const results = feeds.map((feed) => ({
+      type: "feed",
+      url: feed.url,
+    }));
+
+    response.json({ results });
+  } catch (error) {
+    if (error instanceof IndiekitError) {
+      throw error;
+    }
+    throw new IndiekitError(`Feed discovery failed: ${error.message}`, {
+      status: 502,
+    });
+  }
+}
+
+/**
+ * Search feeds or items
+ * POST ?action=search
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function search(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { query, channel } = request.body;
+
+  if (!query) {
+    throw new IndiekitError("Missing required parameter: query", {
+      status: 400,
+    });
+  }
+
+  // If channel is provided, search within channel items
+  if (channel) {
+    validateChannel(channel);
+
+    const channelDocument = await getChannel(application, channel, userId);
+    if (!channelDocument) {
+      throw new IndiekitError("Channel not found", { status: 404 });
+    }
+
+    const items = await searchWithFallback(
+      application,
+      channelDocument._id,
+      query,
+    );
+    return response.json({ items });
+  }
+
+  // Check if query is a URL (feed discovery)
+  try {
+    validateUrl(query, "query");
+
+    // Use the discover function for URL queries
+    const fetchResponse = await fetch(query, {
+      headers: {
+        Accept: "text/html, application/xhtml+xml, */*",
+        "User-Agent": "Indiekit Microsub/1.0 (+https://getindiekit.com)",
+      },
+    });
+
+    if (!fetchResponse.ok) {
+      throw new IndiekitError(`Failed to fetch URL: ${fetchResponse.status}`, {
+        status: 502,
+      });
+    }
+
+    const content = await fetchResponse.text();
+    const feeds = await discoverFeeds(content, query);
+
+    const results = feeds.map((feed) => ({
+      type: "feed",
+      url: feed.url,
+    }));
+
+    return response.json({ results });
+  } catch (error) {
+    // Not a URL or fetch failed, return empty results
+    if (error instanceof IndiekitError) {
+      throw error;
+    }
+    return response.json({ results: [] });
+  }
+}
+
+export const searchController = { discover, search };

package/lib/controllers/timeline.js

@@ -0,0 +1,117 @@
+/**
+ * Timeline controller
+ * @module controllers/timeline
+ */
+
+import { IndiekitError } from "@indiekit/error";
+
+import { getChannel } from "../storage/channels.js";
+import {
+  getTimelineItems,
+  markItemsRead,
+  markItemsUnread,
+  removeItems,
+} from "../storage/items.js";
+import {
+  validateChannel,
+  validateEntries,
+  parseArrayParameter as parseArrayParametereter,
+} from "../utils/validation.js";
+
+/**
+ * Get timeline items for a channel
+ * GET ?action=timeline&channel=<uid>
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function get(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { channel, before, after, limit } = request.query;
+
+  validateChannel(channel);
+
+  // Verify channel exists
+  const channelDocument = await getChannel(application, channel, userId);
+  if (!channelDocument) {
+    throw new IndiekitError("Channel not found", {
+      status: 404,
+    });
+  }
+
+  const timeline = await getTimelineItems(application, channelDocument._id, {
+    before,
+    after,
+    limit,
+    userId,
+  });
+
+  response.json(timeline);
+}
+
+/**
+ * Handle timeline actions (mark_read, mark_unread, remove)
+ * POST ?action=timeline
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function action(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { method, channel } = request.body;
+
+  validateChannel(channel);
+
+  // Verify channel exists
+  const channelDocument = await getChannel(application, channel, userId);
+  if (!channelDocument) {
+    throw new IndiekitError("Channel not found", {
+      status: 404,
+    });
+  }
+
+  // Get entry IDs from request
+  const entries = parseArrayParametereter(request.body, "entry");
+
+  switch (method) {
+    case "mark_read": {
+      validateEntries(entries);
+      const count = await markItemsRead(
+        application,
+        channelDocument._id,
+        entries,
+        userId,
+      );
+      return response.json({ result: "ok", updated: count });
+    }
+
+    case "mark_unread": {
+      validateEntries(entries);
+      const count = await markItemsUnread(
+        application,
+        channelDocument._id,
+        entries,
+        userId,
+      );
+      return response.json({ result: "ok", updated: count });
+    }
+
+    case "remove": {
+      validateEntries(entries);
+      const count = await removeItems(
+        application,
+        channelDocument._id,
+        entries,
+      );
+      return response.json({ result: "ok", removed: count });
+    }
+
+    default: {
+      throw new IndiekitError(`Invalid timeline method: ${method}`, {
+        status: 400,
+      });
+    }
+  }
+}
+
+export const timelineController = { get, action };

package/lib/feeds/atom.js

@@ -0,0 +1,61 @@
+/**
+ * Atom feed parser
+ * @module feeds/atom
+ */
+
+import { Readable } from "node:stream";
+
+import FeedParser from "feedparser";
+
+import { normalizeItem, normalizeFeedMeta } from "./normalizer.js";
+
+/**
+ * Parse Atom feed content
+ * @param {string} content - Atom XML content
+ * @param {string} feedUrl - URL of the feed
+ * @returns {Promise<object>} Parsed feed with metadata and items
+ */
+export async function parseAtom(content, feedUrl) {
+  return new Promise((resolve, reject) => {
+    const feedparser = new FeedParser({ feedurl: feedUrl });
+    const items = [];
+    let meta;
+
+    feedparser.on("error", (error) => {
+      reject(new Error(`Atom parse error: ${error.message}`));
+    });
+
+    feedparser.on("meta", (feedMeta) => {
+      meta = feedMeta;
+    });
+
+    feedparser.on("readable", function () {
+      let item;
+      while ((item = this.read())) {
+        items.push(item);
+      }
+    });
+
+    feedparser.on("end", () => {
+      try {
+        const normalizedMeta = normalizeFeedMeta(meta, feedUrl);
+        const normalizedItems = items.map((item) =>
+          normalizeItem(item, feedUrl, "atom"),
+        );
+
+        resolve({
+          type: "feed",
+          url: feedUrl,
+          ...normalizedMeta,
+          items: normalizedItems,
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+
+    // Create readable stream from string and pipe to feedparser
+    const stream = Readable.from([content]);
+    stream.pipe(feedparser);
+  });
+}

package/lib/feeds/fetcher.js

@@ -0,0 +1,205 @@
+/**
+ * Feed fetcher with HTTP caching
+ * @module feeds/fetcher
+ */
+
+import { getCache, setCache } from "../cache/redis.js";
+
+const DEFAULT_TIMEOUT = 30_000; // 30 seconds
+const DEFAULT_USER_AGENT = "Indiekit Microsub/1.0 (+https://getindiekit.com)";
+
+/**
+ * Fetch feed content with caching
+ * @param {string} url - Feed URL
+ * @param {object} options - Fetch options
+ * @param {string} [options.etag] - Previous ETag for conditional request
+ * @param {string} [options.lastModified] - Previous Last-Modified for conditional request
+ * @param {number} [options.timeout] - Request timeout in ms
+ * @param {object} [options.redis] - Redis client for caching
+ * @returns {Promise<object>} Fetch result with content and headers
+ */
+export async function fetchFeed(url, options = {}) {
+  const { etag, lastModified, timeout = DEFAULT_TIMEOUT, redis } = options;
+
+  // Check cache first
+  if (redis) {
+    const cached = await getCache(redis, `feed:${url}`);
+    if (cached) {
+      return {
+        content: cached.content,
+        contentType: cached.contentType,
+        etag: cached.etag,
+        lastModified: cached.lastModified,
+        fromCache: true,
+        status: 200,
+      };
+    }
+  }
+
+  const headers = {
+    Accept:
+      "application/atom+xml, application/rss+xml, application/json, application/feed+json, text/xml, text/html;q=0.9, */*;q=0.8",
+    "User-Agent": DEFAULT_USER_AGENT,
+  };
+
+  // Add conditional request headers
+  if (etag) {
+    headers["If-None-Match"] = etag;
+  }
+  if (lastModified) {
+    headers["If-Modified-Since"] = lastModified;
+  }
+
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeout);
+
+  try {
+    const response = await fetch(url, {
+      headers,
+      signal: controller.signal,
+      redirect: "follow",
+    });
+
+    clearTimeout(timeoutId);
+
+    // Not modified - use cached version
+    if (response.status === 304) {
+      return {
+        content: undefined,
+        contentType: undefined,
+        etag,
+        lastModified,
+        notModified: true,
+        status: 304,
+      };
+    }
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const content = await response.text();
+    const responseEtag = response.headers.get("ETag");
+    const responseLastModified = response.headers.get("Last-Modified");
+    const contentType = response.headers.get("Content-Type") || "";
+
+    const result = {
+      content,
+      contentType,
+      etag: responseEtag,
+      lastModified: responseLastModified,
+      fromCache: false,
+      status: response.status,
+    };
+
+    // Extract hub URL from Link header for WebSub
+    const linkHeader = response.headers.get("Link");
+    if (linkHeader) {
+      result.hub = extractHubFromLinkHeader(linkHeader);
+      result.self = extractSelfFromLinkHeader(linkHeader);
+    }
+
+    // Cache the result
+    if (redis) {
+      const cacheData = {
+        content,
+        contentType,
+        etag: responseEtag,
+        lastModified: responseLastModified,
+      };
+      // Cache for 5 minutes by default
+      await setCache(redis, `feed:${url}`, cacheData, 300);
+    }
+
+    return result;
+  } catch (error) {
+    clearTimeout(timeoutId);
+
+    if (error.name === "AbortError") {
+      throw new Error(`Request timeout after ${timeout}ms`);
+    }
+
+    throw error;
+  }
+}
+
+/**
+ * Extract hub URL from Link header
+ * @param {string} linkHeader - Link header value
+ * @returns {string|undefined} Hub URL
+ */
+function extractHubFromLinkHeader(linkHeader) {
+  const hubMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?hub["']?/i);
+  return hubMatch ? hubMatch[1] : undefined;
+}
+
+/**
+ * Extract self URL from Link header
+ * @param {string} linkHeader - Link header value
+ * @returns {string|undefined} Self URL
+ */
+function extractSelfFromLinkHeader(linkHeader) {
+  const selfMatch = linkHeader.match(/<([^>]+)>;\s*rel=["']?self["']?/i);
+  return selfMatch ? selfMatch[1] : undefined;
+}
+
+/**
+ * Fetch feed and parse it
+ * @param {string} url - Feed URL
+ * @param {object} options - Options
+ * @returns {Promise<object>} Parsed feed
+ */
+export async function fetchAndParseFeed(url, options = {}) {
+  const { parseFeed } = await import("./parser.js");
+
+  const result = await fetchFeed(url, options);
+
+  if (result.notModified) {
+    return {
+      ...result,
+      items: [],
+    };
+  }
+
+  const parsed = await parseFeed(result.content, url, {
+    contentType: result.contentType,
+  });
+
+  return {
+    ...result,
+    ...parsed,
+    hub: result.hub || parsed._hub,
+  };
+}
+
+/**
+ * Discover feeds from a URL
+ * @param {string} url - Page URL
+ * @param {object} options - Options
+ * @returns {Promise<Array>} Array of discovered feeds
+ */
+export async function discoverFeedsFromUrl(url, options = {}) {
+  const result = await fetchFeed(url, options);
+  const { discoverFeeds } = await import("./hfeed.js");
+
+  // If it's already a feed, return it
+  const contentType = result.contentType?.toLowerCase() || "";
+  if (
+    contentType.includes("xml") ||
+    contentType.includes("rss") ||
+    contentType.includes("atom") ||
+    contentType.includes("json")
+  ) {
+    return [
+      {
+        url,
+        type: contentType.includes("json") ? "jsonfeed" : "xml",
+        rel: "self",
+      },
+    ];
+  }
+
+  // Otherwise, discover feeds from HTML
+  const feeds = await discoverFeeds(result.content, url);
+  return feeds;
+}

package/lib/feeds/hfeed.js

@@ -0,0 +1,177 @@
+/**
+ * h-feed (Microformats2) parser
+ * @module feeds/hfeed
+ */
+
+import { mf2 } from "microformats-parser";
+
+import { normalizeHfeedItem, normalizeHfeedMeta } from "./normalizer.js";
+
+/**
+ * Parse h-feed content from HTML
+ * @param {string} content - HTML content with h-feed
+ * @param {string} feedUrl - URL of the page
+ * @returns {Promise<object>} Parsed feed with metadata and items
+ */
+export async function parseHfeed(content, feedUrl) {
+  let parsed;
+
+  try {
+    parsed = mf2(content, { baseUrl: feedUrl });
+  } catch (error) {
+    throw new Error(`h-feed parse error: ${error.message}`);
+  }
+
+  // Find h-feed in the parsed microformats
+  const hfeed = findHfeed(parsed);
+
+  if (!hfeed) {
+    // If no h-feed, look for h-entry items at the root
+    const entries = parsed.items.filter(
+      (item) => item.type && item.type.includes("h-entry"),
+    );
+
+    if (entries.length === 0) {
+      throw new Error("No h-feed or h-entry found on page");
+    }
+
+    // Create synthetic feed from entries
+    return {
+      type: "feed",
+      url: feedUrl,
+      name: parsed.rels?.canonical?.[0] || feedUrl,
+      items: entries.map((entry) => normalizeHfeedItem(entry, feedUrl)),
+    };
+  }
+
+  const normalizedMeta = normalizeHfeedMeta(hfeed, feedUrl);
+
+  // Get children entries from h-feed
+  const entries = hfeed.children || [];
+  const normalizedItems = entries
+    .filter((child) => child.type && child.type.includes("h-entry"))
+    .map((entry) => normalizeHfeedItem(entry, feedUrl));
+
+  return {
+    type: "feed",
+    url: feedUrl,
+    ...normalizedMeta,
+    items: normalizedItems,
+  };
+}
+
+/**
+ * Find h-feed in parsed microformats
+ * @param {object} parsed - Parsed microformats object
+ * @returns {object|undefined} h-feed object or undefined
+ */
+function findHfeed(parsed) {
+  // Look for h-feed at top level
+  for (const item of parsed.items) {
+    if (item.type && item.type.includes("h-feed")) {
+      return item;
+    }
+
+    // Check nested children
+    if (item.children) {
+      for (const child of item.children) {
+        if (child.type && child.type.includes("h-feed")) {
+          return child;
+        }
+      }
+    }
+  }
+
+  return;
+}
+
+/**
+ * Discover feeds from HTML page
+ * @param {string} content - HTML content
+ * @param {string} pageUrl - URL of the page
+ * @returns {Promise<Array>} Array of discovered feed URLs with types
+ */
+export async function discoverFeeds(content, pageUrl) {
+  const feeds = [];
+  const parsed = mf2(content, { baseUrl: pageUrl });
+
+  // Check for rel="alternate" feed links
+  const alternates = parsed.rels?.alternate || [];
+  for (const url of alternates) {
+    // Try to determine feed type from URL
+    if (url.includes("feed") || url.endsWith(".xml") || url.endsWith(".json")) {
+      feeds.push({
+        url,
+        type: "unknown",
+        rel: "alternate",
+      });
+    }
+  }
+
+  // Check for rel="feed" links (Microsub discovery)
+  const feedLinks = parsed.rels?.feed || [];
+  for (const url of feedLinks) {
+    feeds.push({
+      url,
+      type: "hfeed",
+      rel: "feed",
+    });
+  }
+
+  // Check if page itself has h-feed
+  const hfeed = findHfeed(parsed);
+  if (hfeed) {
+    feeds.push({
+      url: pageUrl,
+      type: "hfeed",
+      rel: "self",
+    });
+  }
+
+  // Parse <link> elements for feed discovery
+  const linkFeeds = extractLinkFeeds(content, pageUrl);
+  feeds.push(...linkFeeds);
+
+  return feeds;
+}
+
+/**
+ * Extract feed links from HTML <link> elements
+ * @param {string} content - HTML content
+ * @param {string} baseUrl - Base URL for resolving relative URLs
+ * @returns {Array} Array of discovered feeds
+ */
+function extractLinkFeeds(content, baseUrl) {
+  const feeds = [];
+  const linkRegex = /<link[^>]+rel=["'](?:alternate|feed)["'][^>]*>/gi;
+  const matches = content.match(linkRegex) || [];
+
+  for (const link of matches) {
+    const hrefMatch = link.match(/href=["']([^"']+)["']/i);
+    const typeMatch = link.match(/type=["']([^"']+)["']/i);
+
+    if (hrefMatch) {
+      const href = hrefMatch[1];
+      const type = typeMatch ? typeMatch[1] : "unknown";
+      const url = new URL(href, baseUrl).href;
+
+      let feedType = "unknown";
+      if (type.includes("rss")) {
+        feedType = "rss";
+      } else if (type.includes("atom")) {
+        feedType = "atom";
+      } else if (type.includes("json")) {
+        feedType = "jsonfeed";
+      }
+
+      feeds.push({
+        url,
+        type: feedType,
+        contentType: type,
+        rel: "link",
+      });
+    }
+  }
+
+  return feeds;
+}