npm - @rmdes/indiekit-endpoint-microsub - Versions diffs - 1.0.0-beta.1 - Mend

@rmdes/indiekit-endpoint-microsub 1.0.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (52) hide show

package/README.md +111 -0
package/index.js +140 -0
package/lib/cache/redis.js +133 -0
package/lib/controllers/block.js +85 -0
package/lib/controllers/channels.js +135 -0
package/lib/controllers/events.js +56 -0
package/lib/controllers/follow.js +108 -0
package/lib/controllers/microsub.js +138 -0
package/lib/controllers/mute.js +124 -0
package/lib/controllers/preview.js +67 -0
package/lib/controllers/reader.js +218 -0
package/lib/controllers/search.js +142 -0
package/lib/controllers/timeline.js +117 -0
package/lib/feeds/atom.js +61 -0
package/lib/feeds/fetcher.js +205 -0
package/lib/feeds/hfeed.js +177 -0
package/lib/feeds/jsonfeed.js +43 -0
package/lib/feeds/normalizer.js +586 -0
package/lib/feeds/parser.js +124 -0
package/lib/feeds/rss.js +61 -0
package/lib/polling/processor.js +201 -0
package/lib/polling/scheduler.js +128 -0
package/lib/polling/tier.js +139 -0
package/lib/realtime/broker.js +241 -0
package/lib/search/indexer.js +90 -0
package/lib/search/query.js +197 -0
package/lib/storage/channels.js +281 -0
package/lib/storage/feeds.js +286 -0
package/lib/storage/filters.js +265 -0
package/lib/storage/items.js +419 -0
package/lib/storage/read-state.js +109 -0
package/lib/utils/jf2.js +170 -0
package/lib/utils/pagination.js +157 -0
package/lib/utils/validation.js +217 -0
package/lib/webmention/processor.js +214 -0
package/lib/webmention/receiver.js +54 -0
package/lib/webmention/verifier.js +308 -0
package/lib/websub/discovery.js +129 -0
package/lib/websub/handler.js +163 -0
package/lib/websub/subscriber.js +181 -0
package/locales/en.json +80 -0
package/package.json +54 -0
package/views/channel-new.njk +33 -0
package/views/channel.njk +41 -0
package/views/compose.njk +61 -0
package/views/item.njk +85 -0
package/views/partials/actions.njk +15 -0
package/views/partials/author.njk +17 -0
package/views/partials/item-card.njk +65 -0
package/views/partials/timeline.njk +10 -0
package/views/reader.njk +37 -0
package/views/settings.njk +81 -0

package/README.md ADDED Viewed

@@ -0,0 +1,111 @@
+# @indiekit/endpoint-microsub
+Microsub endpoint for Indiekit. Enables subscribing to feeds and reading content using the [Microsub protocol](https://indieweb.org/Microsub).
+## Features
+- **Full Microsub API** - Channels, timeline, follow/unfollow, search, preview, mute, block
+- **Multiple feed formats** - RSS 1.0/2.0, Atom, JSON Feed, h-feed (Microformats)
+- **External client support** - Works with Monocle, Together, IndiePass
+- **Built-in reader UI** - Social reading experience in Indiekit admin
+- **Real-time updates** - Server-Sent Events (SSE) and WebSub integration
+- **Adaptive polling** - Tier-based feed fetching inspired by Ekster
+- **Direct webmention receiving** - Notifications channel for mentions
+## Installation
+`npm install @indiekit/endpoint-microsub`
+## Usage
+Add `@indiekit/endpoint-microsub` to the list of plugins in your configuration:
+```js
+export default {
+  plugins: ["@indiekit/endpoint-microsub"],
+};
+```
+## Options
+| Option       | Type     | Description                                                      |
+| :----------- | :------- | :--------------------------------------------------------------- |
+| `mountPath`  | `string` | Path to mount Microsub API. _Optional_, defaults to `/microsub`. |
+| `readerPath` | `string` | Path to mount reader UI. _Optional_, defaults to `/reader`.      |
+## Endpoints
+### Microsub API
+The main Microsub endpoint is mounted at `/microsub` (configurable).
+**Discovery**: Add this to your site's `<head>`:
+```html
+<link rel="microsub" href="https://yoursite.com/microsub" />
+```
+### Supported actions
+| Action     | GET | POST | Description                                    |
+| :--------- | :-- | :--- | :--------------------------------------------- |
+| `channels` | ✓   | ✓    | List, create, update, delete, reorder channels |
+| `timeline` | ✓   | ✓    | Get timeline, mark read/unread, remove items   |
+| `follow`   | ✓   | ✓    | List followed feeds, subscribe to new feeds    |
+| `unfollow` | -   | ✓    | Unsubscribe from feeds                         |
+| `search`   | ✓   | ✓    | Feed discovery and full-text search            |
+| `preview`  | ✓   | ✓    | Preview feed before subscribing                |
+| `mute`     | ✓   | ✓    | List muted URLs, mute/unmute                   |
+| `block`    | ✓   | ✓    | List blocked URLs, block/unblock               |
+| `events`   | ✓   | -    | Server-Sent Events stream                      |
+### Reader UI
+The built-in reader is mounted at `/reader` (configurable) and provides:
+- Channel list with unread counts
+- Timeline view with items
+- Mark as read on scroll/click
+- Like/reply/repost via Micropub
+- Channel settings (filters)
+- Compose modal
+### WebSub callbacks
+WebSub hub callbacks are handled at `/microsub/websub/:id`.
+### Webmention receiving
+Direct webmentions can be sent to `/microsub/webmention`.
+## MongoDB Collections
+This plugin creates the following collections:
+- `microsub_channels` - User's feed channels
+- `microsub_feeds` - Subscribed feeds
+- `microsub_items` - Timeline entries
+- `microsub_notifications` - Webmention notifications
+- `microsub_muted` - Muted URLs
+- `microsub_blocked` - Blocked URLs
+## Dependencies
+- **feedparser** - RSS/Atom parsing
+- **microformats-parser** - h-feed parsing
+- **ioredis** - Redis client (optional, for caching/pub-sub)
+- **sanitize-html** - XSS prevention
+## External Clients
+This endpoint is compatible with:
+- [Monocle](https://monocle.p3k.io/) - Web-based reader
+- [Together](https://together.tpxl.io/) - Web-based reader
+- [IndiePass](https://indiepass.app/) - Mobile/desktop app (archived)
+## References
+- [Microsub Specification](https://indieweb.org/Microsub-spec)
+- [Ekster](https://github.com/pstuifzand/ekster) - Reference implementation in Go
+- [Aperture](https://github.com/aaronpk/Aperture) - Popular Microsub server in PHP

package/index.js ADDED Viewed

@@ -0,0 +1,140 @@
+import path from "node:path";
+import express from "express";
+import { microsubController } from "./lib/controllers/microsub.js";
+import { readerController } from "./lib/controllers/reader.js";
+import { startScheduler, stopScheduler } from "./lib/polling/scheduler.js";
+import { webmentionReceiver } from "./lib/webmention/receiver.js";
+import { websubHandler } from "./lib/websub/handler.js";
+const defaults = {
+  mountPath: "/microsub",
+};
+const router = express.Router();
+const readerRouter = express.Router();
+export default class MicrosubEndpoint {
+  name = "Microsub endpoint";
+  /**
+   * @param {object} options - Plugin options
+   * @param {string} [options.mountPath] - Path to mount Microsub endpoint
+   */
+  constructor(options = {}) {
+    this.options = { ...defaults, ...options };
+    this.mountPath = this.options.mountPath;
+  }
+  /**
+   * Navigation items for Indiekit admin
+   * @returns {object} Navigation item configuration
+   */
+  get navigationItems() {
+    return {
+      href: path.join(this.options.mountPath, "reader"),
+      text: "microsub.reader.title",
+      requiresDatabase: true,
+    };
+  }
+  /**
+   * Shortcut items for quick actions
+   * @returns {object} Shortcut item configuration
+   */
+  get shortcutItems() {
+    return {
+      url: path.join(this.options.mountPath, "reader", "channels"),
+      name: "microsub.channels.title",
+      iconName: "feed",
+      requiresDatabase: true,
+    };
+  }
+  /**
+   * Microsub API and reader UI routes (authenticated)
+   * @returns {import("express").Router} Express router
+   */
+  get routes() {
+    // Main Microsub endpoint - dispatches based on action parameter
+    router.get("/", microsubController.get);
+    router.post("/", microsubController.post);
+    // WebSub callback endpoint
+    router.get("/websub/:id", websubHandler.verify);
+    router.post("/websub/:id", websubHandler.receive);
+    // Webmention receiving endpoint
+    router.post("/webmention", webmentionReceiver.receive);
+    // Reader UI routes (mounted as sub-router for correct baseUrl)
+    readerRouter.get("/", readerController.index);
+    readerRouter.get("/channels", readerController.channels);
+    readerRouter.get("/channels/new", readerController.newChannel);
+    readerRouter.post("/channels/new", readerController.createChannel);
+    readerRouter.get("/channels/:uid", readerController.channel);
+    readerRouter.get("/channels/:uid/settings", readerController.settings);
+    readerRouter.post(
+      "/channels/:uid/settings",
+      readerController.updateSettings,
+    );
+    readerRouter.get("/item/:id", readerController.item);
+    readerRouter.get("/compose", readerController.compose);
+    readerRouter.post("/compose", readerController.submitCompose);
+    router.use("/reader", readerRouter);
+    return router;
+  }
+  /**
+   * Public routes (no authentication required)
+   * @returns {import("express").Router} Express router
+   */
+  get routesPublic() {
+    const publicRouter = express.Router();
+    // WebSub verification must be public for hubs to verify
+    publicRouter.get("/websub/:id", websubHandler.verify);
+    publicRouter.post("/websub/:id", websubHandler.receive);
+    // Webmention endpoint must be public
+    publicRouter.post("/webmention", webmentionReceiver.receive);
+    return publicRouter;
+  }
+  /**
+   * Initialize plugin
+   * @param {object} indiekit - Indiekit instance
+   */
+  init(indiekit) {
+    // Register MongoDB collections
+    indiekit.addCollection("microsub_channels");
+    indiekit.addCollection("microsub_feeds");
+    indiekit.addCollection("microsub_items");
+    indiekit.addCollection("microsub_notifications");
+    indiekit.addCollection("microsub_muted");
+    indiekit.addCollection("microsub_blocked");
+    // Register endpoint
+    indiekit.addEndpoint(this);
+    // Set microsub endpoint URL in config
+    if (!indiekit.config.application.microsubEndpoint) {
+      indiekit.config.application.microsubEndpoint = this.mountPath;
+    }
+    // Start feed polling scheduler when server starts
+    // This will be called after the server is ready
+    if (indiekit.database) {
+      startScheduler(indiekit);
+    }
+  }
+  /**
+   * Cleanup on shutdown
+   */
+  destroy() {
+    stopScheduler();
+  }
+}

package/lib/cache/redis.js ADDED Viewed

@@ -0,0 +1,133 @@
+/**
+ * Redis caching utilities
+ * @module cache/redis
+ */
+/**
+ * Get Redis client from application
+ * @param {object} application - Indiekit application
+ * @returns {object|undefined} Redis client or undefined
+ */
+export function getRedisClient(application) {
+  // Check if Redis is configured
+  if (application.redis) {
+    return application.redis;
+  }
+  // Check for Redis URL in config
+  if (application.config?.application?.redisUrl) {
+    // Lazily create Redis connection
+    // This will be implemented when Redis support is added to Indiekit core
+    return;
+  }
+}
+/**
+ * Get value from cache
+ * @param {object} redis - Redis client
+ * @param {string} key - Cache key
+ * @returns {Promise<object|undefined>} Cached value or undefined
+ */
+export async function getCache(redis, key) {
+  if (!redis) {
+    return;
+  }
+  try {
+    const value = await redis.get(key);
+    if (value) {
+      return JSON.parse(value);
+    }
+  } catch {
+    // Ignore cache errors
+  }
+}
+/**
+ * Set value in cache
+ * @param {object} redis - Redis client
+ * @param {string} key - Cache key
+ * @param {object} value - Value to cache
+ * @param {number} [ttl] - Time to live in seconds
+ * @returns {Promise<void>}
+ */
+export async function setCache(redis, key, value, ttl = 300) {
+  if (!redis) {
+    return;
+  }
+  try {
+    const serialized = JSON.stringify(value);
+    await (ttl
+      ? redis.set(key, serialized, "EX", ttl)
+      : redis.set(key, serialized));
+  } catch {
+    // Ignore cache errors
+  }
+}
+/**
+ * Delete value from cache
+ * @param {object} redis - Redis client
+ * @param {string} key - Cache key
+ * @returns {Promise<void>}
+ */
+export async function deleteCache(redis, key) {
+  if (!redis) {
+    return;
+  }
+  try {
+    await redis.del(key);
+  } catch {
+    // Ignore cache errors
+  }
+}
+/**
+ * Publish event to channel
+ * @param {object} redis - Redis client
+ * @param {string} channel - Channel name
+ * @param {object} data - Event data
+ * @returns {Promise<void>}
+ */
+export async function publishEvent(redis, channel, data) {
+  if (!redis) {
+    return;
+  }
+  try {
+    await redis.publish(channel, JSON.stringify(data));
+  } catch {
+    // Ignore pub/sub errors
+  }
+}
+/**
+ * Subscribe to channel
+ * @param {object} redis - Redis client (must be separate connection for pub/sub)
+ * @param {string} channel - Channel name
+ * @param {(data: object) => void} callback - Callback function for messages
+ * @returns {Promise<void>}
+ */
+export async function subscribeToChannel(redis, channel, callback) {
+  if (!redis) {
+    return;
+  }
+  try {
+    await redis.subscribe(channel);
+    redis.on("message", (receivedChannel, message) => {
+      if (receivedChannel === channel) {
+        try {
+          const data = JSON.parse(message);
+          callback(data);
+        } catch {
+          callback(message);
+        }
+      }
+    });
+  } catch {
+    // Ignore subscription errors
+  }
+}

package/lib/controllers/block.js ADDED Viewed

@@ -0,0 +1,85 @@
+/**
+ * Block controller
+ * @module controllers/block
+ */
+import { deleteItemsByAuthorUrl } from "../storage/items.js";
+import { validateUrl } from "../utils/validation.js";
+/**
+ * Get blocked collection
+ * @param {object} application - Indiekit application
+ * @returns {object} MongoDB collection
+ */
+function getCollection(application) {
+  return application.collections.get("microsub_blocked");
+}
+/**
+ * List blocked URLs
+ * GET ?action=block
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function list(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const collection = getCollection(application);
+  const blocked = await collection.find({ userId }).toArray();
+  const items = blocked.map((b) => ({ url: b.url }));
+  response.json({ items });
+}
+/**
+ * Block a URL
+ * POST ?action=block
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function block(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { url } = request.body;
+  validateUrl(url);
+  const collection = getCollection(application);
+  // Check if already blocked
+  const existing = await collection.findOne({ userId, url });
+  if (!existing) {
+    await collection.insertOne({
+      userId,
+      url,
+      createdAt: new Date(),
+    });
+  }
+  // Remove past items from blocked URL
+  await deleteItemsByAuthorUrl(application, userId, url);
+  response.json({ result: "ok" });
+}
+/**
+ * Unblock a URL
+ * POST ?action=unblock
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function unblock(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { url } = request.body;
+  validateUrl(url);
+  const collection = getCollection(application);
+  await collection.deleteOne({ userId, url });
+  response.json({ result: "ok" });
+}
+export const blockController = { list, block, unblock };

package/lib/controllers/channels.js ADDED Viewed

@@ -0,0 +1,135 @@
+/**
+ * Channel management controller
+ * @module controllers/channels
+ */
+import { IndiekitError } from "@indiekit/error";
+import {
+  getChannels,
+  getChannel,
+  createChannel,
+  updateChannel,
+  deleteChannel,
+  reorderChannels,
+} from "../storage/channels.js";
+import {
+  validateChannel,
+  validateChannelName,
+  parseArrayParameter as parseArrayParametereter,
+} from "../utils/validation.js";
+/**
+ * List all channels
+ * GET ?action=channels
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function list(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const channels = await getChannels(application, userId);
+  response.json({ channels });
+}
+/**
+ * Handle channel actions (create, update, delete, order)
+ * POST ?action=channels
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function action(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { method, name, uid } = request.body;
+  // Delete channel
+  if (method === "delete") {
+    validateChannel(uid);
+    const deleted = await deleteChannel(application, uid, userId);
+    if (!deleted) {
+      throw new IndiekitError("Channel not found or cannot be deleted", {
+        status: 404,
+      });
+    }
+    return response.json({ deleted: uid });
+  }
+  // Reorder channels
+  if (method === "order") {
+    const channelUids = parseArrayParametereter(request.body, "channels");
+    if (channelUids.length === 0) {
+      throw new IndiekitError("Missing channels[] parameter", {
+        status: 400,
+      });
+    }
+    await reorderChannels(application, channelUids, userId);
+    const channels = await getChannels(application, userId);
+    return response.json({ channels });
+  }
+  // Update existing channel
+  if (uid) {
+    validateChannel(uid);
+    if (name) {
+      validateChannelName(name);
+    }
+    const channel = await updateChannel(application, uid, { name }, userId);
+    if (!channel) {
+      throw new IndiekitError("Channel not found", {
+        status: 404,
+      });
+    }
+    return response.json({
+      uid: channel.uid,
+      name: channel.name,
+    });
+  }
+  // Create new channel
+  validateChannelName(name);
+  const channel = await createChannel(application, { name, userId });
+  response.status(201).json({
+    uid: channel.uid,
+    name: channel.name,
+  });
+}
+/**
+ * Get a single channel by UID
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function get(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  const { uid } = request.params;
+  validateChannel(uid);
+  const channel = await getChannel(application, uid, userId);
+  if (!channel) {
+    throw new IndiekitError("Channel not found", {
+      status: 404,
+    });
+  }
+  response.json({
+    uid: channel.uid,
+    name: channel.name,
+    settings: channel.settings,
+  });
+}
+export const channelsController = { list, action, get };

package/lib/controllers/events.js ADDED Viewed

@@ -0,0 +1,56 @@
+/**
+ * Server-Sent Events controller
+ * @module controllers/events
+ */
+import {
+  addClient,
+  removeClient,
+  sendEvent,
+  subscribeClient,
+} from "../realtime/broker.js";
+/**
+ * SSE stream endpoint
+ * GET ?action=events
+ * @param {object} request - Express request
+ * @param {object} response - Express response
+ */
+export async function stream(request, response) {
+  const { application } = request.app.locals;
+  const userId = request.session?.userId;
+  // Set SSE headers
+  response.setHeader("Content-Type", "text/event-stream");
+  response.setHeader("Cache-Control", "no-cache");
+  response.setHeader("Connection", "keep-alive");
+  response.setHeader("X-Accel-Buffering", "no"); // Disable nginx buffering
+  // Flush headers immediately
+  response.flushHeaders();
+  // Add client to broker (handles ping internally)
+  const client = addClient(response, userId, application);
+  // Subscribe to channels from query parameter
+  const { channels } = request.query;
+  if (channels) {
+    const channelList = Array.isArray(channels) ? channels : [channels];
+    for (const channelId of channelList) {
+      subscribeClient(response, channelId);
+    }
+  }
+  // Send initial event
+  sendEvent(response, "started", {
+    version: "1.0.0",
+    channels: [...client.channels],
+  });
+  // Handle client disconnect
+  request.on("close", () => {
+    removeClient(response);
+  });
+}
+export const eventsController = { stream };