@rmdes/indiekit-endpoint-webmention-io 1.0.0

package/lib/storage/webmentions.js

@@ -0,0 +1,252 @@
+/**
+ * Webmentions MongoDB storage
+ */
+
+import { extractDomain, sanitiseHtml } from "../utils.js";
+
+/**
+ * Ensure indexes exist
+ * @param {object} collection - MongoDB collection
+ */
+export async function ensureIndexes(collection) {
+  await collection.createIndex({ wmId: 1 }, { unique: true });
+  await collection.createIndex({ wmTarget: 1, hidden: 1 });
+  await collection.createIndex({ sourceDomain: 1 });
+  await collection.createIndex({ wmReceived: -1 });
+}
+
+/**
+ * Transform a JF2 webmention entry into our storage format
+ * @param {object} item - JF2 entry from webmention.io
+ * @returns {object} Document for MongoDB
+ */
+export function jf2ToDocument(item) {
+  let contentHtml = null;
+  let contentText = null;
+
+  if (item.content?.html) {
+    contentHtml = sanitiseHtml(item.content.html);
+  } else if (item.content?.text) {
+    contentHtml = `<p>${item.content.text}</p>`;
+  }
+
+  if (item.content?.text) {
+    contentText = item.content.text;
+  }
+
+  return {
+    wmId: item["wm-id"],
+    wmReceived: item["wm-received"] ? new Date(item["wm-received"]) : new Date(),
+    wmProperty: item["wm-property"],
+    wmTarget: item["wm-target"],
+    authorName: item.author?.name || null,
+    authorUrl: item.author?.url || null,
+    authorPhoto: item.author?.photo || null,
+    sourceUrl: item.url || null,
+    sourceDomain: extractDomain(item.author?.url || item.url || ""),
+    published: item.published ? new Date(item.published) : null,
+    contentHtml,
+    contentText,
+    name: item.name || null,
+    hidden: false,
+    hiddenAt: null,
+    hiddenReason: null,
+    syncedAt: new Date(),
+    raw: item,
+  };
+}
+
+/**
+ * Convert a stored document back to JF2 format for the public API
+ * @param {object} doc - MongoDB document
+ * @returns {object} JF2 entry
+ */
+export function documentToJf2(doc) {
+  const jf2 = {
+    type: "entry",
+    "wm-id": doc.wmId,
+    "wm-received": doc.wmReceived?.toISOString?.() || doc.wmReceived,
+    "wm-property": doc.wmProperty,
+    "wm-target": doc.wmTarget,
+    author: {
+      type: "card",
+      name: doc.authorName || "",
+      url: doc.authorUrl || "",
+      photo: doc.authorPhoto || "",
+    },
+    url: doc.sourceUrl || "",
+    published: doc.published?.toISOString?.() || doc.published || doc.wmReceived,
+  };
+
+  if (doc.name) {
+    jf2.name = doc.name;
+  }
+
+  if (doc.contentHtml || doc.contentText) {
+    jf2.content = {};
+    if (doc.contentHtml) jf2.content.html = doc.contentHtml;
+    if (doc.contentText) jf2.content.text = doc.contentText;
+  }
+
+  return jf2;
+}
+
+/**
+ * Upsert a webmention into MongoDB
+ * @param {object} collection - MongoDB collection
+ * @param {object} item - JF2 entry
+ * @returns {Promise<boolean>} true if inserted (new), false if updated
+ */
+export async function upsertWebmention(collection, item) {
+  const doc = jf2ToDocument(item);
+  const result = await collection.updateOne(
+    { wmId: doc.wmId },
+    {
+      $setOnInsert: doc,
+    },
+    { upsert: true },
+  );
+  return result.upsertedCount > 0;
+}
+
+/**
+ * Get webmentions with filters
+ * @param {object} collection - MongoDB collection
+ * @param {object} options - Query options
+ * @returns {Promise<{items: Array, total: number}>}
+ */
+export async function getWebmentions(collection, options = {}) {
+  const {
+    target,
+    wmProperty,
+    showHidden = false,
+    page = 0,
+    perPage = 20,
+  } = options;
+
+  const query = {};
+
+  if (!showHidden) {
+    query.hidden = { $ne: true };
+  }
+
+  if (target) {
+    // Match with and without trailing slash
+    const targetClean = target.replace(/\/$/, "");
+    query.wmTarget = { $in: [targetClean, targetClean + "/"] };
+  }
+
+  if (wmProperty) {
+    query.wmProperty = wmProperty;
+  }
+
+  const total = await collection.countDocuments(query);
+  const items = await collection
+    .find(query)
+    .sort({ wmReceived: -1 })
+    .skip(page * perPage)
+    .limit(perPage)
+    .toArray();
+
+  return { items, total };
+}
+
+/**
+ * Get webmention counts
+ * @param {object} collection - MongoDB collection
+ * @returns {Promise<{total: number, hidden: number, visible: number}>}
+ */
+export async function getWebmentionCounts(collection) {
+  const total = await collection.countDocuments({});
+  const hidden = await collection.countDocuments({ hidden: true });
+  return { total, hidden, visible: total - hidden };
+}
+
+/**
+ * Get the highest wmId in the collection (for incremental sync)
+ * @param {object} collection - MongoDB collection
+ * @returns {Promise<number>} Highest wmId or 0
+ */
+export async function getMaxWmId(collection) {
+  const result = await collection
+    .find({})
+    .sort({ wmId: -1 })
+    .limit(1)
+    .toArray();
+  return result.length > 0 ? result[0].wmId : 0;
+}
+
+/**
+ * Hide a webmention
+ * @param {object} collection - MongoDB collection
+ * @param {number} wmId - Webmention ID
+ * @param {string} reason - Reason ("manual", "blocklist", "privacy")
+ */
+export async function hideWebmention(collection, wmId, reason = "manual") {
+  await collection.updateOne(
+    { wmId },
+    { $set: { hidden: true, hiddenAt: new Date(), hiddenReason: reason } },
+  );
+}
+
+/**
+ * Unhide a webmention
+ * @param {object} collection - MongoDB collection
+ * @param {number} wmId - Webmention ID
+ */
+export async function unhideWebmention(collection, wmId) {
+  await collection.updateOne(
+    { wmId },
+    { $set: { hidden: false, hiddenAt: null, hiddenReason: null } },
+  );
+}
+
+/**
+ * Hide all webmentions from a domain
+ * @param {object} collection - MongoDB collection
+ * @param {string} domain - Domain to hide
+ * @param {string} reason - Reason
+ * @returns {Promise<number>} Number of mentions hidden
+ */
+export async function hideByDomain(collection, domain, reason = "blocklist") {
+  const result = await collection.updateMany(
+    { sourceDomain: domain, hidden: { $ne: true } },
+    { $set: { hidden: true, hiddenAt: new Date(), hiddenReason: reason } },
+  );
+  return result.modifiedCount;
+}
+
+/**
+ * Unhide webmentions from a domain that were hidden by blocklist
+ * @param {object} collection - MongoDB collection
+ * @param {string} domain - Domain to unhide
+ * @returns {Promise<number>} Number of mentions unhidden
+ */
+export async function unhideByDomain(collection, domain) {
+  const result = await collection.updateMany(
+    { sourceDomain: domain, hiddenReason: "blocklist" },
+    { $set: { hidden: false, hiddenAt: null, hiddenReason: null } },
+  );
+  return result.modifiedCount;
+}
+
+/**
+ * Permanently delete all webmentions from a domain (for privacy removal)
+ * @param {object} collection - MongoDB collection
+ * @param {string} domain - Domain
+ * @returns {Promise<number>} Number deleted
+ */
+export async function deleteByDomain(collection, domain) {
+  const result = await collection.deleteMany({ sourceDomain: domain });
+  return result.deletedCount;
+}
+
+/**
+ * Delete all webmentions (for full re-sync)
+ * @param {object} collection - MongoDB collection
+ * @returns {Promise<number>} Number deleted
+ */
+export async function deleteAll(collection) {
+  const result = await collection.deleteMany({});
+  return result.deletedCount;
+}

package/lib/sync.js

@@ -0,0 +1,289 @@
+/**
+ * Background sync from webmention.io
+ */
+
+import { extractDomain } from "./utils.js";
+import {
+  ensureIndexes,
+  upsertWebmention,
+  getMaxWmId,
+  deleteAll,
+  hideByDomain,
+} from "./storage/webmentions.js";
+import {
+  ensureBlocklistIndexes,
+  getBlockedDomainSet,
+} from "./storage/blocklist.js";
+
+let syncInterval = null;
+let syncState = {
+  lastSync: null,
+  syncing: false,
+  lastError: null,
+  mentionsAdded: 0,
+  mentionsFiltered: 0,
+};
+
+/**
+ * Get current sync state
+ * @returns {object}
+ */
+export function getSyncState() {
+  return { ...syncState };
+}
+
+/**
+ * Start background sync
+ * @param {object} Indiekit - Indiekit instance
+ * @param {object} options - Plugin options
+ */
+export function startSync(Indiekit, options) {
+  const intervalMs = options.syncInterval || 900_000; // 15 minutes
+
+  console.log(
+    `[Webmentions] Starting background sync with ${intervalMs / 60_000}min interval`,
+  );
+
+  // Initial sync after delay
+  setTimeout(() => {
+    runSync(Indiekit, options).catch((err) => {
+      console.error("[Webmentions] Initial sync error:", err.message);
+    });
+  }, 10_000);
+
+  // Recurring sync
+  syncInterval = setInterval(() => {
+    runSync(Indiekit, options).catch((err) => {
+      console.error("[Webmentions] Sync error:", err.message);
+    });
+  }, intervalMs);
+}
+
+/**
+ * Stop background sync
+ */
+export function stopSync() {
+  if (syncInterval) {
+    clearInterval(syncInterval);
+    syncInterval = null;
+  }
+}
+
+/**
+ * Run a single incremental sync cycle
+ * @param {object} dbOrIndiekit - Database or Indiekit instance
+ * @param {object} options - Plugin options
+ * @returns {Promise<object>}
+ */
+export async function runSync(dbOrIndiekit, options) {
+  const db = dbOrIndiekit.database || dbOrIndiekit;
+  if (!db || typeof db.collection !== "function") {
+    syncState.lastError = "No database available";
+    return { error: syncState.lastError };
+  }
+
+  if (syncState.syncing) {
+    return { error: "Sync already in progress" };
+  }
+
+  syncState.syncing = true;
+  syncState.lastError = null;
+  syncState.mentionsAdded = 0;
+  syncState.mentionsFiltered = 0;
+
+  try {
+    const wmCollection = db.collection("webmentions");
+    const blockCollection = db.collection("webmentionBlocklist");
+
+    await ensureIndexes(wmCollection);
+    await ensureBlocklistIndexes(blockCollection);
+
+    // Get highest wmId for incremental sync
+    const sinceId = await getMaxWmId(wmCollection);
+
+    // Get blocked domains
+    const blockedDomains = await getBlockedDomainSet(blockCollection);
+
+    // Fetch pages from webmention.io
+    let page = 0;
+    let hasMore = true;
+    const perPage = 100;
+
+    while (hasMore) {
+      const items = await fetchPage(options, { page, perPage, sinceId });
+
+      if (!items || items.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      for (const item of items) {
+        const domain = extractDomain(item.author?.url || item.url || "");
+
+        if (domain && blockedDomains.has(domain)) {
+          syncState.mentionsFiltered++;
+          continue;
+        }
+
+        const isNew = await upsertWebmention(wmCollection, item);
+        if (isNew) {
+          syncState.mentionsAdded++;
+        }
+      }
+
+      page++;
+
+      // Rate limit: small delay between pages
+      if (items.length >= perPage) {
+        await delay(500);
+      } else {
+        hasMore = false;
+      }
+    }
+
+    syncState.lastSync = new Date();
+    syncState.syncing = false;
+
+    console.log(
+      `[Webmentions] Sync complete: ${syncState.mentionsAdded} new, ${syncState.mentionsFiltered} filtered`,
+    );
+
+    return {
+      mentionsAdded: syncState.mentionsAdded,
+      mentionsFiltered: syncState.mentionsFiltered,
+    };
+  } catch (error) {
+    syncState.lastError = error.message;
+    syncState.syncing = false;
+    console.error("[Webmentions] Sync failed:", error.message);
+    return { error: error.message };
+  }
+}
+
+/**
+ * Run a full re-sync (clear + fetch all)
+ * @param {object} dbOrIndiekit - Database or Indiekit instance
+ * @param {object} options - Plugin options
+ * @returns {Promise<object>}
+ */
+export async function runFullSync(dbOrIndiekit, options) {
+  const db = dbOrIndiekit.database || dbOrIndiekit;
+  if (!db || typeof db.collection !== "function") {
+    return { error: "No database available" };
+  }
+
+  if (syncState.syncing) {
+    return { error: "Sync already in progress" };
+  }
+
+  syncState.syncing = true;
+  syncState.lastError = null;
+  syncState.mentionsAdded = 0;
+  syncState.mentionsFiltered = 0;
+
+  try {
+    const wmCollection = db.collection("webmentions");
+    const blockCollection = db.collection("webmentionBlocklist");
+
+    await ensureIndexes(wmCollection);
+    await ensureBlocklistIndexes(blockCollection);
+
+    // Clear all existing webmentions
+    const deleted = await deleteAll(wmCollection);
+    console.log(`[Webmentions] Full sync: cleared ${deleted} existing mentions`);
+
+    // Get blocked domains
+    const blockedDomains = await getBlockedDomainSet(blockCollection);
+
+    // Fetch ALL pages from webmention.io (no sinceId)
+    let page = 0;
+    let hasMore = true;
+    const perPage = 100;
+
+    while (hasMore) {
+      const items = await fetchPage(options, { page, perPage });
+
+      if (!items || items.length === 0) {
+        hasMore = false;
+        break;
+      }
+
+      for (const item of items) {
+        const domain = extractDomain(item.author?.url || item.url || "");
+
+        if (domain && blockedDomains.has(domain)) {
+          syncState.mentionsFiltered++;
+          continue;
+        }
+
+        const isNew = await upsertWebmention(wmCollection, item);
+        if (isNew) {
+          syncState.mentionsAdded++;
+        }
+      }
+
+      page++;
+
+      // Rate limit between pages
+      if (items.length >= perPage) {
+        await delay(1000);
+      } else {
+        hasMore = false;
+      }
+    }
+
+    syncState.lastSync = new Date();
+    syncState.syncing = false;
+
+    console.log(
+      `[Webmentions] Full sync complete: ${syncState.mentionsAdded} imported, ${syncState.mentionsFiltered} filtered`,
+    );
+
+    return {
+      mentionsAdded: syncState.mentionsAdded,
+      mentionsFiltered: syncState.mentionsFiltered,
+    };
+  } catch (error) {
+    syncState.lastError = error.message;
+    syncState.syncing = false;
+    console.error("[Webmentions] Full sync failed:", error.message);
+    return { error: error.message };
+  }
+}
+
+/**
+ * Fetch a single page from webmention.io API
+ * @param {object} options - Plugin options (token, domain)
+ * @param {object} params - Fetch params (page, perPage, sinceId)
+ * @returns {Promise<Array>} Array of JF2 entries
+ */
+async function fetchPage(options, params = {}) {
+  const url = new URL("https://webmention.io/api/mentions.jf2");
+  url.searchParams.set("token", options.token);
+  url.searchParams.set("domain", options.domain);
+  url.searchParams.set("per-page", String(params.perPage || 100));
+
+  if (params.page) {
+    url.searchParams.set("page", String(params.page));
+  }
+
+  if (params.sinceId) {
+    url.searchParams.set("since_id", String(params.sinceId));
+  }
+
+  const response = await fetch(url.href, {
+    headers: { accept: "application/json" },
+    signal: AbortSignal.timeout(15_000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`webmention.io returned ${response.status}`);
+  }
+
+  const body = await response.json();
+  return body?.children || [];
+}
+
+function delay(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}

package/lib/utils.js

@@ -0,0 +1,115 @@
+import sanitize from "sanitize-html";
+
+/**
+ * Get mention type from `wm-property`
+ * @param {string} wmProperty - Webmention.io `wm-property` value
+ * @returns {string} Icon name
+ */
+export const getMentionType = (wmProperty) => {
+  switch (true) {
+    case wmProperty === "in-reply-to": {
+      return "reply";
+    }
+    case wmProperty === "like-of": {
+      return "like";
+    }
+    case wmProperty === "repost-of": {
+      return "repost";
+    }
+    case wmProperty === "bookmark-of": {
+      return "bookmark";
+    }
+    case wmProperty === "rsvp": {
+      return "rsvp";
+    }
+    default: {
+      return "mention";
+    }
+  }
+};
+
+const upperFirst = (string) => {
+  return String(string).charAt(0).toUpperCase() + String(string).slice(1);
+};
+
+/**
+ * Get mention title
+ * @param {object} jf2 - JF2
+ * @returns {string} Mention title
+ */
+export const getMentionTitle = (jf2) => {
+  let type = getMentionType(jf2["wm-property"]);
+  type = upperFirst(type).replace("Rsvp", "RSVP");
+
+  return jf2.name || type;
+};
+
+/**
+ * Get author name
+ * @param {object} jf2 - JF2
+ * @returns {string} Author name or URL fallback
+ */
+export const getAuthorName = (jf2) => {
+  if (jf2.author?.name) return jf2.author.name;
+
+  try {
+    let url = jf2.author?.url || jf2.url;
+    url = new URL(url);
+    return url.hostname + url.pathname.replace(/\/$/, "");
+  } catch {
+    return "Unknown";
+  }
+};
+
+/**
+ * Normalise paragraphs
+ * @param {string} html - HTML
+ * @returns {string} HTML with normalised paragraphs
+ */
+export const normaliseParagraphs = (html) => {
+  html = `<p>${html}</p>`;
+  html = html.replaceAll(/<br\s*\/?>\s*<br\s*\/?>/g, "</p><p>");
+  return html;
+};
+
+/**
+ * Sanitise incoming mention HTML
+ * @param {string} html - HTML
+ * @returns {string} Sanitised HTML
+ */
+export const sanitiseHtml = (html) => {
+  html = normaliseParagraphs(html);
+  html = sanitize(html, {
+    exclusiveFilter: function (frame) {
+      return (
+        (frame.tag === "a" &&
+          frame.attribs?.href?.includes("brid.gy") &&
+          !frame.text.trim()) ||
+        (frame.tag === "p" && !frame.text.trim())
+      );
+    },
+    transformTags: {
+      h1: "h3",
+      h2: "h4",
+      h3: "h5",
+      h4: "h6",
+      h5: "h6",
+      h6: "h6",
+    },
+  });
+
+  return html;
+};
+
+/**
+ * Extract domain from a URL string
+ * @param {string} url - URL
+ * @returns {string|null} Domain or null
+ */
+export const extractDomain = (url) => {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return null;
+  }
+};