@rmdes/indiekit-endpoint-webmention-io 1.0.6 → 1.0.7

package/lib/hcard.js

@@ -0,0 +1,236 @@
+/**
+ * h-card discovery for author photo enrichment
+ *
+ * When webmention.io returns empty author photos (common with IndieWeb
+ * sites that only have h-cards on their homepage, not individual post pages),
+ * this module fetches the source domain's homepage and parses the h-card
+ * to find the author's photo and URL.
+ *
+ * Results are cached in MongoDB (7-day TTL) and in-memory (process lifetime)
+ * to avoid redundant HTTP requests.
+ */
+
+// In-memory cache: domain -> { photoUrl, authorUrl } | null
+const memoryCache = new Map();
+
+const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+
+/**
+ * Discover author data (photo + URL) from a domain's homepage h-card
+ * @param {string} domain - Domain to check (e.g., "crowdersoup.com")
+ * @param {object} [cacheCollection] - MongoDB collection for persistent cache
+ * @returns {Promise<{photoUrl: string|null, authorUrl: string|null}>}
+ */
+export async function discoverAuthorData(domain, cacheCollection) {
+  if (!domain) return { photoUrl: null, authorUrl: null };
+
+  // Check in-memory cache
+  if (memoryCache.has(domain)) {
+    return memoryCache.get(domain);
+  }
+
+  // Check MongoDB cache
+  if (cacheCollection) {
+    try {
+      const cached = await cacheCollection.findOne({ domain });
+      if (cached && !isExpired(cached.fetchedAt)) {
+        const result = {
+          photoUrl: cached.photoUrl || null,
+          authorUrl: cached.authorUrl || null,
+        };
+        memoryCache.set(domain, result);
+        return result;
+      }
+    } catch {
+      // Cache read failure is non-fatal
+    }
+  }
+
+  // Fetch and parse the homepage h-card
+  let result = { photoUrl: null, authorUrl: null };
+  try {
+    result = await fetchHcardData(domain);
+  } catch (error) {
+    console.log(
+      `[Webmentions] h-card discovery failed for ${domain}: ${error.message}`,
+    );
+  }
+
+  // Store in caches
+  memoryCache.set(domain, result);
+
+  if (cacheCollection) {
+    try {
+      await cacheCollection.updateOne(
+        { domain },
+        {
+          $set: {
+            domain,
+            photoUrl: result.photoUrl,
+            authorUrl: result.authorUrl,
+            fetchedAt: new Date().toISOString(),
+          },
+        },
+        { upsert: true },
+      );
+    } catch {
+      // Cache write failure is non-fatal
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Fetch a domain's homepage and extract h-card data
+ * @param {string} domain
+ * @returns {Promise<{photoUrl: string|null, authorUrl: string|null}>}
+ */
+async function fetchHcardData(domain) {
+  const baseUrl = `https://${domain}`;
+  const response = await fetch(baseUrl, {
+    headers: {
+      accept: "text/html",
+      "user-agent": "Indiekit-Webmention/1.0 (h-card discovery)",
+    },
+    signal: AbortSignal.timeout(10_000),
+    redirect: "follow",
+  });
+
+  if (!response.ok) {
+    return { photoUrl: null, authorUrl: null };
+  }
+
+  const contentType = response.headers.get("content-type") || "";
+  if (!contentType.includes("text/html")) {
+    return { photoUrl: null, authorUrl: null };
+  }
+
+  const html = await response.text();
+  return parseHcard(html, baseUrl);
+}
+
+/**
+ * Parse h-card microformat from HTML to extract photo and URL
+ * @param {string} html - Page HTML
+ * @param {string} baseUrl - Base URL for resolving relative URLs
+ * @returns {{photoUrl: string|null, authorUrl: string|null}}
+ */
+export function parseHcard(html, baseUrl) {
+  let photoUrl = null;
+  let authorUrl = null;
+
+  // Find u-photo images and u-url links in the page.
+  // These are microformat-specific class names that are unlikely to
+  // appear outside h-card context.
+  photoUrl = findUPhoto(html);
+  authorUrl = findUUrl(html);
+
+  // Resolve relative URLs
+  if (photoUrl) {
+    photoUrl = resolveUrl(photoUrl, baseUrl);
+  }
+  if (authorUrl) {
+    authorUrl = resolveUrl(authorUrl, baseUrl);
+  }
+
+  return { photoUrl, authorUrl };
+}
+
+/**
+ * Find u-photo value from img elements
+ * @param {string} html
+ * @returns {string|null}
+ */
+function findUPhoto(html) {
+  const imgRegex = /<img\s[^>]*?>/gi;
+  let match;
+
+  while ((match = imgRegex.exec(html)) !== null) {
+    const tag = match[0];
+
+    if (!hasClass(tag, "u-photo")) continue;
+
+    const src = getAttr(tag, "src");
+    if (src) return src;
+  }
+
+  return null;
+}
+
+/**
+ * Find u-url or u-uid value from anchor elements
+ * @param {string} html
+ * @returns {string|null}
+ */
+function findUUrl(html) {
+  const aRegex = /<a\s[^>]*?>/gi;
+  let match;
+
+  while ((match = aRegex.exec(html)) !== null) {
+    const tag = match[0];
+
+    if (!hasClass(tag, "u-url") && !hasClass(tag, "u-uid")) continue;
+
+    const href = getAttr(tag, "href");
+    if (href) return href;
+  }
+
+  return null;
+}
+
+/**
+ * Check if an HTML tag has a specific class
+ * @param {string} tag - HTML tag string
+ * @param {string} className - Class to check for
+ * @returns {boolean}
+ */
+function hasClass(tag, className) {
+  const classMatch = tag.match(/class=["']([^"']*)["']/i);
+  if (!classMatch) return false;
+  return classMatch[1].split(/\s+/).includes(className);
+}
+
+/**
+ * Get an attribute value from an HTML tag
+ * @param {string} tag - HTML tag string
+ * @param {string} attr - Attribute name
+ * @returns {string|null}
+ */
+function getAttr(tag, attr) {
+  const regex = new RegExp(`${attr}=["']([^"']*)["']`, "i");
+  const match = tag.match(regex);
+  return match ? match[1] : null;
+}
+
+/**
+ * Resolve a potentially relative URL against a base
+ * @param {string} url
+ * @param {string} base
+ * @returns {string}
+ */
+function resolveUrl(url, base) {
+  try {
+    return new URL(url, base).href;
+  } catch {
+    return url;
+  }
+}
+
+/**
+ * Check if a cached entry has expired
+ * @param {string} fetchedAt - ISO timestamp
+ * @returns {boolean}
+ */
+function isExpired(fetchedAt) {
+  if (!fetchedAt) return true;
+  const age = Date.now() - new Date(fetchedAt).getTime();
+  return age > CACHE_TTL_MS;
+}
+
+/**
+ * Clear the in-memory cache (useful for testing)
+ */
+export function clearMemoryCache() {
+  memoryCache.clear();
+}

package/lib/storage/webmentions.js

@@ -250,3 +250,52 @@ export async function deleteAll(collection) {
   const result = await collection.deleteMany({});
   return result.deletedCount;
 }
+
+/**
+ * Get distinct source domains that have entries with missing author photos
+ * @param {object} collection - MongoDB collection
+ * @returns {Promise<string[]>} Array of domain strings
+ */
+export async function getDomainsWithMissingPhotos(collection) {
+  return collection.distinct("sourceDomain", {
+    authorPhoto: { $in: [null, ""] },
+    sourceDomain: { $ne: null },
+  });
+}
+
+/**
+ * Update author photo and URL for all entries from a domain that are missing them
+ * @param {object} collection - MongoDB collection
+ * @param {string} domain - Source domain
+ * @param {object} data - Author data to set
+ * @param {string} [data.photoUrl] - Author photo URL
+ * @param {string} [data.authorUrl] - Author profile URL
+ * @returns {Promise<number>} Number of entries updated
+ */
+export async function updateAuthorDataByDomain(collection, domain, data) {
+  const setFields = {};
+  if (data.photoUrl) {
+    setFields.authorPhoto = data.photoUrl;
+  }
+  if (data.authorUrl) {
+    setFields.authorUrl = data.authorUrl;
+  }
+
+  if (Object.keys(setFields).length === 0) return 0;
+
+  // Only update entries that are missing the data
+  const query = { sourceDomain: domain };
+  const conditions = [];
+  if (data.photoUrl) {
+    conditions.push({ authorPhoto: { $in: [null, ""] } });
+  }
+  if (data.authorUrl) {
+    conditions.push({ authorUrl: { $in: [null, ""] } });
+  }
+  if (conditions.length > 0) {
+    query.$or = conditions;
+  }
+
+  const result = await collection.updateMany(query, { $set: setFields });
+  return result.modifiedCount;
+}

package/lib/sync.js

@@ -9,11 +9,14 @@ import {
   getMaxWmId,
   deleteAll,
   hideByDomain,
+  getDomainsWithMissingPhotos,
+  updateAuthorDataByDomain,
 } from "./storage/webmentions.js";
 import {
   ensureBlocklistIndexes,
   getBlockedDomainSet,
 } from "./storage/blocklist.js";
+import { discoverAuthorData } from "./hcard.js";
 
 let syncInterval = null;
 let syncState = {
@@ -141,16 +144,20 @@ export async function runSync(dbOrIndiekit, options) {
       }
     }
 
+    // Enrich entries with missing author photos via h-card discovery
+    const enriched = await enrichMissingPhotos(db, wmCollection);
+
     syncState.lastSync = new Date().toISOString();
     syncState.syncing = false;
 
     console.log(
-      `[Webmentions] Sync complete: ${syncState.mentionsAdded} new, ${syncState.mentionsFiltered} filtered`,
+      `[Webmentions] Sync complete: ${syncState.mentionsAdded} new, ${syncState.mentionsFiltered} filtered, ${enriched} enriched`,
     );
 
     return {
       mentionsAdded: syncState.mentionsAdded,
       mentionsFiltered: syncState.mentionsFiltered,
+      mentionsEnriched: enriched,
     };
   } catch (error) {
     syncState.lastError = error.message;
@@ -232,16 +239,20 @@ export async function runFullSync(dbOrIndiekit, options) {
       }
     }
 
+    // Enrich entries with missing author photos via h-card discovery
+    const enriched = await enrichMissingPhotos(db, wmCollection);
+
     syncState.lastSync = new Date().toISOString();
     syncState.syncing = false;
 
     console.log(
-      `[Webmentions] Full sync complete: ${syncState.mentionsAdded} imported, ${syncState.mentionsFiltered} filtered`,
+      `[Webmentions] Full sync complete: ${syncState.mentionsAdded} imported, ${syncState.mentionsFiltered} filtered, ${enriched} enriched`,
     );
 
     return {
       mentionsAdded: syncState.mentionsAdded,
       mentionsFiltered: syncState.mentionsFiltered,
+      mentionsEnriched: enriched,
     };
   } catch (error) {
     syncState.lastError = error.message;
@@ -251,6 +262,55 @@ export async function runFullSync(dbOrIndiekit, options) {
   }
 }
 
+/**
+ * Enrich webmention entries that have missing author photos by discovering
+ * h-card data from the source domain's homepage.
+ * @param {object} db - MongoDB database instance
+ * @param {object} wmCollection - Webmentions collection
+ * @returns {Promise<number>} Total entries updated
+ */
+async function enrichMissingPhotos(db, wmCollection) {
+  let totalUpdated = 0;
+
+  try {
+    const domains = await getDomainsWithMissingPhotos(wmCollection);
+
+    if (domains.length === 0) return 0;
+
+    const cacheCollection = db.collection("hcardCache");
+    await cacheCollection.createIndex({ domain: 1 }, { unique: true });
+
+    for (const domain of domains) {
+      const data = await discoverAuthorData(domain, cacheCollection);
+
+      if (data.photoUrl || data.authorUrl) {
+        const updated = await updateAuthorDataByDomain(
+          wmCollection,
+          domain,
+          data,
+        );
+        totalUpdated += updated;
+
+        if (updated > 0) {
+          console.log(
+            `[Webmentions] Enriched ${updated} entries for ${domain}`,
+          );
+        }
+      }
+
+      // Small delay between domain lookups
+      await delay(200);
+    }
+  } catch (error) {
+    console.error(
+      "[Webmentions] h-card enrichment error:",
+      error.message,
+    );
+  }
+
+  return totalUpdated;
+}
+
 /**
  * Fetch a single page from webmention.io API
  * @param {object} options - Plugin options (token, domain)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-webmention-io",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Webmention moderation endpoint for Indiekit. Syncs webmentions from webmention.io into MongoDB with delete, block, and privacy removal capabilities.",
   "keywords": [
     "indiekit",