@rmdes/indiekit-endpoint-conversations 2.3.1 → 2.4.0

package/lib/controllers/conversations.js

@@ -80,6 +80,19 @@ async function dashboard(request, response) {
       }
     }
 
+    // Get item counts by channel (ingestion path)
+    let channelCounts = {};
+    if (itemsCollection) {
+      const counts = await itemsCollection
+        .aggregate([
+          { $group: { _id: "$channel", count: { $sum: 1 } } },
+        ])
+        .toArray();
+      for (const c of counts) {
+        channelCounts[c._id] = c.count;
+      }
+    }
+
     response.render("conversations", {
       title: response.__
         ? response.__("conversations.title")
@@ -90,6 +103,7 @@ async function dashboard(request, response) {
       summaries,
       recentItems,
       platformCounts,
+      channelCounts,
       typeCounts,
       baseUrl: config.mountPath || "/conversations",
     });
@@ -103,6 +117,7 @@ async function dashboard(request, response) {
       summaries: [],
       recentItems: [],
       platformCounts: {},
+      channelCounts: {},
       typeCounts: {},
     });
   }
@@ -399,6 +414,7 @@ async function ingest(request, response) {
     const item = {
       canonical_url: canonicalUrl,
       source: classification.source,
+      channel: "webhook",
       type: classification.type,
       author: webmention.author || {
         name: "Unknown",

package/lib/matching/syndication-map.js

@@ -33,10 +33,10 @@ export async function findCanonicalPost(application, syndicationUrl) {
 export async function resolveCanonicalUrl(application, targetUrl, siteUrl) {
   // If the target is already on our domain, it's likely canonical
   if (targetUrl.startsWith(siteUrl)) {
-    return targetUrl;
+    return targetUrl.replace(/\/$/, "");
   }
 
   // Otherwise try to find via syndication reverse lookup
   const canonical = await findCanonicalPost(application, targetUrl);
-  return canonical || targetUrl;
+  return (canonical || targetUrl).replace(/\/$/, "");
 }

package/lib/polling/scheduler.js

@@ -95,11 +95,89 @@ export async function runPollCycle(indiekit, options) {
     await pollActivityPub(indiekit, stateCollection, state);
   }
 
+  // Normalize trailing slashes in canonical_url (one-time migration)
+  await normalizeCanonicalUrls(indiekit, stateCollection);
+
   // Backfill missing avatars from ap_notifications (one-time sweep per cycle)
   await backfillMissingAvatars(indiekit, stateCollection);
 
   // Backfill platform names for items stored as "activitypub" (one-time)
   await backfillPlatformNames(indiekit, stateCollection);
+
+  // Backfill channel field for items predating its introduction (one-time)
+  await backfillChannelField(indiekit, stateCollection);
+}
+
+/**
+ * One-time migration: normalize trailing slashes in canonical_url.
+ * Strips trailing slash from all canonical_url values, then removes
+ * duplicates that were created by the slash inconsistency.
+ */
+async function normalizeCanonicalUrls(indiekit, stateCollection) {
+  try {
+    const state = await stateCollection.findOne({ _id: "poll_cursors" });
+    if (state?.canonical_url_normalized) return;
+
+    const itemsCollection = indiekit.collections.get("conversation_items");
+    if (!itemsCollection) return;
+
+    // Find all items with trailing slash in canonical_url
+    const itemsWithSlash = await itemsCollection
+      .find({ canonical_url: /\/$/ })
+      .toArray();
+
+    if (itemsWithSlash.length === 0) {
+      await stateCollection.findOneAndUpdate(
+        { _id: "poll_cursors" },
+        { $set: { canonical_url_normalized: true } },
+        { upsert: true },
+      );
+      return;
+    }
+
+    let updated = 0;
+    let deduped = 0;
+
+    for (const item of itemsWithSlash) {
+      const normalized = item.canonical_url.replace(/\/$/, "");
+
+      // Check if a non-slash version already exists with same platform_id
+      const existing = await itemsCollection.findOne({
+        canonical_url: normalized,
+        platform_id: item.platform_id,
+      });
+
+      if (existing) {
+        // Duplicate — remove the trailing-slash version
+        await itemsCollection.deleteOne({ _id: item._id });
+        deduped++;
+      } else {
+        // No duplicate — just normalize the URL
+        await itemsCollection.updateOne(
+          { _id: item._id },
+          { $set: { canonical_url: normalized } },
+        );
+        updated++;
+      }
+    }
+
+    if (updated > 0 || deduped > 0) {
+      console.info(
+        `[Conversations] URL normalization: updated ${updated}, removed ${deduped} duplicates from ${itemsWithSlash.length} items with trailing slashes`,
+      );
+    }
+
+    await stateCollection.findOneAndUpdate(
+      { _id: "poll_cursors" },
+      { $set: { canonical_url_normalized: true } },
+      { upsert: true },
+    );
+  } catch (error) {
+    console.warn(
+      "[Conversations] URL normalization error:",
+      error.message,
+    );
+  }
 }
 
 /**
@@ -302,6 +380,84 @@ async function backfillPlatformNames(indiekit, stateCollection) {
   }
 }
 
+/**
+ * Backfill the `channel` field for existing items that predate its introduction.
+ * Derives channel from platform_id prefix and bridgy_url presence.
+ * One-time operation — marks complete after first successful run.
+ */
+async function backfillChannelField(indiekit, stateCollection) {
+  try {
+    const itemsCollection = indiekit.collections.get("conversation_items");
+    if (!itemsCollection) return;
+
+    const state = await stateCollection.findOne({ _id: "poll_cursors" });
+    if (state?.channel_backfill_complete) return;
+
+    // Find all items missing the channel field
+    const itemsWithoutChannel = await itemsCollection
+      .find({ channel: { $exists: false } })
+      .project({ _id: 1, platform_id: 1, bridgy_url: 1 })
+      .toArray();
+
+    if (itemsWithoutChannel.length === 0) {
+      await stateCollection.findOneAndUpdate(
+        { _id: "poll_cursors" },
+        { $set: { channel_backfill_complete: true } },
+        { upsert: true },
+      );
+      return;
+    }
+
+    // Group items by derived channel for batched updateMany
+    const groups = new Map();
+
+    for (const item of itemsWithoutChannel) {
+      const pid = item.platform_id || "";
+      const hasBridgy = !!item.bridgy_url;
+      let channel;
+
+      if (pid.startsWith("activitypub:")) {
+        channel = "activitypub_inbox";
+      } else if (hasBridgy) {
+        channel = "webhook";
+      } else if (pid.startsWith("mastodon:")) {
+        channel = "mastodon_api";
+      } else if (pid.startsWith("bluesky:")) {
+        channel = "bluesky_api";
+      } else {
+        channel = "webhook";
+      }
+
+      if (!groups.has(channel)) groups.set(channel, []);
+      groups.get(channel).push(item._id);
+    }
+
+    let updated = 0;
+
+    for (const [channel, ids] of groups) {
+      const result = await itemsCollection.updateMany(
+        { _id: { $in: ids } },
+        { $set: { channel } },
+      );
+      updated += result.modifiedCount;
+    }
+
+    if (updated > 0) {
+      console.info(
+        `[Conversations] Channel backfill: set channel on ${updated} items`,
+      );
+    }
+
+    await stateCollection.findOneAndUpdate(
+      { _id: "poll_cursors" },
+      { $set: { channel_backfill_complete: true } },
+      { upsert: true },
+    );
+  } catch (error) {
+    console.warn("[Conversations] Channel backfill error:", error.message);
+  }
+}
+
 /**
  * Poll Mastodon notifications and store matching interactions
  */
@@ -356,6 +512,7 @@ async function pollMastodon(indiekit, stateCollection, state, credentials) {
         await upsertConversationItem(indiekit, {
           canonical_url: canonicalUrl,
           source: "mastodon",
+          channel: "mastodon_api",
           type: notification.type,
           author: notification.author,
           content: notification.content,
@@ -452,6 +609,7 @@ async function pollBluesky(indiekit, stateCollection, state, credentials) {
         await upsertConversationItem(indiekit, {
           canonical_url: canonicalUrl,
           source: "bluesky",
+          channel: "bluesky_api",
           type: notification.type,
           author: notification.author,
           content: notification.content,
@@ -569,6 +727,7 @@ async function pollActivityPub(indiekit, stateCollection, state) {
       await upsertConversationItem(indiekit, {
         canonical_url: interaction.canonical_url,
         source: interaction.platform,
+        channel: "activitypub_inbox",
         type: interaction.type,
         author: interaction.author,
         content: interaction.content,

package/lib/storage/conversation-items.js

@@ -22,6 +22,11 @@ function getCollection(application) {
 export async function upsertConversationItem(application, item) {
   const collection = getCollection(application);
 
+  // Normalize canonical_url — strip trailing slash for consistent deduplication
+  if (item.canonical_url) {
+    item.canonical_url = item.canonical_url.replace(/\/$/, "");
+  }
+
   const result = await collection.findOneAndUpdate(
     {
       canonical_url: item.canonical_url,
@@ -159,4 +164,9 @@ export async function createIndexes(application) {
     { name: "source_filter" },
   );
 
+  await collection.createIndex(
+    { channel: 1, received_at: -1 },
+    { name: "channel_filter" },
+  );
+
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-conversations",
-  "version": "2.3.1",
+  "version": "2.4.0",
   "description": "Conversation aggregation endpoint for Indiekit. Backend enrichment service that polls Mastodon/Bluesky notifications and serves JF2-compatible data for the interactions page.",
   "keywords": [
     "indiekit",

package/views/conversations.njk

@@ -37,7 +37,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.mastodon or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.mastodon_api or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -71,7 +71,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.bluesky or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.bluesky_api or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -105,7 +105,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.activitypub or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.activitypub_inbox or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -130,7 +130,7 @@
         POST {{ baseUrl }}/ingest
       </code>
       <p style="font-size: 0.85em; margin: 0.25rem 0">
-        {{ platformCounts.webmention or 0 }} {{ __("conversations.dashboard.itemsReceived") }}
+        {{ channelCounts.webhook or 0 }} {{ __("conversations.dashboard.itemsReceived") }}
       </p>
     </div>
   </div>