@rmdes/indiekit-endpoint-conversations 2.3.2 → 2.4.0

package/lib/controllers/conversations.js

@@ -80,6 +80,19 @@ async function dashboard(request, response) {
       }
     }
 
+    // Get item counts by channel (ingestion path)
+    let channelCounts = {};
+    if (itemsCollection) {
+      const counts = await itemsCollection
+        .aggregate([
+          { $group: { _id: "$channel", count: { $sum: 1 } } },
+        ])
+        .toArray();
+      for (const c of counts) {
+        channelCounts[c._id] = c.count;
+      }
+    }
+
     response.render("conversations", {
       title: response.__
         ? response.__("conversations.title")
@@ -90,6 +103,7 @@ async function dashboard(request, response) {
       summaries,
       recentItems,
       platformCounts,
+      channelCounts,
       typeCounts,
       baseUrl: config.mountPath || "/conversations",
     });
@@ -103,6 +117,7 @@ async function dashboard(request, response) {
       summaries: [],
       recentItems: [],
       platformCounts: {},
+      channelCounts: {},
       typeCounts: {},
     });
   }
@@ -399,6 +414,7 @@ async function ingest(request, response) {
     const item = {
       canonical_url: canonicalUrl,
       source: classification.source,
+      channel: "webhook",
       type: classification.type,
       author: webmention.author || {
         name: "Unknown",

package/lib/polling/scheduler.js

@@ -103,6 +103,9 @@ export async function runPollCycle(indiekit, options) {
 
   // Backfill platform names for items stored as "activitypub" (one-time)
   await backfillPlatformNames(indiekit, stateCollection);
+
+  // Backfill channel field for items predating its introduction (one-time)
+  await backfillChannelField(indiekit, stateCollection);
 }
 
 /**
@@ -377,6 +380,84 @@ async function backfillPlatformNames(indiekit, stateCollection) {
   }
 }
 
+/**
+ * Backfill the `channel` field for existing items that predate its introduction.
+ * Derives channel from platform_id prefix and bridgy_url presence.
+ * One-time operation — marks complete after first successful run.
+ */
+async function backfillChannelField(indiekit, stateCollection) {
+  try {
+    const itemsCollection = indiekit.collections.get("conversation_items");
+    if (!itemsCollection) return;
+
+    const state = await stateCollection.findOne({ _id: "poll_cursors" });
+    if (state?.channel_backfill_complete) return;
+
+    // Find all items missing the channel field
+    const itemsWithoutChannel = await itemsCollection
+      .find({ channel: { $exists: false } })
+      .project({ _id: 1, platform_id: 1, bridgy_url: 1 })
+      .toArray();
+
+    if (itemsWithoutChannel.length === 0) {
+      await stateCollection.findOneAndUpdate(
+        { _id: "poll_cursors" },
+        { $set: { channel_backfill_complete: true } },
+        { upsert: true },
+      );
+      return;
+    }
+
+    // Group items by derived channel for batched updateMany
+    const groups = new Map();
+
+    for (const item of itemsWithoutChannel) {
+      const pid = item.platform_id || "";
+      const hasBridgy = !!item.bridgy_url;
+      let channel;
+
+      if (pid.startsWith("activitypub:")) {
+        channel = "activitypub_inbox";
+      } else if (hasBridgy) {
+        channel = "webhook";
+      } else if (pid.startsWith("mastodon:")) {
+        channel = "mastodon_api";
+      } else if (pid.startsWith("bluesky:")) {
+        channel = "bluesky_api";
+      } else {
+        channel = "webhook";
+      }
+
+      if (!groups.has(channel)) groups.set(channel, []);
+      groups.get(channel).push(item._id);
+    }
+
+    let updated = 0;
+
+    for (const [channel, ids] of groups) {
+      const result = await itemsCollection.updateMany(
+        { _id: { $in: ids } },
+        { $set: { channel } },
+      );
+      updated += result.modifiedCount;
+    }
+
+    if (updated > 0) {
+      console.info(
+        `[Conversations] Channel backfill: set channel on ${updated} items`,
+      );
+    }
+
+    await stateCollection.findOneAndUpdate(
+      { _id: "poll_cursors" },
+      { $set: { channel_backfill_complete: true } },
+      { upsert: true },
+    );
+  } catch (error) {
+    console.warn("[Conversations] Channel backfill error:", error.message);
+  }
+}
+
 /**
  * Poll Mastodon notifications and store matching interactions
  */
@@ -431,6 +512,7 @@ async function pollMastodon(indiekit, stateCollection, state, credentials) {
         await upsertConversationItem(indiekit, {
           canonical_url: canonicalUrl,
           source: "mastodon",
+          channel: "mastodon_api",
           type: notification.type,
           author: notification.author,
           content: notification.content,
@@ -527,6 +609,7 @@ async function pollBluesky(indiekit, stateCollection, state, credentials) {
         await upsertConversationItem(indiekit, {
           canonical_url: canonicalUrl,
           source: "bluesky",
+          channel: "bluesky_api",
           type: notification.type,
           author: notification.author,
           content: notification.content,
@@ -644,6 +727,7 @@ async function pollActivityPub(indiekit, stateCollection, state) {
       await upsertConversationItem(indiekit, {
         canonical_url: interaction.canonical_url,
         source: interaction.platform,
+        channel: "activitypub_inbox",
         type: interaction.type,
         author: interaction.author,
         content: interaction.content,

package/lib/storage/conversation-items.js

@@ -164,4 +164,9 @@ export async function createIndexes(application) {
     { name: "source_filter" },
   );
 
+  await collection.createIndex(
+    { channel: 1, received_at: -1 },
+    { name: "channel_filter" },
+  );
+
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rmdes/indiekit-endpoint-conversations",
-  "version": "2.3.2",
+  "version": "2.4.0",
   "description": "Conversation aggregation endpoint for Indiekit. Backend enrichment service that polls Mastodon/Bluesky notifications and serves JF2-compatible data for the interactions page.",
   "keywords": [
     "indiekit",

package/views/conversations.njk

@@ -37,7 +37,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.mastodon or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.mastodon_api or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -71,7 +71,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.bluesky or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.bluesky_api or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -105,7 +105,7 @@
         </p>
         {% endif %}
         <p style="font-size: 0.85em; margin: 0.25rem 0">
-          {{ platformCounts.activitypub or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
+          {{ channelCounts.activitypub_inbox or 0 }} {{ __("conversations.dashboard.itemsCollected") }}
         </p>
       {% else %}
         <p style="font-size: 0.85em; color: #6b7280; margin: 0.25rem 0">
@@ -130,7 +130,7 @@
         POST {{ baseUrl }}/ingest
       </code>
       <p style="font-size: 0.85em; margin: 0.25rem 0">
-        {{ platformCounts.webmention or 0 }} {{ __("conversations.dashboard.itemsReceived") }}
+        {{ channelCounts.webhook or 0 }} {{ __("conversations.dashboard.itemsReceived") }}
       </p>
     </div>
   </div>