@rmdes/indiekit-endpoint-conversations 2.1.7 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -0
- package/index.js +1 -0
- package/lib/controllers/conversations.js +84 -39
- package/lib/nodeinfo/resolver.js +153 -0
- package/lib/notifications/activitypub.js +29 -4
- package/lib/notifications/bluesky.js +4 -2
- package/lib/polling/scheduler.js +82 -13
- package/package.json +1 -1
package/README.md
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# @rmdes/indiekit-endpoint-conversations
|
|
2
|
+
|
|
3
|
+
Conversation aggregation endpoint for [Indiekit](https://getindiekit.com/). Polls Mastodon, Bluesky, and ActivityPub notifications, stores interactions in MongoDB, and serves them as a JF2-compatible API — including threaded owner replies.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Multi-platform polling** — Mastodon, Bluesky, and native ActivityPub (via Fedify)
|
|
8
|
+
- **JF2 API** — serves likes, reposts, and replies in webmention-compatible format
|
|
9
|
+
- **Owner reply threading** — enriches API responses with the site owner's replies from the `posts` collection, with threading metadata
|
|
10
|
+
- **Webmention ingestion** — accepts incoming webmentions from Bridgy or external services
|
|
11
|
+
- **Admin dashboard** — connection status, polling stats, platform health
|
|
12
|
+
- **Syndication URL matching** — resolves canonical post URLs from syndicated copies
|
|
13
|
+
|
|
14
|
+
## Installation
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
npm install @rmdes/indiekit-endpoint-conversations
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
```javascript
|
|
21
|
+
// indiekit.config.js
|
|
22
|
+
import ConversationsEndpoint from "@rmdes/indiekit-endpoint-conversations";
|
|
23
|
+
|
|
24
|
+
export default {
|
|
25
|
+
plugins: [
|
|
26
|
+
new ConversationsEndpoint({
|
|
27
|
+
mountPath: "/conversations",
|
|
28
|
+
}),
|
|
29
|
+
],
|
|
30
|
+
};
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Environment Variables
|
|
34
|
+
|
|
35
|
+
| Variable | Required | Description |
|
|
36
|
+
|----------|----------|-------------|
|
|
37
|
+
| `MASTODON_ACCESS_TOKEN` | For Mastodon | Mastodon API access token |
|
|
38
|
+
| `MASTODON_URL` or `MASTODON_INSTANCE` | For Mastodon | Mastodon instance URL |
|
|
39
|
+
| `BLUESKY_IDENTIFIER` or `BLUESKY_HANDLE` | For Bluesky | Bluesky account identifier |
|
|
40
|
+
| `BLUESKY_PASSWORD` | For Bluesky | Bluesky app password |
|
|
41
|
+
| `AUTHOR_NAME` | Optional | Owner display name (falls back to site hostname) |
|
|
42
|
+
| `AUTHOR_AVATAR` | Optional | Owner avatar URL |
|
|
43
|
+
|
|
44
|
+
ActivityPub polling is auto-detected when `@rmdes/indiekit-endpoint-activitypub` is installed.
|
|
45
|
+
|
|
46
|
+
## API
|
|
47
|
+
|
|
48
|
+
### GET /conversations/api/mentions
|
|
49
|
+
|
|
50
|
+
Returns interactions for a target URL in JF2 feed format. Compatible with the webmention.io API shape used by `@chrisburnell/eleventy-cache-webmentions`.
|
|
51
|
+
|
|
52
|
+
**Parameters:**
|
|
53
|
+
|
|
54
|
+
| Param | Type | Description |
|
|
55
|
+
|-------|------|-------------|
|
|
56
|
+
| `target` | string | Target URL to fetch interactions for |
|
|
57
|
+
| `wm-property` | string | Filter by type: `like-of`, `repost-of`, `in-reply-to` |
|
|
58
|
+
| `per-page` | number | Results per page (default: 50, max: 100) |
|
|
59
|
+
| `page` | number | Page number (default: 0) |
|
|
60
|
+
|
|
61
|
+
**Response:**
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"type": "feed",
|
|
66
|
+
"name": "Conversations",
|
|
67
|
+
"children": [
|
|
68
|
+
{
|
|
69
|
+
"type": "entry",
|
|
70
|
+
"wm-id": "conv-mastodon:12345",
|
|
71
|
+
"wm-property": "in-reply-to",
|
|
72
|
+
"wm-target": "https://example.com/posts/hello",
|
|
73
|
+
"author": {
|
|
74
|
+
"type": "card",
|
|
75
|
+
"name": "Jane Doe",
|
|
76
|
+
"url": "https://mastodon.social/@jane",
|
|
77
|
+
"photo": "https://..."
|
|
78
|
+
},
|
|
79
|
+
"url": "https://mastodon.social/@jane/67890",
|
|
80
|
+
"published": "2026-03-11T16:19:52.652Z",
|
|
81
|
+
"platform": "mastodon",
|
|
82
|
+
"content": {
|
|
83
|
+
"html": "<p>Great post!</p>",
|
|
84
|
+
"text": "Great post!"
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"type": "entry",
|
|
89
|
+
"wm-id": "owner-reply-abc123",
|
|
90
|
+
"wm-property": "in-reply-to",
|
|
91
|
+
"wm-target": "https://example.com/posts/hello",
|
|
92
|
+
"author": {
|
|
93
|
+
"type": "card",
|
|
94
|
+
"name": "Site Owner",
|
|
95
|
+
"url": "https://example.com",
|
|
96
|
+
"photo": "https://..."
|
|
97
|
+
},
|
|
98
|
+
"url": "https://example.com/replies/2026/03/11/65e12",
|
|
99
|
+
"published": "2026-03-11T17:00:00.000Z",
|
|
100
|
+
"content": {
|
|
101
|
+
"html": "<p>Thanks!</p>",
|
|
102
|
+
"text": "Thanks!"
|
|
103
|
+
},
|
|
104
|
+
"is_owner": true,
|
|
105
|
+
"parent_url": "https://mastodon.social/@jane/67890"
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### Owner Reply Enrichment
|
|
112
|
+
|
|
113
|
+
When the API returns replies (`wm-property: "in-reply-to"`), it checks the Indiekit `posts` collection for owner posts whose `properties.in-reply-to` matches any reply's `url`. Matching owner posts are appended to the response with two extra fields:
|
|
114
|
+
|
|
115
|
+
| Field | Type | Description |
|
|
116
|
+
|-------|------|-------------|
|
|
117
|
+
| `is_owner` | boolean | Always `true` for owner replies |
|
|
118
|
+
| `parent_url` | string | The URL of the interaction this reply responds to |
|
|
119
|
+
|
|
120
|
+
The frontend uses `parent_url` to thread the owner's reply under the correct parent interaction. See [`indiekit-eleventy-theme`](https://github.com/rmdes/indiekit-eleventy-theme) for the client-side threading implementation.
|
|
121
|
+
|
|
122
|
+
### GET /conversations/api/status
|
|
123
|
+
|
|
124
|
+
Returns connection health and platform status.
|
|
125
|
+
|
|
126
|
+
### POST /conversations/ingest
|
|
127
|
+
|
|
128
|
+
Accepts incoming webmentions. Body: `{ source, target }`.
|
|
129
|
+
|
|
130
|
+
### POST /conversations/poll (authenticated)
|
|
131
|
+
|
|
132
|
+
Triggers an immediate poll of all configured platforms.
|
|
133
|
+
|
|
134
|
+
## Architecture
|
|
135
|
+
|
|
136
|
+
```
|
|
137
|
+
Mastodon API ──┐
|
|
138
|
+
Bluesky API ──┼──> Scheduler ──> conversation_items (MongoDB)
|
|
139
|
+
ActivityPub ──┘ │
|
|
140
|
+
v
|
|
141
|
+
GET /api/mentions ──> JF2 response
|
|
142
|
+
+ owner reply enrichment
|
|
143
|
+
(from posts collection)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Collections
|
|
147
|
+
|
|
148
|
+
| Collection | Purpose |
|
|
149
|
+
|------------|---------|
|
|
150
|
+
| `conversation_items` | Stored interactions (likes, reposts, replies) |
|
|
151
|
+
| `conversation_state` | Polling state (last poll timestamps, cursors) |
|
|
152
|
+
|
|
153
|
+
### Dependencies
|
|
154
|
+
|
|
155
|
+
- **`@rmdes/indiekit-endpoint-activitypub`** — Optional. When installed, the scheduler also polls native ActivityPub interactions from the `ap_interactions` collection.
|
|
156
|
+
- **`indiekit-eleventy-theme`** — The theme's `webmentions.js` consumes the `/api/mentions` endpoint and threads owner replies using the `is_owner` and `parent_url` fields.
|
|
157
|
+
- **`@rmdes/indiekit-endpoint-comments`** — Handles native comment replies (not platform interactions). Owner replies to native comments go through the comments API, not this plugin.
|
|
158
|
+
|
|
159
|
+
## License
|
|
160
|
+
|
|
161
|
+
MIT
|
package/index.js
CHANGED
|
@@ -166,54 +166,99 @@ async function apiMentions(request, response) {
|
|
|
166
166
|
const children = items.map(conversationItemToJf2);
|
|
167
167
|
|
|
168
168
|
// Enrich with owner replies from the posts collection
|
|
169
|
-
// Owner replies are Micropub posts with in-reply-to matching an interaction URL
|
|
169
|
+
// Owner replies are Micropub posts with in-reply-to matching an interaction URL.
|
|
170
|
+
// We collect reply URLs from conversations DB items, but also need to find
|
|
171
|
+
// owner replies to interactions that only exist in webmention.io (e.g., Bluesky
|
|
172
|
+
// replies via Bridgy). Strategy: query for reply URLs from conversations items,
|
|
173
|
+
// plus find owner posts replying to any URL that the frontend might display
|
|
174
|
+
// by checking the canonical post's syndication targets.
|
|
170
175
|
const replyUrls = children
|
|
171
176
|
.filter((c) => c["wm-property"] === "in-reply-to")
|
|
172
177
|
.map((c) => c.url)
|
|
173
178
|
.filter(Boolean);
|
|
174
179
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
180
|
+
const postsCollection = application.collections?.get("posts");
|
|
181
|
+
if (postsCollection) {
|
|
182
|
+
const siteUrl = application.publication?.me || application.url || "";
|
|
183
|
+
const ownerName =
|
|
184
|
+
process.env.AUTHOR_NAME ||
|
|
185
|
+
(siteUrl ? new URL(siteUrl).hostname : "Owner");
|
|
186
|
+
|
|
187
|
+
// Find the canonical post to get its syndication URLs
|
|
188
|
+
// Interactions on syndicated copies (e.g., Bluesky replies to the bsky.app
|
|
189
|
+
// syndicated post) arrive via webmention.io but not conversations DB.
|
|
190
|
+
// Owner replies to those interactions have in-reply-to pointing to external
|
|
191
|
+
// URLs (bsky.app, mastodon, etc.) — we need to find them too.
|
|
192
|
+
let syndicationDomains = [];
|
|
193
|
+
if (target) {
|
|
194
|
+
const targetWithout = target.endsWith("/") ? target.slice(0, -1) : target;
|
|
195
|
+
const canonicalPost = await postsCollection.findOne({
|
|
196
|
+
$or: [
|
|
197
|
+
{ "properties.url": target },
|
|
198
|
+
{ "properties.url": targetWithout },
|
|
199
|
+
],
|
|
200
|
+
});
|
|
201
|
+
if (canonicalPost?.properties?.syndication) {
|
|
202
|
+
const syns = Array.isArray(canonicalPost.properties.syndication)
|
|
203
|
+
? canonicalPost.properties.syndication
|
|
204
|
+
: [canonicalPost.properties.syndication];
|
|
205
|
+
for (const syn of syns) {
|
|
206
|
+
try {
|
|
207
|
+
const domain = new URL(syn).hostname;
|
|
208
|
+
if (domain && !domain.includes(new URL(siteUrl).hostname)) {
|
|
209
|
+
syndicationDomains.push(domain);
|
|
210
|
+
}
|
|
211
|
+
} catch { /* skip invalid URLs */ }
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Build query: replies to known conversation URLs OR replies to URLs
|
|
217
|
+
// on syndication domains (for webmention.io items not in our DB)
|
|
218
|
+
const orClauses = [];
|
|
219
|
+
if (replyUrls.length > 0) {
|
|
220
|
+
orClauses.push({ "properties.in-reply-to": { $in: replyUrls } });
|
|
221
|
+
}
|
|
222
|
+
for (const domain of syndicationDomains) {
|
|
223
|
+
orClauses.push({
|
|
224
|
+
"properties.in-reply-to": { $regex: domain.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") },
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
let ownerPosts = [];
|
|
229
|
+
if (orClauses.length > 0) {
|
|
230
|
+
ownerPosts = await postsCollection
|
|
231
|
+
.find({ $or: orClauses })
|
|
187
232
|
.sort({ "properties.published": -1 })
|
|
188
233
|
.limit(50)
|
|
189
234
|
.toArray();
|
|
235
|
+
}
|
|
190
236
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
}
|
|
237
|
+
for (const post of ownerPosts) {
|
|
238
|
+
const inReplyTo = post.properties?.["in-reply-to"];
|
|
239
|
+
if (!inReplyTo || typeof inReplyTo !== "string") continue;
|
|
240
|
+
|
|
241
|
+
children.push({
|
|
242
|
+
type: "entry",
|
|
243
|
+
"wm-id": `owner-reply-${post._id}`,
|
|
244
|
+
"wm-property": "in-reply-to",
|
|
245
|
+
"wm-target": target || "",
|
|
246
|
+
"wm-received": post.properties?.published || "",
|
|
247
|
+
author: {
|
|
248
|
+
type: "card",
|
|
249
|
+
name: ownerName,
|
|
250
|
+
url: siteUrl,
|
|
251
|
+
photo: process.env.AUTHOR_AVATAR || "",
|
|
252
|
+
},
|
|
253
|
+
url: post.properties?.url || "",
|
|
254
|
+
published: post.properties?.published || "",
|
|
255
|
+
content: {
|
|
256
|
+
text: post.properties?.content?.text || "",
|
|
257
|
+
html: post.properties?.content?.html || "",
|
|
258
|
+
},
|
|
259
|
+
is_owner: true,
|
|
260
|
+
parent_url: inReplyTo,
|
|
261
|
+
});
|
|
217
262
|
}
|
|
218
263
|
}
|
|
219
264
|
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NodeInfo-based server software resolver
|
|
3
|
+
* Fetches /.well-known/nodeinfo from a domain, follows the link,
|
|
4
|
+
* and returns the software name (e.g., "mastodon", "pleroma", "misskey").
|
|
5
|
+
* Results are cached in-memory and optionally persisted to MongoDB.
|
|
6
|
+
* @module nodeinfo/resolver
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const NODEINFO_TIMEOUT_MS = 5000;
|
|
10
|
+
const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
11
|
+
|
|
12
|
+
// In-memory cache: domain -> { software, resolvedAt }
|
|
13
|
+
const memoryCache = new Map();
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Resolve the server software for a given actor URL via NodeInfo.
|
|
17
|
+
* Returns a lowercase software name like "mastodon", "pleroma", "misskey",
|
|
18
|
+
* "gotosocial", "fedify", etc. Falls back to "activitypub" if NodeInfo
|
|
19
|
+
* is unavailable or unrecognizable.
|
|
20
|
+
*
|
|
21
|
+
* @param {string} actorUrl - The actor's URL (e.g., "https://mastodon.social/@user")
|
|
22
|
+
* @param {object} [collection] - Optional MongoDB collection for persistent cache
|
|
23
|
+
* @returns {Promise<string>} Lowercase software name or "activitypub"
|
|
24
|
+
*/
|
|
25
|
+
export async function resolveServerSoftware(actorUrl, collection) {
|
|
26
|
+
const domain = extractDomain(actorUrl);
|
|
27
|
+
if (!domain) return "activitypub";
|
|
28
|
+
|
|
29
|
+
// Check in-memory cache first
|
|
30
|
+
const cached = memoryCache.get(domain);
|
|
31
|
+
if (cached && Date.now() - cached.resolvedAt < CACHE_TTL_MS) {
|
|
32
|
+
return cached.software;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// Check MongoDB cache
|
|
36
|
+
if (collection) {
|
|
37
|
+
try {
|
|
38
|
+
const doc = await collection.findOne({ _id: domain });
|
|
39
|
+
if (doc && Date.now() - new Date(doc.resolvedAt).getTime() < CACHE_TTL_MS) {
|
|
40
|
+
memoryCache.set(domain, {
|
|
41
|
+
software: doc.software,
|
|
42
|
+
resolvedAt: new Date(doc.resolvedAt).getTime(),
|
|
43
|
+
});
|
|
44
|
+
return doc.software;
|
|
45
|
+
}
|
|
46
|
+
} catch { /* proceed to live fetch */ }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Live fetch via NodeInfo protocol
|
|
50
|
+
const software = await fetchNodeInfo(domain);
|
|
51
|
+
|
|
52
|
+
// Cache result (even "activitypub" fallback — avoids repeated failed lookups)
|
|
53
|
+
const entry = { software, resolvedAt: Date.now() };
|
|
54
|
+
memoryCache.set(domain, entry);
|
|
55
|
+
|
|
56
|
+
if (collection) {
|
|
57
|
+
try {
|
|
58
|
+
await collection.findOneAndUpdate(
|
|
59
|
+
{ _id: domain },
|
|
60
|
+
{ $set: { software, resolvedAt: new Date().toISOString() } },
|
|
61
|
+
{ upsert: true },
|
|
62
|
+
);
|
|
63
|
+
} catch { /* non-critical */ }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return software;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Batch-resolve software for multiple actor URLs.
|
|
71
|
+
* Deduplicates by domain so each domain is only queried once.
|
|
72
|
+
*
|
|
73
|
+
* @param {string[]} actorUrls - Array of actor URLs
|
|
74
|
+
* @param {object} [collection] - Optional MongoDB collection for persistent cache
|
|
75
|
+
* @returns {Promise<Map<string, string>>} Map of domain -> software name
|
|
76
|
+
*/
|
|
77
|
+
export async function batchResolve(actorUrls, collection) {
|
|
78
|
+
const domains = new Set();
|
|
79
|
+
for (const url of actorUrls) {
|
|
80
|
+
const domain = extractDomain(url);
|
|
81
|
+
if (domain) domains.add(domain);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const results = new Map();
|
|
85
|
+
for (const domain of domains) {
|
|
86
|
+
results.set(
|
|
87
|
+
domain,
|
|
88
|
+
await resolveServerSoftware(`https://${domain}/`, collection),
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
return results;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Extract domain from a URL
|
|
96
|
+
* @param {string} url
|
|
97
|
+
* @returns {string|null}
|
|
98
|
+
*/
|
|
99
|
+
function extractDomain(url) {
|
|
100
|
+
try {
|
|
101
|
+
return new URL(url).hostname;
|
|
102
|
+
} catch {
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Fetch NodeInfo for a domain and return the software name
|
|
109
|
+
* @param {string} domain
|
|
110
|
+
* @returns {Promise<string>} Software name or "activitypub"
|
|
111
|
+
*/
|
|
112
|
+
async function fetchNodeInfo(domain) {
|
|
113
|
+
try {
|
|
114
|
+
// Step 1: Fetch /.well-known/nodeinfo
|
|
115
|
+
const wellKnownUrl = `https://${domain}/.well-known/nodeinfo`;
|
|
116
|
+
const wellKnownResp = await fetch(wellKnownUrl, {
|
|
117
|
+
headers: { Accept: "application/json" },
|
|
118
|
+
signal: AbortSignal.timeout(NODEINFO_TIMEOUT_MS),
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
if (!wellKnownResp.ok) return "activitypub";
|
|
122
|
+
|
|
123
|
+
const wellKnown = await wellKnownResp.json();
|
|
124
|
+
const links = wellKnown.links;
|
|
125
|
+
if (!Array.isArray(links) || links.length === 0) return "activitypub";
|
|
126
|
+
|
|
127
|
+
// Prefer NodeInfo 2.x, fall back to any available link
|
|
128
|
+
const link =
|
|
129
|
+
links.find((l) => l.rel?.includes("nodeinfo/2.")) ||
|
|
130
|
+
links[0];
|
|
131
|
+
|
|
132
|
+
if (!link?.href) return "activitypub";
|
|
133
|
+
|
|
134
|
+
// Step 2: Fetch the actual NodeInfo document
|
|
135
|
+
const nodeInfoResp = await fetch(link.href, {
|
|
136
|
+
headers: { Accept: "application/json" },
|
|
137
|
+
signal: AbortSignal.timeout(NODEINFO_TIMEOUT_MS),
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
if (!nodeInfoResp.ok) return "activitypub";
|
|
141
|
+
|
|
142
|
+
const nodeInfo = await nodeInfoResp.json();
|
|
143
|
+
const softwareName = nodeInfo.software?.name;
|
|
144
|
+
|
|
145
|
+
if (typeof softwareName === "string" && softwareName.trim()) {
|
|
146
|
+
return softwareName.trim().toLowerCase();
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
return "activitypub";
|
|
150
|
+
} catch {
|
|
151
|
+
return "activitypub";
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -20,11 +20,12 @@ const typeMap = {
|
|
|
20
20
|
* @param {object} options
|
|
21
21
|
* @param {object} options.ap_activities - MongoDB collection
|
|
22
22
|
* @param {object} options.ap_followers - MongoDB collection (for avatar lookup)
|
|
23
|
+
* @param {object} [options.nodeinfoCache] - MongoDB collection for NodeInfo cache
|
|
23
24
|
* @param {string} [options.since] - ISO 8601 timestamp cursor (process activities after this)
|
|
24
25
|
* @returns {Promise<{items: Array, cursor: string|null}>}
|
|
25
26
|
*/
|
|
26
27
|
export async function fetchActivityPubInteractions(options) {
|
|
27
|
-
const { ap_activities, ap_followers, since } = options;
|
|
28
|
+
const { ap_activities, ap_followers, nodeinfoCache, since } = options;
|
|
28
29
|
|
|
29
30
|
const query = {
|
|
30
31
|
direction: "inbound",
|
|
@@ -45,6 +46,11 @@ export async function fetchActivityPubInteractions(options) {
|
|
|
45
46
|
return { items: [], cursor: null };
|
|
46
47
|
}
|
|
47
48
|
|
|
49
|
+
// Resolve server software for all actor domains in this batch
|
|
50
|
+
const { batchResolve } = await import("../nodeinfo/resolver.js");
|
|
51
|
+
const actorUrls = activities.map((a) => a.actorUrl).filter(Boolean);
|
|
52
|
+
const domainSoftware = await batchResolve(actorUrls, nodeinfoCache);
|
|
53
|
+
|
|
48
54
|
const items = [];
|
|
49
55
|
|
|
50
56
|
for (const activity of activities) {
|
|
@@ -53,7 +59,12 @@ export async function fetchActivityPubInteractions(options) {
|
|
|
53
59
|
const avatar =
|
|
54
60
|
activity.actorAvatar ||
|
|
55
61
|
(await lookupAvatar(ap_followers, activity.actorUrl));
|
|
56
|
-
|
|
62
|
+
|
|
63
|
+
// Resolve platform from NodeInfo (e.g., "mastodon", "pleroma", "misskey")
|
|
64
|
+
const domain = extractDomain(activity.actorUrl);
|
|
65
|
+
const platform = domain ? (domainSoftware.get(domain) || "activitypub") : "activitypub";
|
|
66
|
+
|
|
67
|
+
items.push(normalizeActivity(activity, avatar, platform));
|
|
57
68
|
}
|
|
58
69
|
|
|
59
70
|
// Cursor is the receivedAt of the last activity processed
|
|
@@ -62,13 +73,27 @@ export async function fetchActivityPubInteractions(options) {
|
|
|
62
73
|
return { items, cursor };
|
|
63
74
|
}
|
|
64
75
|
|
|
76
|
+
/**
|
|
77
|
+
* Extract hostname from a URL
|
|
78
|
+
* @param {string} url
|
|
79
|
+
* @returns {string|null}
|
|
80
|
+
*/
|
|
81
|
+
function extractDomain(url) {
|
|
82
|
+
try {
|
|
83
|
+
return new URL(url).hostname;
|
|
84
|
+
} catch {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
65
89
|
/**
|
|
66
90
|
* Normalize an ap_activities document into conversations internal format
|
|
67
91
|
* @param {object} activity - Document from ap_activities
|
|
68
92
|
* @param {string} avatar - Avatar URL from ap_followers lookup
|
|
93
|
+
* @param {string} platform - Resolved server software (e.g., "mastodon", "pleroma")
|
|
69
94
|
* @returns {object} Normalized interaction
|
|
70
95
|
*/
|
|
71
|
-
function normalizeActivity(activity, avatar) {
|
|
96
|
+
function normalizeActivity(activity, avatar, platform) {
|
|
72
97
|
const type = typeMap[activity.type] || "mention";
|
|
73
98
|
const isReply = activity.type === "Reply";
|
|
74
99
|
|
|
@@ -81,7 +106,7 @@ function normalizeActivity(activity, avatar) {
|
|
|
81
106
|
const url = isReply ? activity.objectUrl : activity.actorUrl;
|
|
82
107
|
|
|
83
108
|
return {
|
|
84
|
-
platform
|
|
109
|
+
platform,
|
|
85
110
|
platform_id: `activitypub:${activity.type}:${activity.actorUrl}:${activity.objectUrl}`,
|
|
86
111
|
type,
|
|
87
112
|
author: {
|
|
@@ -27,9 +27,11 @@ export async function fetchBlueskyNotifications(options) {
|
|
|
27
27
|
// Get or refresh session
|
|
28
28
|
const session = await getSession(serviceUrl, identifier, password);
|
|
29
29
|
|
|
30
|
-
// Fetch notifications
|
|
30
|
+
// Fetch the most recent notifications (no cursor)
|
|
31
|
+
// Bluesky's listNotifications returns newest first; the cursor pages backward
|
|
32
|
+
// into history. For polling, we always want the latest batch and rely on
|
|
33
|
+
// upsert deduplication (platform_id) to skip already-stored items.
|
|
31
34
|
const params = new URLSearchParams({ limit: "50" });
|
|
32
|
-
if (options.cursor) params.set("cursor", options.cursor);
|
|
33
35
|
|
|
34
36
|
let notifResponse = await fetch(
|
|
35
37
|
`${serviceUrl}/xrpc/app.bsky.notification.listNotifications?${params.toString()}`,
|
package/lib/polling/scheduler.js
CHANGED
|
@@ -97,6 +97,9 @@ export async function runPollCycle(indiekit, options) {
|
|
|
97
97
|
|
|
98
98
|
// Backfill missing avatars from ap_notifications (one-time sweep per cycle)
|
|
99
99
|
await backfillMissingAvatars(indiekit, stateCollection);
|
|
100
|
+
|
|
101
|
+
// Backfill platform names for items stored as "activitypub" (one-time)
|
|
102
|
+
await backfillPlatformNames(indiekit, stateCollection);
|
|
100
103
|
}
|
|
101
104
|
|
|
102
105
|
/**
|
|
@@ -233,6 +236,72 @@ async function backfillMissingAvatars(indiekit, stateCollection) {
|
|
|
233
236
|
}
|
|
234
237
|
}
|
|
235
238
|
|
|
239
|
+
/**
|
|
240
|
+
* Backfill platform names for existing items stored with source "activitypub".
|
|
241
|
+
* Uses NodeInfo to resolve the actual server software (e.g., "mastodon", "pleroma").
|
|
242
|
+
* One-time operation — marks complete after first successful run.
|
|
243
|
+
*/
|
|
244
|
+
async function backfillPlatformNames(indiekit, stateCollection) {
|
|
245
|
+
try {
|
|
246
|
+
const itemsCollection = indiekit.collections.get("conversation_items");
|
|
247
|
+
if (!itemsCollection) return;
|
|
248
|
+
|
|
249
|
+
// Check if backfill already completed
|
|
250
|
+
const state = await stateCollection.findOne({ _id: "poll_cursors" });
|
|
251
|
+
if (state?.platform_backfill_complete) return;
|
|
252
|
+
|
|
253
|
+
// Find unique author URLs for items with source "activitypub"
|
|
254
|
+
const actorUrls = await itemsCollection.distinct("author.url", {
|
|
255
|
+
source: "activitypub",
|
|
256
|
+
});
|
|
257
|
+
|
|
258
|
+
if (actorUrls.length === 0) {
|
|
259
|
+
await stateCollection.findOneAndUpdate(
|
|
260
|
+
{ _id: "poll_cursors" },
|
|
261
|
+
{ $set: { platform_backfill_complete: true } },
|
|
262
|
+
{ upsert: true },
|
|
263
|
+
);
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const { batchResolve } = await import("../nodeinfo/resolver.js");
|
|
268
|
+
const nodeinfoCache = indiekit.collections?.get("nodeinfo_cache") || null;
|
|
269
|
+
const domainSoftware = await batchResolve(
|
|
270
|
+
actorUrls.filter(Boolean),
|
|
271
|
+
nodeinfoCache,
|
|
272
|
+
);
|
|
273
|
+
|
|
274
|
+
let updated = 0;
|
|
275
|
+
|
|
276
|
+
for (const [domain, software] of domainSoftware) {
|
|
277
|
+
if (software === "activitypub") continue; // No change needed
|
|
278
|
+
|
|
279
|
+
const result = await itemsCollection.updateMany(
|
|
280
|
+
{
|
|
281
|
+
source: "activitypub",
|
|
282
|
+
"author.url": { $regex: `^https?://${domain.replace(/\./g, "\\.")}/` },
|
|
283
|
+
},
|
|
284
|
+
{ $set: { source: software } },
|
|
285
|
+
);
|
|
286
|
+
if (result.modifiedCount > 0) updated += result.modifiedCount;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
if (updated > 0) {
|
|
290
|
+
console.info(
|
|
291
|
+
`[Conversations] Platform backfill: updated ${updated} items from "activitypub" to resolved software names`,
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
await stateCollection.findOneAndUpdate(
|
|
296
|
+
{ _id: "poll_cursors" },
|
|
297
|
+
{ $set: { platform_backfill_complete: true } },
|
|
298
|
+
{ upsert: true },
|
|
299
|
+
);
|
|
300
|
+
} catch (error) {
|
|
301
|
+
console.warn("[Conversations] Platform backfill error:", error.message);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
236
305
|
/**
|
|
237
306
|
* Poll Mastodon notifications and store matching interactions
|
|
238
307
|
*/
|
|
@@ -356,7 +425,6 @@ async function pollBluesky(indiekit, stateCollection, state, credentials) {
|
|
|
356
425
|
const result = await fetchBlueskyNotifications({
|
|
357
426
|
identifier: credentials.identifier,
|
|
358
427
|
password: credentials.password,
|
|
359
|
-
cursor: state.bluesky_cursor,
|
|
360
428
|
});
|
|
361
429
|
|
|
362
430
|
let stored = 0;
|
|
@@ -388,22 +456,19 @@ async function pollBluesky(indiekit, stateCollection, state, credentials) {
|
|
|
388
456
|
}
|
|
389
457
|
}
|
|
390
458
|
|
|
391
|
-
// Update
|
|
392
|
-
const updateFields = {
|
|
393
|
-
bluesky_last_poll: new Date().toISOString(),
|
|
394
|
-
bluesky_last_error: null,
|
|
395
|
-
};
|
|
396
|
-
if (result.cursor) {
|
|
397
|
-
updateFields.bluesky_cursor = result.cursor;
|
|
398
|
-
}
|
|
399
|
-
|
|
459
|
+
// Update poll timestamp
|
|
400
460
|
await stateCollection.findOneAndUpdate(
|
|
401
461
|
{ _id: "poll_cursors" },
|
|
402
|
-
{
|
|
462
|
+
{
|
|
463
|
+
$set: {
|
|
464
|
+
bluesky_last_poll: new Date().toISOString(),
|
|
465
|
+
bluesky_last_error: null,
|
|
466
|
+
},
|
|
467
|
+
},
|
|
403
468
|
{ upsert: true },
|
|
404
469
|
);
|
|
405
470
|
|
|
406
|
-
if (stored > 0) {
|
|
471
|
+
if (stored > 0 || result.items.length > 0) {
|
|
407
472
|
console.info(
|
|
408
473
|
`[Conversations] Bluesky: stored ${stored}/${result.items.length} interactions`,
|
|
409
474
|
);
|
|
@@ -470,9 +535,13 @@ async function pollActivityPub(indiekit, stateCollection, state) {
|
|
|
470
535
|
""
|
|
471
536
|
).replace(/\/$/, "");
|
|
472
537
|
|
|
538
|
+
// NodeInfo cache collection for resolving server software per domain
|
|
539
|
+
const nodeinfoCache = indiekit.collections?.get("nodeinfo_cache") || null;
|
|
540
|
+
|
|
473
541
|
const result = await fetchActivityPubInteractions({
|
|
474
542
|
ap_activities,
|
|
475
543
|
ap_followers,
|
|
544
|
+
nodeinfoCache,
|
|
476
545
|
since: state.activitypub_last_received_at || null,
|
|
477
546
|
});
|
|
478
547
|
|
|
@@ -491,7 +560,7 @@ async function pollActivityPub(indiekit, stateCollection, state) {
|
|
|
491
560
|
|
|
492
561
|
await upsertConversationItem(indiekit, {
|
|
493
562
|
canonical_url: interaction.canonical_url,
|
|
494
|
-
source:
|
|
563
|
+
source: interaction.platform,
|
|
495
564
|
type: interaction.type,
|
|
496
565
|
author: interaction.author,
|
|
497
566
|
content: interaction.content,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rmdes/indiekit-endpoint-conversations",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.3.0",
|
|
4
4
|
"description": "Conversation aggregation endpoint for Indiekit. Backend enrichment service that polls Mastodon/Bluesky notifications and serves JF2-compatible data for the interactions page.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"indiekit",
|