@rmdes/indiekit-endpoint-webmention-io 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +16 -9
- package/lib/blocks.js +36 -0
- package/lib/controllers/api.js +6 -4
- package/lib/controllers/blocklist.js +16 -8
- package/lib/controllers/dashboard.js +67 -28
- package/lib/controllers/sync-controller.js +28 -8
- package/lib/hcard.js +4 -4
- package/lib/storage/blocklist.js +10 -3
- package/lib/storage/webmentions.js +21 -7
- package/lib/sync.js +33 -28
- package/package.json +12 -7
package/index.js
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
import express from "express";
|
|
2
|
-
import { fileURLToPath } from "node:url";
|
|
3
1
|
import path from "node:path";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
4
3
|
|
|
5
|
-
import {
|
|
4
|
+
import { waitForReady } from "@rmdes/indiekit-startup-gate";
|
|
5
|
+
import express from "express";
|
|
6
|
+
|
|
7
|
+
import { WEBMENTION_BLOCKS } from "./lib/blocks.js";
|
|
8
|
+
import { apiController } from "./lib/controllers/api.js";
|
|
6
9
|
import { blocklistController } from "./lib/controllers/blocklist.js";
|
|
10
|
+
import { dashboardController } from "./lib/controllers/dashboard.js";
|
|
7
11
|
import { syncController } from "./lib/controllers/sync-controller.js";
|
|
8
|
-
import { apiController } from "./lib/controllers/api.js";
|
|
9
12
|
import { startSync, stopSync } from "./lib/sync.js";
|
|
10
|
-
import { waitForReady } from "@rmdes/indiekit-startup-gate";
|
|
11
13
|
|
|
12
14
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
15
|
|
|
@@ -21,6 +23,8 @@ const defaults = {
|
|
|
21
23
|
};
|
|
22
24
|
|
|
23
25
|
export default class WebmentionEndpoint {
|
|
26
|
+
_stopGate;
|
|
27
|
+
|
|
24
28
|
name = "Webmention moderation endpoint";
|
|
25
29
|
|
|
26
30
|
constructor(options = {}) {
|
|
@@ -44,6 +48,10 @@ export default class WebmentionEndpoint {
|
|
|
44
48
|
};
|
|
45
49
|
}
|
|
46
50
|
|
|
51
|
+
get blocks() {
|
|
52
|
+
return WEBMENTION_BLOCKS;
|
|
53
|
+
}
|
|
54
|
+
|
|
47
55
|
/**
|
|
48
56
|
* Protected routes (require authentication)
|
|
49
57
|
* Admin dashboard, moderation, sync controls
|
|
@@ -104,10 +112,9 @@ export default class WebmentionEndpoint {
|
|
|
104
112
|
|
|
105
113
|
// Start background sync if database is available
|
|
106
114
|
if (Indiekit.config.application.mongodbUrl) {
|
|
107
|
-
this._stopGate = waitForReady(
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
);
|
|
115
|
+
this._stopGate = waitForReady(() => startSync(Indiekit, this.options), {
|
|
116
|
+
label: "Webmention.io",
|
|
117
|
+
});
|
|
111
118
|
}
|
|
112
119
|
}
|
|
113
120
|
|
package/lib/blocks.js
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Webmention-io v2 block declaration (Phase 7b — plugin block ownership).
|
|
3
|
+
*
|
|
4
|
+
* The `webmentions` sidebar widget was a site-config BUILTIN_BLOCKS seed
|
|
5
|
+
* (requiresPlugin null, gated only by the theme's legacy widgetPluginRequirements
|
|
6
|
+
* render-map). Declaring it here makes site-config's scanPlugins stamp
|
|
7
|
+
* `sourcePlugin` → `requiresPlugin` ("Webmention moderation endpoint"), so the
|
|
8
|
+
* block is properly plugin-gated (theme ENDPOINT_SLUGS maps it to the
|
|
9
|
+
* `webmention-io` loadout slug). scanPlugins precedence is `built-in < plugin
|
|
10
|
+
* blocks`, so this entry OVERWRITES the builtin seed on sites where the plugin is
|
|
11
|
+
* loaded; the seed itself is removed from site-config in Phase 7d alongside the
|
|
12
|
+
* legacy-map bridge.
|
|
13
|
+
*
|
|
14
|
+
* Descriptor is byte-faithful to the BUILTIN_BLOCKS entry. Bespoke template: the
|
|
15
|
+
* theme owns `components/widgets/webmentions.njk` (no generic `render.renderer`);
|
|
16
|
+
* `data.source:"api"` documents the runtime fetch.
|
|
17
|
+
* @module lib/blocks
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
@type {Array<object>}
|
|
22
|
+
*/
|
|
23
|
+
export const WEBMENTION_BLOCKS = [
|
|
24
|
+
{
|
|
25
|
+
id: "webmentions",
|
|
26
|
+
version: 1,
|
|
27
|
+
label: "Webmentions",
|
|
28
|
+
description: "Recent inbound/outbound webmentions",
|
|
29
|
+
icon: "message-circle",
|
|
30
|
+
category: "social",
|
|
31
|
+
placement: { regions: ["sidebar"], surfaces: ["homepage", "postType"] },
|
|
32
|
+
multiple: false,
|
|
33
|
+
data: { source: "api" },
|
|
34
|
+
schema: { type: "object", additionalProperties: false, properties: {} },
|
|
35
|
+
},
|
|
36
|
+
];
|
package/lib/controllers/api.js
CHANGED
|
@@ -8,21 +8,23 @@ import { getWebmentions, documentToJf2 } from "../storage/webmentions.js";
|
|
|
8
8
|
export const apiController = {
|
|
9
9
|
/**
|
|
10
10
|
* GET /api/mentions - Public JF2 webmentions API
|
|
11
|
+
* @param request
|
|
12
|
+
* @param response
|
|
11
13
|
*/
|
|
12
14
|
async getMentions(request, response) {
|
|
13
15
|
try {
|
|
14
16
|
const { application } = request.app.locals;
|
|
15
|
-
const
|
|
17
|
+
const database = application.getWebmentionDb();
|
|
16
18
|
|
|
17
|
-
if (!
|
|
19
|
+
if (!database) {
|
|
18
20
|
return response.status(503).json({ error: "Database unavailable" });
|
|
19
21
|
}
|
|
20
22
|
|
|
21
|
-
const collection =
|
|
23
|
+
const collection = database.collection("webmentions");
|
|
22
24
|
|
|
23
25
|
const target = request.query.target || null;
|
|
24
26
|
const wmProperty = request.query["wm-property"] || null;
|
|
25
|
-
const perPage = Math.min(Number(request.query["per-page"]) || 50,
|
|
27
|
+
const perPage = Math.min(Number(request.query["per-page"]) || 50, 10_000);
|
|
26
28
|
const page = Number(request.query.page) || 0;
|
|
27
29
|
|
|
28
30
|
const { items } = await getWebmentions(collection, {
|
|
@@ -9,16 +9,18 @@ import { ensureISOString } from "../utils.js";
|
|
|
9
9
|
export const blocklistController = {
|
|
10
10
|
/**
|
|
11
11
|
* GET /blocklist - Blocklist management page
|
|
12
|
+
* @param request
|
|
13
|
+
* @param response
|
|
12
14
|
*/
|
|
13
15
|
async list(request, response) {
|
|
14
16
|
const { application } = request.app.locals;
|
|
15
17
|
|
|
16
18
|
try {
|
|
17
|
-
const
|
|
19
|
+
const database = application.getWebmentionDb();
|
|
18
20
|
let entries = [];
|
|
19
21
|
|
|
20
|
-
if (
|
|
21
|
-
const collection =
|
|
22
|
+
if (database) {
|
|
23
|
+
const collection = database.collection("webmentionBlocklist");
|
|
22
24
|
const raw = await getBlocklist(collection);
|
|
23
25
|
entries = raw.map((entry) => ({
|
|
24
26
|
...entry,
|
|
@@ -43,15 +45,17 @@ export const blocklistController = {
|
|
|
43
45
|
|
|
44
46
|
/**
|
|
45
47
|
* POST /blocklist/:domain/delete - Unblock a domain
|
|
48
|
+
* @param request
|
|
49
|
+
* @param response
|
|
46
50
|
*/
|
|
47
51
|
async unblock(request, response) {
|
|
48
52
|
const { application } = request.app.locals;
|
|
49
53
|
|
|
50
54
|
try {
|
|
51
55
|
const domain = decodeURIComponent(request.params.domain);
|
|
52
|
-
const
|
|
53
|
-
const blockCollection =
|
|
54
|
-
const wmCollection =
|
|
56
|
+
const database = application.getWebmentionDb();
|
|
57
|
+
const blockCollection = database.collection("webmentionBlocklist");
|
|
58
|
+
const wmCollection = database.collection("webmentions");
|
|
55
59
|
|
|
56
60
|
await unblockDomain(blockCollection, domain);
|
|
57
61
|
|
|
@@ -59,11 +63,15 @@ export const blocklistController = {
|
|
|
59
63
|
const unhidden = await unhideByDomain(wmCollection, domain);
|
|
60
64
|
|
|
61
65
|
response.redirect(
|
|
62
|
-
application.webmentionEndpoint +
|
|
66
|
+
application.webmentionEndpoint +
|
|
67
|
+
"/blocklist?unblocked=1&unhidden=" +
|
|
68
|
+
unhidden,
|
|
63
69
|
);
|
|
64
70
|
} catch (error) {
|
|
65
71
|
console.error("[Webmentions] Unblock error:", error);
|
|
66
|
-
response.redirect(
|
|
72
|
+
response.redirect(
|
|
73
|
+
application.webmentionEndpoint + "/blocklist?error=unblock-failed",
|
|
74
|
+
);
|
|
67
75
|
}
|
|
68
76
|
},
|
|
69
77
|
};
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* Admin UI for webmention moderation
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
+
import { blockDomain } from "../storage/blocklist.js";
|
|
6
7
|
import {
|
|
7
8
|
getWebmentions,
|
|
8
9
|
getWebmentionCounts,
|
|
@@ -11,20 +12,26 @@ import {
|
|
|
11
12
|
hideByDomain,
|
|
12
13
|
deleteByDomain,
|
|
13
14
|
} from "../storage/webmentions.js";
|
|
14
|
-
import { blockDomain } from "../storage/blocklist.js";
|
|
15
15
|
import { getSyncState } from "../sync.js";
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
getMentionType,
|
|
18
|
+
getMentionTitle,
|
|
19
|
+
getAuthorName,
|
|
20
|
+
ensureISOString,
|
|
21
|
+
} from "../utils.js";
|
|
17
22
|
|
|
18
23
|
export const dashboardController = {
|
|
19
24
|
/**
|
|
20
25
|
* GET / - Webmentions dashboard
|
|
26
|
+
* @param request
|
|
27
|
+
* @param response
|
|
21
28
|
*/
|
|
22
29
|
async list(request, response) {
|
|
23
30
|
const { application } = request.app.locals;
|
|
24
31
|
|
|
25
32
|
try {
|
|
26
|
-
const
|
|
27
|
-
if (!
|
|
33
|
+
const database = application.getWebmentionDb();
|
|
34
|
+
if (!database) {
|
|
28
35
|
return response.render("webmentions", {
|
|
29
36
|
title: response.locals.__("webmention-io.title"),
|
|
30
37
|
webmentions: [],
|
|
@@ -37,7 +44,7 @@ export const dashboardController = {
|
|
|
37
44
|
});
|
|
38
45
|
}
|
|
39
46
|
|
|
40
|
-
const collection =
|
|
47
|
+
const collection = database.collection("webmentions");
|
|
41
48
|
|
|
42
49
|
const page = Number(request.query.page) || 0;
|
|
43
50
|
const limit = Number(request.query.limit) || 20;
|
|
@@ -80,16 +87,22 @@ export const dashboardController = {
|
|
|
80
87
|
"wm-target": item.wmTarget,
|
|
81
88
|
icon: getMentionType(item.wmProperty),
|
|
82
89
|
locale: application.locale,
|
|
83
|
-
title: getMentionTitle({
|
|
90
|
+
title: getMentionTitle({
|
|
91
|
+
name: item.name,
|
|
92
|
+
"wm-property": item.wmProperty,
|
|
93
|
+
}),
|
|
84
94
|
description: html ? { html } : undefined,
|
|
85
|
-
published:
|
|
95
|
+
published:
|
|
96
|
+
ensureISOString(item.published) || ensureISOString(item.wmReceived),
|
|
86
97
|
url: item.sourceUrl,
|
|
87
98
|
user: {
|
|
88
99
|
avatar: item.authorPhoto ? { src: item.authorPhoto } : undefined,
|
|
89
|
-
name:
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
100
|
+
name:
|
|
101
|
+
item.authorName ||
|
|
102
|
+
getAuthorName({
|
|
103
|
+
author: { name: item.authorName, url: item.authorUrl },
|
|
104
|
+
url: item.sourceUrl,
|
|
105
|
+
}),
|
|
93
106
|
url: item.authorUrl,
|
|
94
107
|
},
|
|
95
108
|
// Moderation metadata
|
|
@@ -104,7 +117,9 @@ export const dashboardController = {
|
|
|
104
117
|
next: { href: `?page=${page + 1}&filter=${filter}&type=${typeFilter}` },
|
|
105
118
|
};
|
|
106
119
|
if (page > 0) {
|
|
107
|
-
cursor.previous = {
|
|
120
|
+
cursor.previous = {
|
|
121
|
+
href: `?page=${page - 1}&filter=${filter}&type=${typeFilter}`,
|
|
122
|
+
};
|
|
108
123
|
}
|
|
109
124
|
|
|
110
125
|
response.render("webmentions", {
|
|
@@ -129,14 +144,16 @@ export const dashboardController = {
|
|
|
129
144
|
|
|
130
145
|
/**
|
|
131
146
|
* POST /:wmId/hide - Hide a webmention
|
|
147
|
+
* @param request
|
|
148
|
+
* @param response
|
|
132
149
|
*/
|
|
133
150
|
async hide(request, response) {
|
|
134
151
|
const { application } = request.app.locals;
|
|
135
152
|
|
|
136
153
|
try {
|
|
137
154
|
const wmId = Number.parseInt(request.params.wmId, 10);
|
|
138
|
-
const
|
|
139
|
-
const collection =
|
|
155
|
+
const database = application.getWebmentionDb();
|
|
156
|
+
const collection = database.collection("webmentions");
|
|
140
157
|
|
|
141
158
|
await hideWebmention(collection, wmId, "manual");
|
|
142
159
|
|
|
@@ -149,26 +166,32 @@ export const dashboardController = {
|
|
|
149
166
|
|
|
150
167
|
/**
|
|
151
168
|
* POST /:wmId/unhide - Restore a webmention
|
|
169
|
+
* @param request
|
|
170
|
+
* @param response
|
|
152
171
|
*/
|
|
153
172
|
async unhide(request, response) {
|
|
154
173
|
const { application } = request.app.locals;
|
|
155
174
|
|
|
156
175
|
try {
|
|
157
176
|
const wmId = Number.parseInt(request.params.wmId, 10);
|
|
158
|
-
const
|
|
159
|
-
const collection =
|
|
177
|
+
const database = application.getWebmentionDb();
|
|
178
|
+
const collection = database.collection("webmentions");
|
|
160
179
|
|
|
161
180
|
await unhideWebmention(collection, wmId);
|
|
162
181
|
|
|
163
182
|
response.redirect(application.webmentionEndpoint + "?unhidden=1");
|
|
164
183
|
} catch (error) {
|
|
165
184
|
console.error("[Webmentions] Unhide error:", error);
|
|
166
|
-
response.redirect(
|
|
185
|
+
response.redirect(
|
|
186
|
+
application.webmentionEndpoint + "?error=unhide-failed",
|
|
187
|
+
);
|
|
167
188
|
}
|
|
168
189
|
},
|
|
169
190
|
|
|
170
191
|
/**
|
|
171
192
|
* POST /block - Block a domain
|
|
193
|
+
* @param request
|
|
194
|
+
* @param response
|
|
172
195
|
*/
|
|
173
196
|
async blockDomainHandler(request, response) {
|
|
174
197
|
const { application } = request.app.locals;
|
|
@@ -176,12 +199,14 @@ export const dashboardController = {
|
|
|
176
199
|
try {
|
|
177
200
|
const { domain } = request.body;
|
|
178
201
|
if (!domain) {
|
|
179
|
-
return response.redirect(
|
|
202
|
+
return response.redirect(
|
|
203
|
+
application.webmentionEndpoint + "?error=no-domain",
|
|
204
|
+
);
|
|
180
205
|
}
|
|
181
206
|
|
|
182
|
-
const
|
|
183
|
-
const wmCollection =
|
|
184
|
-
const blockCollection =
|
|
207
|
+
const database = application.getWebmentionDb();
|
|
208
|
+
const wmCollection = database.collection("webmentions");
|
|
209
|
+
const blockCollection = database.collection("webmentionBlocklist");
|
|
185
210
|
|
|
186
211
|
// Hide all existing mentions from this domain
|
|
187
212
|
const hidden = await hideByDomain(wmCollection, domain, "blocklist");
|
|
@@ -189,7 +214,11 @@ export const dashboardController = {
|
|
|
189
214
|
// Add to blocklist
|
|
190
215
|
await blockDomain(blockCollection, domain, "spam", hidden);
|
|
191
216
|
|
|
192
|
-
response.redirect(
|
|
217
|
+
response.redirect(
|
|
218
|
+
application.webmentionEndpoint +
|
|
219
|
+
"?blocked=1&domain=" +
|
|
220
|
+
encodeURIComponent(domain),
|
|
221
|
+
);
|
|
193
222
|
} catch (error) {
|
|
194
223
|
console.error("[Webmentions] Block error:", error);
|
|
195
224
|
response.redirect(application.webmentionEndpoint + "?error=block-failed");
|
|
@@ -198,6 +227,8 @@ export const dashboardController = {
|
|
|
198
227
|
|
|
199
228
|
/**
|
|
200
229
|
* POST /privacy-remove - Privacy removal (delete + block)
|
|
230
|
+
* @param request
|
|
231
|
+
* @param response
|
|
201
232
|
*/
|
|
202
233
|
async privacyRemove(request, response) {
|
|
203
234
|
const { application } = request.app.locals;
|
|
@@ -205,12 +236,14 @@ export const dashboardController = {
|
|
|
205
236
|
try {
|
|
206
237
|
const { domain } = request.body;
|
|
207
238
|
if (!domain) {
|
|
208
|
-
return response.redirect(
|
|
239
|
+
return response.redirect(
|
|
240
|
+
application.webmentionEndpoint + "/blocklist?error=no-domain",
|
|
241
|
+
);
|
|
209
242
|
}
|
|
210
243
|
|
|
211
|
-
const
|
|
212
|
-
const wmCollection =
|
|
213
|
-
const blockCollection =
|
|
244
|
+
const database = application.getWebmentionDb();
|
|
245
|
+
const wmCollection = database.collection("webmentions");
|
|
246
|
+
const blockCollection = database.collection("webmentionBlocklist");
|
|
214
247
|
|
|
215
248
|
// Permanently delete all mentions from this domain
|
|
216
249
|
const deleted = await deleteByDomain(wmCollection, domain);
|
|
@@ -218,10 +251,16 @@ export const dashboardController = {
|
|
|
218
251
|
// Add to blocklist with privacy reason
|
|
219
252
|
await blockDomain(blockCollection, domain, "privacy", deleted);
|
|
220
253
|
|
|
221
|
-
response.redirect(
|
|
254
|
+
response.redirect(
|
|
255
|
+
application.webmentionEndpoint +
|
|
256
|
+
"/blocklist?removed=1&count=" +
|
|
257
|
+
deleted,
|
|
258
|
+
);
|
|
222
259
|
} catch (error) {
|
|
223
260
|
console.error("[Webmentions] Privacy remove error:", error);
|
|
224
|
-
response.redirect(
|
|
261
|
+
response.redirect(
|
|
262
|
+
application.webmentionEndpoint + "/blocklist?error=remove-failed",
|
|
263
|
+
);
|
|
225
264
|
}
|
|
226
265
|
},
|
|
227
266
|
};
|
|
@@ -7,19 +7,29 @@ import { runSync, runFullSync } from "../sync.js";
|
|
|
7
7
|
export const syncController = {
|
|
8
8
|
/**
|
|
9
9
|
* POST /sync - Trigger incremental sync
|
|
10
|
+
* @param request
|
|
11
|
+
* @param response
|
|
10
12
|
*/
|
|
11
13
|
async sync(request, response) {
|
|
12
14
|
const { application } = request.app.locals;
|
|
13
15
|
|
|
14
16
|
try {
|
|
15
|
-
const
|
|
17
|
+
const database = application.getWebmentionDb();
|
|
16
18
|
const options = application.webmentionConfig || {};
|
|
17
|
-
const result = await runSync(
|
|
19
|
+
const result = await runSync(database, options);
|
|
18
20
|
|
|
19
21
|
if (result.error) {
|
|
20
|
-
response.redirect(
|
|
22
|
+
response.redirect(
|
|
23
|
+
application.webmentionEndpoint +
|
|
24
|
+
"?error=" +
|
|
25
|
+
encodeURIComponent(result.error),
|
|
26
|
+
);
|
|
21
27
|
} else {
|
|
22
|
-
response.redirect(
|
|
28
|
+
response.redirect(
|
|
29
|
+
application.webmentionEndpoint +
|
|
30
|
+
"?synced=1&added=" +
|
|
31
|
+
result.mentionsAdded,
|
|
32
|
+
);
|
|
23
33
|
}
|
|
24
34
|
} catch (error) {
|
|
25
35
|
console.error("[Webmentions] Manual sync error:", error);
|
|
@@ -29,19 +39,29 @@ export const syncController = {
|
|
|
29
39
|
|
|
30
40
|
/**
|
|
31
41
|
* POST /sync/full - Trigger full re-sync
|
|
42
|
+
* @param request
|
|
43
|
+
* @param response
|
|
32
44
|
*/
|
|
33
45
|
async fullSync(request, response) {
|
|
34
46
|
const { application } = request.app.locals;
|
|
35
47
|
|
|
36
48
|
try {
|
|
37
|
-
const
|
|
49
|
+
const database = application.getWebmentionDb();
|
|
38
50
|
const options = application.webmentionConfig || {};
|
|
39
|
-
const result = await runFullSync(
|
|
51
|
+
const result = await runFullSync(database, options);
|
|
40
52
|
|
|
41
53
|
if (result.error) {
|
|
42
|
-
response.redirect(
|
|
54
|
+
response.redirect(
|
|
55
|
+
application.webmentionEndpoint +
|
|
56
|
+
"?error=" +
|
|
57
|
+
encodeURIComponent(result.error),
|
|
58
|
+
);
|
|
43
59
|
} else {
|
|
44
|
-
response.redirect(
|
|
60
|
+
response.redirect(
|
|
61
|
+
application.webmentionEndpoint +
|
|
62
|
+
"?synced=1&added=" +
|
|
63
|
+
result.mentionsAdded,
|
|
64
|
+
);
|
|
45
65
|
}
|
|
46
66
|
} catch (error) {
|
|
47
67
|
console.error("[Webmentions] Full sync error:", error);
|
package/lib/hcard.js
CHANGED
|
@@ -151,8 +151,8 @@ function findUPhoto(html) {
|
|
|
151
151
|
|
|
152
152
|
if (!hasClass(tag, "u-photo")) continue;
|
|
153
153
|
|
|
154
|
-
const
|
|
155
|
-
if (
|
|
154
|
+
const source = getAttribute(tag, "src");
|
|
155
|
+
if (source) return source;
|
|
156
156
|
}
|
|
157
157
|
|
|
158
158
|
return null;
|
|
@@ -172,7 +172,7 @@ function findUUrl(html) {
|
|
|
172
172
|
|
|
173
173
|
if (!hasClass(tag, "u-url") && !hasClass(tag, "u-uid")) continue;
|
|
174
174
|
|
|
175
|
-
const href =
|
|
175
|
+
const href = getAttribute(tag, "href");
|
|
176
176
|
if (href) return href;
|
|
177
177
|
}
|
|
178
178
|
|
|
@@ -197,7 +197,7 @@ function hasClass(tag, className) {
|
|
|
197
197
|
* @param {string} attr - Attribute name
|
|
198
198
|
* @returns {string|null}
|
|
199
199
|
*/
|
|
200
|
-
function
|
|
200
|
+
function getAttribute(tag, attr) {
|
|
201
201
|
const regex = new RegExp(`${attr}=["']([^"']*)["']`, "i");
|
|
202
202
|
const match = tag.match(regex);
|
|
203
203
|
return match ? match[1] : null;
|
package/lib/storage/blocklist.js
CHANGED
|
@@ -18,7 +18,12 @@ export async function ensureBlocklistIndexes(collection) {
|
|
|
18
18
|
* @param {number} mentionsHidden - Count of mentions hidden
|
|
19
19
|
* @returns {Promise<boolean>} true if inserted, false if already existed
|
|
20
20
|
*/
|
|
21
|
-
export async function blockDomain(
|
|
21
|
+
export async function blockDomain(
|
|
22
|
+
collection,
|
|
23
|
+
domain,
|
|
24
|
+
reason = "spam",
|
|
25
|
+
mentionsHidden = 0,
|
|
26
|
+
) {
|
|
22
27
|
try {
|
|
23
28
|
await collection.insertOne({
|
|
24
29
|
domain,
|
|
@@ -29,7 +34,7 @@ export async function blockDomain(collection, domain, reason = "spam", mentionsH
|
|
|
29
34
|
return true;
|
|
30
35
|
} catch (error) {
|
|
31
36
|
// Duplicate key — domain already blocked
|
|
32
|
-
if (error.code ===
|
|
37
|
+
if (error.code === 11_000) {
|
|
33
38
|
// Update reason and count
|
|
34
39
|
await collection.updateOne(
|
|
35
40
|
{ domain },
|
|
@@ -79,6 +84,8 @@ export async function isDomainBlocked(collection, domain) {
|
|
|
79
84
|
* @returns {Promise<Set<string>>}
|
|
80
85
|
*/
|
|
81
86
|
export async function getBlockedDomainSet(collection) {
|
|
82
|
-
const entries = await collection
|
|
87
|
+
const entries = await collection
|
|
88
|
+
.find({}, { projection: { domain: 1 } })
|
|
89
|
+
.toArray();
|
|
83
90
|
return new Set(entries.map((e) => e.domain));
|
|
84
91
|
}
|
|
@@ -36,7 +36,8 @@ export function jf2ToDocument(item) {
|
|
|
36
36
|
|
|
37
37
|
return {
|
|
38
38
|
wmId: item["wm-id"],
|
|
39
|
-
wmReceived:
|
|
39
|
+
wmReceived:
|
|
40
|
+
ensureISOString(item["wm-received"]) || new Date().toISOString(),
|
|
40
41
|
wmProperty: item["wm-property"],
|
|
41
42
|
wmTarget: item["wm-target"],
|
|
42
43
|
authorName: item.author?.name || null,
|
|
@@ -75,7 +76,8 @@ export function documentToJf2(doc) {
|
|
|
75
76
|
photo: doc.authorPhoto || "",
|
|
76
77
|
},
|
|
77
78
|
url: doc.sourceUrl || "",
|
|
78
|
-
published:
|
|
79
|
+
published:
|
|
80
|
+
ensureISOString(doc.published) || ensureISOString(doc.wmReceived),
|
|
79
81
|
};
|
|
80
82
|
|
|
81
83
|
if (doc.name) {
|
|
@@ -98,11 +100,11 @@ export function documentToJf2(doc) {
|
|
|
98
100
|
* @returns {Promise<boolean>} true if inserted (new), false if updated
|
|
99
101
|
*/
|
|
100
102
|
export async function upsertWebmention(collection, item) {
|
|
101
|
-
const
|
|
103
|
+
const document = jf2ToDocument(item);
|
|
102
104
|
const result = await collection.updateOne(
|
|
103
|
-
{ wmId:
|
|
105
|
+
{ wmId: document.wmId },
|
|
104
106
|
{
|
|
105
|
-
$setOnInsert:
|
|
107
|
+
$setOnInsert: document,
|
|
106
108
|
},
|
|
107
109
|
{ upsert: true },
|
|
108
110
|
);
|
|
@@ -185,7 +187,13 @@ export async function getMaxWmId(collection) {
|
|
|
185
187
|
export async function hideWebmention(collection, wmId, reason = "manual") {
|
|
186
188
|
await collection.updateOne(
|
|
187
189
|
{ wmId },
|
|
188
|
-
{
|
|
190
|
+
{
|
|
191
|
+
$set: {
|
|
192
|
+
hidden: true,
|
|
193
|
+
hiddenAt: new Date().toISOString(),
|
|
194
|
+
hiddenReason: reason,
|
|
195
|
+
},
|
|
196
|
+
},
|
|
189
197
|
);
|
|
190
198
|
}
|
|
191
199
|
|
|
@@ -211,7 +219,13 @@ export async function unhideWebmention(collection, wmId) {
|
|
|
211
219
|
export async function hideByDomain(collection, domain, reason = "blocklist") {
|
|
212
220
|
const result = await collection.updateMany(
|
|
213
221
|
{ sourceDomain: domain, hidden: { $ne: true } },
|
|
214
|
-
{
|
|
222
|
+
{
|
|
223
|
+
$set: {
|
|
224
|
+
hidden: true,
|
|
225
|
+
hiddenAt: new Date().toISOString(),
|
|
226
|
+
hiddenReason: reason,
|
|
227
|
+
},
|
|
228
|
+
},
|
|
215
229
|
);
|
|
216
230
|
return result.modifiedCount;
|
|
217
231
|
}
|
package/lib/sync.js
CHANGED
|
@@ -2,7 +2,11 @@
|
|
|
2
2
|
* Background sync from webmention.io
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { discoverAuthorData } from "./hcard.js";
|
|
6
|
+
import {
|
|
7
|
+
ensureBlocklistIndexes,
|
|
8
|
+
getBlockedDomainSet,
|
|
9
|
+
} from "./storage/blocklist.js";
|
|
6
10
|
import {
|
|
7
11
|
ensureIndexes,
|
|
8
12
|
upsertWebmention,
|
|
@@ -12,11 +16,7 @@ import {
|
|
|
12
16
|
getDomainsWithMissingPhotos,
|
|
13
17
|
updateAuthorDataByDomain,
|
|
14
18
|
} from "./storage/webmentions.js";
|
|
15
|
-
import {
|
|
16
|
-
ensureBlocklistIndexes,
|
|
17
|
-
getBlockedDomainSet,
|
|
18
|
-
} from "./storage/blocklist.js";
|
|
19
|
-
import { discoverAuthorData } from "./hcard.js";
|
|
19
|
+
import { extractDomain } from "./utils.js";
|
|
20
20
|
|
|
21
21
|
let syncInterval = null;
|
|
22
22
|
let syncState = {
|
|
@@ -49,15 +49,15 @@ export function startSync(Indiekit, options) {
|
|
|
49
49
|
|
|
50
50
|
// Initial sync after delay
|
|
51
51
|
setTimeout(() => {
|
|
52
|
-
runSync(Indiekit, options).catch((
|
|
53
|
-
console.error("[Webmentions] Initial sync error:",
|
|
52
|
+
runSync(Indiekit, options).catch((error) => {
|
|
53
|
+
console.error("[Webmentions] Initial sync error:", error.message);
|
|
54
54
|
});
|
|
55
55
|
}, 10_000);
|
|
56
56
|
|
|
57
57
|
// Recurring sync
|
|
58
58
|
syncInterval = setInterval(() => {
|
|
59
|
-
runSync(Indiekit, options).catch((
|
|
60
|
-
console.error("[Webmentions] Sync error:",
|
|
59
|
+
runSync(Indiekit, options).catch((error) => {
|
|
60
|
+
console.error("[Webmentions] Sync error:", error.message);
|
|
61
61
|
});
|
|
62
62
|
}, intervalMs);
|
|
63
63
|
}
|
|
@@ -66,10 +66,12 @@ export function startSync(Indiekit, options) {
|
|
|
66
66
|
* Stop background sync
|
|
67
67
|
*/
|
|
68
68
|
export function stopSync() {
|
|
69
|
-
if (syncInterval) {
|
|
70
|
-
|
|
71
|
-
syncInterval = null;
|
|
69
|
+
if (!syncInterval) {
|
|
70
|
+
return;
|
|
72
71
|
}
|
|
72
|
+
|
|
73
|
+
clearInterval(syncInterval);
|
|
74
|
+
syncInterval = null;
|
|
73
75
|
}
|
|
74
76
|
|
|
75
77
|
/**
|
|
@@ -79,8 +81,8 @@ export function stopSync() {
|
|
|
79
81
|
* @returns {Promise<object>}
|
|
80
82
|
*/
|
|
81
83
|
export async function runSync(dbOrIndiekit, options) {
|
|
82
|
-
const
|
|
83
|
-
if (!
|
|
84
|
+
const database = dbOrIndiekit.database || dbOrIndiekit;
|
|
85
|
+
if (!database || typeof database.collection !== "function") {
|
|
84
86
|
syncState.lastError = "No database available";
|
|
85
87
|
return { error: syncState.lastError };
|
|
86
88
|
}
|
|
@@ -95,8 +97,8 @@ export async function runSync(dbOrIndiekit, options) {
|
|
|
95
97
|
syncState.mentionsFiltered = 0;
|
|
96
98
|
|
|
97
99
|
try {
|
|
98
|
-
const wmCollection =
|
|
99
|
-
const blockCollection =
|
|
100
|
+
const wmCollection = database.collection("webmentions");
|
|
101
|
+
const blockCollection = database.collection("webmentionBlocklist");
|
|
100
102
|
|
|
101
103
|
await ensureIndexes(wmCollection);
|
|
102
104
|
await ensureBlocklistIndexes(blockCollection);
|
|
@@ -145,7 +147,7 @@ export async function runSync(dbOrIndiekit, options) {
|
|
|
145
147
|
}
|
|
146
148
|
|
|
147
149
|
// Enrich entries with missing author photos via h-card discovery
|
|
148
|
-
const enriched = await enrichMissingPhotos(
|
|
150
|
+
const enriched = await enrichMissingPhotos(database, wmCollection);
|
|
149
151
|
|
|
150
152
|
syncState.lastSync = new Date().toISOString();
|
|
151
153
|
syncState.syncing = false;
|
|
@@ -174,8 +176,8 @@ export async function runSync(dbOrIndiekit, options) {
|
|
|
174
176
|
* @returns {Promise<object>}
|
|
175
177
|
*/
|
|
176
178
|
export async function runFullSync(dbOrIndiekit, options) {
|
|
177
|
-
const
|
|
178
|
-
if (!
|
|
179
|
+
const database = dbOrIndiekit.database || dbOrIndiekit;
|
|
180
|
+
if (!database || typeof database.collection !== "function") {
|
|
179
181
|
return { error: "No database available" };
|
|
180
182
|
}
|
|
181
183
|
|
|
@@ -189,15 +191,17 @@ export async function runFullSync(dbOrIndiekit, options) {
|
|
|
189
191
|
syncState.mentionsFiltered = 0;
|
|
190
192
|
|
|
191
193
|
try {
|
|
192
|
-
const wmCollection =
|
|
193
|
-
const blockCollection =
|
|
194
|
+
const wmCollection = database.collection("webmentions");
|
|
195
|
+
const blockCollection = database.collection("webmentionBlocklist");
|
|
194
196
|
|
|
195
197
|
await ensureIndexes(wmCollection);
|
|
196
198
|
await ensureBlocklistIndexes(blockCollection);
|
|
197
199
|
|
|
198
200
|
// Clear all existing webmentions
|
|
199
201
|
const deleted = await deleteAll(wmCollection);
|
|
200
|
-
console.log(
|
|
202
|
+
console.log(
|
|
203
|
+
`[Webmentions] Full sync: cleared ${deleted} existing mentions`,
|
|
204
|
+
);
|
|
201
205
|
|
|
202
206
|
// Get blocked domains
|
|
203
207
|
const blockedDomains = await getBlockedDomainSet(blockCollection);
|
|
@@ -240,7 +244,7 @@ export async function runFullSync(dbOrIndiekit, options) {
|
|
|
240
244
|
}
|
|
241
245
|
|
|
242
246
|
// Enrich entries with missing author photos via h-card discovery
|
|
243
|
-
const enriched = await enrichMissingPhotos(
|
|
247
|
+
const enriched = await enrichMissingPhotos(database, wmCollection);
|
|
244
248
|
|
|
245
249
|
syncState.lastSync = new Date().toISOString();
|
|
246
250
|
syncState.syncing = false;
|
|
@@ -302,10 +306,7 @@ async function enrichMissingPhotos(db, wmCollection) {
|
|
|
302
306
|
await delay(200);
|
|
303
307
|
}
|
|
304
308
|
} catch (error) {
|
|
305
|
-
console.error(
|
|
306
|
-
"[Webmentions] h-card enrichment error:",
|
|
307
|
-
error.message,
|
|
308
|
-
);
|
|
309
|
+
console.error("[Webmentions] h-card enrichment error:", error.message);
|
|
309
310
|
}
|
|
310
311
|
|
|
311
312
|
return totalUpdated;
|
|
@@ -344,6 +345,10 @@ async function fetchPage(options, params = {}) {
|
|
|
344
345
|
return body?.children || [];
|
|
345
346
|
}
|
|
346
347
|
|
|
348
|
+
/**
|
|
349
|
+
*
|
|
350
|
+
* @param ms
|
|
351
|
+
*/
|
|
347
352
|
function delay(ms) {
|
|
348
353
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
349
354
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rmdes/indiekit-endpoint-webmention-io",
|
|
3
|
-
"version": "1.0
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Webmention moderation endpoint for Indiekit. Syncs webmentions from webmention.io into MongoDB with delete, block, and privacy removal capabilities.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"indiekit",
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
},
|
|
24
24
|
"license": "MIT",
|
|
25
25
|
"engines": {
|
|
26
|
-
"node": ">=
|
|
26
|
+
"node": ">=24.17"
|
|
27
27
|
},
|
|
28
28
|
"type": "module",
|
|
29
29
|
"main": "index.js",
|
|
@@ -37,16 +37,21 @@
|
|
|
37
37
|
"index.js"
|
|
38
38
|
],
|
|
39
39
|
"dependencies": {
|
|
40
|
-
"@
|
|
41
|
-
"
|
|
42
|
-
"
|
|
43
|
-
"express": "^5.0.0",
|
|
44
|
-
"sanitize-html": "^2.14.0"
|
|
40
|
+
"@indiekit/error": "^1.0.0-beta.28",
|
|
41
|
+
"express": "^5.2.1",
|
|
42
|
+
"sanitize-html": "^2.17.7"
|
|
45
43
|
},
|
|
46
44
|
"peerDependencies": {
|
|
47
45
|
"@indiekit/indiekit": ">=1.0.0-beta.25"
|
|
48
46
|
},
|
|
49
47
|
"publishConfig": {
|
|
50
48
|
"access": "public"
|
|
49
|
+
},
|
|
50
|
+
"scripts": {
|
|
51
|
+
"test": "NODE_ENV=test node --test"
|
|
52
|
+
},
|
|
53
|
+
"devDependencies": {
|
|
54
|
+
"@indiekit/util": "^1.0.0-beta.29",
|
|
55
|
+
"mongodb-memory-server": "^11.2.0"
|
|
51
56
|
}
|
|
52
57
|
}
|