fedipod-server 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/README.md +22 -6
  2. package/dist/claims.d.ts +8 -0
  3. package/dist/claims.js +10 -0
  4. package/dist/handler.d.ts +13 -0
  5. package/dist/handler.js +56 -11
  6. package/dist/handler.jsonld +8 -0
  7. package/dist/store-pod.js +18 -4
  8. package/lib/{c2s.mjs → client/c2s.mjs} +10 -3
  9. package/lib/{localapi.mjs → client/localapi.mjs} +2 -2
  10. package/lib/client/masto/accounts.mjs +264 -0
  11. package/lib/client/masto/body.mjs +69 -0
  12. package/lib/client/masto/index.mjs +183 -0
  13. package/lib/client/masto/instance.mjs +104 -0
  14. package/lib/client/masto/media.mjs +133 -0
  15. package/lib/client/masto/oauth.mjs +599 -0
  16. package/lib/client/masto/render.mjs +459 -0
  17. package/lib/client/masto/statuses.mjs +331 -0
  18. package/lib/client/masto/timelines.mjs +316 -0
  19. package/lib/{streaming.mjs → client/streaming.mjs} +1 -1
  20. package/lib/{acctfeed.mjs → connections/acctfeed.mjs} +1 -1
  21. package/lib/{atproto.mjs → connections/atproto.mjs} +15 -16
  22. package/lib/{bskygroup.mjs → connections/bskygroup.mjs} +1 -1
  23. package/lib/{fediacct.mjs → connections/fediacct.mjs} +31 -35
  24. package/lib/{import.mjs → connections/import.mjs} +1 -1
  25. package/lib/{tagfeed.mjs → connections/tagfeed.mjs} +3 -3
  26. package/lib/connections/vault.mjs +114 -0
  27. package/lib/core/as2.mjs +170 -0
  28. package/lib/core/contexts/activitystreams.json +379 -0
  29. package/lib/core/contexts/did-v1.json +57 -0
  30. package/lib/core/contexts/fep-5711.json +36 -0
  31. package/lib/core/contexts/gotosocial.json +86 -0
  32. package/lib/core/contexts/identity-v1.json +152 -0
  33. package/lib/core/contexts/index.mjs +45 -0
  34. package/lib/core/contexts/join-lemmy.json +33 -0
  35. package/lib/core/contexts/joinmastodon.json +28 -0
  36. package/lib/core/contexts/map.json +16 -0
  37. package/lib/core/contexts/miscellany.json +19 -0
  38. package/lib/core/contexts/schemaorg.json +8845 -0
  39. package/lib/core/contexts/security-data-integrity-v1.json +78 -0
  40. package/lib/core/contexts/security-data-integrity-v2.json +81 -0
  41. package/lib/core/contexts/security-multikey-v1.json +35 -0
  42. package/lib/core/contexts/security-v1.json +74 -0
  43. package/lib/core/contexts/webfinger.json +10 -0
  44. package/lib/{deliver.mjs → core/deliver.mjs} +2 -2
  45. package/lib/core/graphview.mjs +269 -0
  46. package/lib/core/intake/activities.mjs +437 -0
  47. package/lib/core/intake/activity.mjs +240 -0
  48. package/lib/core/intake/channel.mjs +144 -0
  49. package/lib/core/intake/group.mjs +222 -0
  50. package/lib/core/intake/index.mjs +629 -0
  51. package/lib/core/intake/notes.mjs +288 -0
  52. package/lib/core/intake/verify.mjs +142 -0
  53. package/lib/{keys.mjs → core/keys.mjs} +1 -1
  54. package/lib/core/publisher/collections.mjs +229 -0
  55. package/lib/core/publisher/index.mjs +421 -0
  56. package/lib/core/publisher/notes.mjs +188 -0
  57. package/lib/core/publisher/questions.mjs +233 -0
  58. package/lib/core/publisher/restore.mjs +199 -0
  59. package/lib/core/shapes/activitystreams.ttl +129 -0
  60. package/lib/core/shapes/index.mjs +107 -0
  61. package/lib/core/shapes/shapes-text.mjs +13 -0
  62. package/lib/{social.mjs → core/social.mjs} +2 -2
  63. package/lib/{store.mjs → core/store.mjs} +4 -0
  64. package/lib/{wire.mjs → core/wire.mjs} +2 -2
  65. package/lib/device/admin/index.mjs +13 -0
  66. package/lib/device/admin/origins.mjs +35 -0
  67. package/lib/device/admin/routes/connections.mjs +144 -0
  68. package/lib/device/admin/routes/gateway.mjs +199 -0
  69. package/lib/device/admin/routes/lifecycle.mjs +191 -0
  70. package/lib/device/admin/routes/owner.mjs +322 -0
  71. package/lib/device/admin/routes/setup.mjs +393 -0
  72. package/lib/device/admin/routes/social.mjs +188 -0
  73. package/lib/device/admin/server.mjs +95 -0
  74. package/lib/device/admin/static.mjs +244 -0
  75. package/lib/device/admin/surface.mjs +274 -0
  76. package/lib/device/cli/commands/account.mjs +586 -0
  77. package/lib/device/cli/commands/run.mjs +278 -0
  78. package/lib/device/cli/commands/service.mjs +221 -0
  79. package/lib/device/cli/commands/setup.mjs +410 -0
  80. package/lib/device/cli/commands/state.mjs +559 -0
  81. package/lib/device/cli/context.mjs +288 -0
  82. package/lib/{migrate.mjs → device/migrate.mjs} +1 -1
  83. package/lib/{remote.mjs → device/remote.mjs} +3 -3
  84. package/lib/{setup.mjs → device/setup.mjs} +3 -3
  85. package/lib/{update.mjs → device/update.mjs} +1 -1
  86. package/lib/{directory.mjs → gateway/directory.mjs} +1 -1
  87. package/lib/{front-core.mjs → gateway/front-core.mjs} +3 -3
  88. package/lib/{gateway-core.mjs → gateway/gateway-core.mjs} +1 -1
  89. package/lib/{httpsig.mjs → gateway/httpsig.mjs} +1 -1
  90. package/lib/server/embed.mjs +405 -0
  91. package/lib/{links.mjs → shared/links.mjs} +1 -1
  92. package/lib/{ua.mjs → shared/ua.mjs} +1 -1
  93. package/package.json +1 -1
  94. package/run-agent.mjs +33 -25
  95. package/web/admin/actors.js +145 -0
  96. package/web/admin/common.js +23 -0
  97. package/web/admin/connections.js +112 -0
  98. package/web/admin/gateway.js +111 -0
  99. package/web/admin/group.js +258 -0
  100. package/web/admin/index.html +7 -1
  101. package/web/admin/record.js +378 -0
  102. package/web/admin/setup/index.html +1 -0
  103. package/web/admin/setup/setup.js +2 -13
  104. package/web/admin/upkeep.js +170 -0
  105. package/web/app/README.md +6 -6
  106. package/web/app/admin-facade.mjs +3 -3
  107. package/web/app/agent.mjs +14 -16
  108. package/web/app/atproto-browser.mjs +1 -1
  109. package/web/app/boot.mjs +2 -3
  110. package/web/app/deliver-relay.mjs +1 -1
  111. package/web/app/dist/boot.js +22 -3
  112. package/web/app/dist/boot.js.map +2 -2
  113. package/web/app/dist/sw.js +21913 -5446
  114. package/web/app/dist/sw.js.map +4 -4
  115. package/web/app/fediacct-browser.mjs +1 -1
  116. package/web/app/keys-browser.mjs +27 -4
  117. package/web/app/shims/shapes-text.mjs +8 -0
  118. package/web/app/signup.mjs +2 -3
  119. package/web/app/site/admin/actors.js +145 -0
  120. package/web/app/site/admin/common.js +23 -0
  121. package/web/app/site/admin/connections.js +112 -0
  122. package/web/app/site/admin/gateway.js +111 -0
  123. package/web/app/site/admin/group.js +258 -0
  124. package/web/app/site/admin/index.html +7 -1
  125. package/web/app/site/admin/record.js +378 -0
  126. package/web/app/site/admin/setup/index.html +1 -0
  127. package/web/app/site/admin/setup/setup.js +2 -13
  128. package/web/app/site/admin/upkeep.js +170 -0
  129. package/web/app/site/boot.js +22 -3
  130. package/web/app/site/sw.js +21913 -5446
  131. package/web/app/sw-src.mjs +17 -2
  132. package/lib/admin.mjs +0 -1913
  133. package/lib/embed.mjs +0 -220
  134. package/lib/intake.mjs +0 -1981
  135. package/lib/mastoapi.mjs +0 -2284
  136. package/lib/publisher.mjs +0 -1192
  137. package/web/admin/admin.js +0 -1181
  138. package/web/app/site/admin/admin.js +0 -1181
  139. /package/lib/{oidc-auth.mjs → client/oidc-auth.mjs} +0 -0
  140. /package/lib/{webpush.mjs → client/webpush.mjs} +0 -0
  141. /package/lib/{bskyfeed.mjs → connections/bskyfeed.mjs} +0 -0
  142. /package/lib/{lease.mjs → core/lease.mjs} +0 -0
  143. /package/lib/{polls.mjs → core/polls.mjs} +0 -0
  144. /package/lib/{proof.mjs → core/proof.mjs} +0 -0
  145. /package/lib/{storage.mjs → core/storage.mjs} +0 -0
  146. /package/lib/{account.mjs → device/account.mjs} +0 -0
  147. /package/lib/{certs.mjs → device/certs.mjs} +0 -0
  148. /package/lib/{export-collections.mjs → device/export-collections.mjs} +0 -0
  149. /package/lib/{home.mjs → device/home.mjs} +0 -0
  150. /package/lib/{ports.mjs → device/ports.mjs} +0 -0
  151. /package/lib/{guard.mjs → shared/guard.mjs} +0 -0
  152. /package/lib/{safefetch.mjs → shared/safefetch.mjs} +0 -0
package/lib/intake.mjs DELETED
@@ -1,1981 +0,0 @@
1
- // intake.mjs — drains the remote pod's public-append inbox and applies side
2
- // effects. Inbound authenticity: LDN bodies don't carry the delivery's
3
- // HTTP-Signature headers, so instead of verifying signatures we VERIFY BY
4
- // DEREFERENCING — re-fetch the claimed object/actor from its origin (signed
5
- // GET, so authorized-fetch instances answer) and trust only what the origin
6
- // itself serves.
7
- //
8
- // Failure policy: a REJECTED item (verification says no) goes to the
9
- // dead-letter store and leaves the inbox; a FAILING item (exception —
10
- // network, remote 5xx) stays in the inbox for the next drain, and moves to
11
- // the dead-letter store after MAX_ITEM_ATTEMPTS. Nothing is silently
12
- // destroyed.
13
- //
14
- // Wake-up: WebSocketChannel2023 push on the inbox container (probe P4), plus
15
- // a poll every POLL_MS as fallback, plus a drain at startup.
16
-
17
- import * as $rdf from 'rdflib';
18
- import * as podInbox from './pod/inbox.mjs';
19
- import * as podNotifications from './pod/notifications.mjs';
20
- import * as podNotes from './pod/notes.mjs';
21
- import { USER_AGENT } from './ua.mjs';
22
- import { PUBLIC } from './wire.mjs';
23
- import { HTTP_TIMEOUT_MS, readCapped } from './safefetch.mjs';
24
- import { linkTargets, REL } from './links.mjs';
25
- import * as polls from './polls.mjs';
26
- import { dropFollower } from './store.mjs';
27
-
28
- const RDF = $rdf.Namespace('http://www.w3.org/1999/02/22-rdf-syntax-ns#');
29
- const NOTIFY = $rdf.Namespace('http://www.w3.org/ns/solid/notifications#');
30
-
31
- const POLL_MS = 2 * 60_000; // fallback cadence when there is no push
32
- // With a live socket the poll is pure redundancy: it exists for the case where
33
- // push is down, so it slows right down while push is up.
34
- const POLL_PUSH_OK_MS = 10 * 60_000;
35
- // The channel a subscription returns outlives a dropped socket, so reconnecting
36
- // reuses it. Creating a new one per reconnect is what buried solidcommunity.net
37
- // in channel records they then had to sweep.
38
- const CHANNEL_DOC = 'inbox-channel.json';
39
- // A flapping socket used to POST a NEW WebSocketChannel2023 channel every two
40
- // seconds — hundreds an hour against a server that is already struggling, and
41
- // channel churn its operators have to sweep up. Backs off instead, and an open
42
- // only triggers a sweep if we have not just swept.
43
- const RECONNECT_MIN_MS = 2_000;
44
- const RECONNECT_MAX_MS = 5 * 60_000;
45
- // How long a socket must stay up before the backoff counts it as a success and
46
- // resets. Shorter than that is a flap, not a connection.
47
- const RECONNECT_STABLE_MS = 60_000;
48
- const OPEN_DRAIN_MIN_GAP_MS = 30_000;
49
- // A container that times out will time out again in two minutes, and each
50
- // attempt holds one of the pod's workers for the full timeout. Sweeping stops
51
- // for a while instead, doubling up to half an hour.
52
- const DRAIN_COOLDOWN_MIN_MS = 2 * 60_000;
53
- const DRAIN_COOLDOWN_MAX_MS = 30 * 60_000;
54
- // Our DELETEs take the same container write lock as the deliveries arriving
55
- // into it — a gap between them keeps a sweep from convoying against inbound.
56
- const DELETE_GAP_MS = 150;
57
- const CHAIN_GAP_MS = 5_000; // pause between chained backlog sweeps
58
- // How many handled items ride on one commit before they are deleted. Small
59
- // enough that a crash re-does little, large enough that a flood of fast
60
- // rejections does not become a pod write per item.
61
- const DELETE_BATCH = 10;
62
- // Attempt counts live in pod state, not in memory: a restart used to hand every
63
- // poison item five fresh tries, and under a crash loop that is unbounded.
64
- const ATTEMPTS_DOC = 'intake-attempts.json';
65
- const ATTEMPTS_TTL_MS = 7 * 24 * 60 * 60_000;
66
- const MAX_ITEM_ATTEMPTS = 5;
67
- const MAX_ITEMS_PER_DRAIN = 50;
68
- // A note's replies collection is rewritten WHOLE every time one is added, so
69
- // without a cap the bytes are quadratic in a number a stranger chooses.
70
- const MAX_REPLIES_RECORDED = 500;
71
- // Ids of activities already forwarded to our followers (§7.1.2), so a re-drain
72
- // never re-broadcasts one. A ceiling on the record, not on forwarding.
73
- const MAX_FORWARDED = 2000;
74
- // How many posts may wait for a group operator's decision. A ceiling, not a
75
- // window: full refuses the newest rather than dropping the oldest.
76
- const MAX_PENDING_REVIEW = 500;
77
- // A group's membership, cached: it changes when someone joins or leaves, and
78
- // re-reading it on every arriving post would spend one stranger's fetch per
79
- // message on a list that moves in days.
80
- const CO_MEMBER_TTL_MS = 24 * 60 * 60_000;
81
- const CO_MEMBER_MAX = 5000;
82
- // An activity is a few kB. This is generous by two orders of magnitude and
83
- // still bounds what one Append can make us hold in memory.
84
- const MAX_ITEM_BYTES = 512 * 1024;
85
- // The AS2 actor types. A group is as much an actor as a person is.
86
- const ACTOR_TYPES = new Set(['Person', 'Group', 'Service', 'Application', 'Organization']);
87
- // The AS2 types the fediverse actually posts. `Note` alone is Mastodon's world
88
- // and not the fediverse's: an Article is a Plume or WriteFreely post, a
89
- // Question is a poll, a Video is PeerTube, a Page is Lemmy, an Audio is
90
- // Funkwhale. Insisting on Note dead-lettered every one of them as "not a
91
- // verifiable Note" — from people the owner had chosen to follow, silently.
92
- //
93
- // They share the shape this code reads: attributedTo, content, published,
94
- // inReplyTo, tag, attachment. A poll's options are dropped, which is a
95
- // degraded rendering rather than a lost post.
96
- const CONTENT_TYPES = new Set(['Note', 'Article', 'Question', 'Page', 'Video', 'Audio', 'Image', 'Event']);
97
-
98
- // A Question is a poll: its options live in oneOf (pick one) or anyOf (pick
99
- // several), each carrying the tally its author's server maintains.
100
- // Custom emojis ride the tag list; the images live at the author's server and
101
- // the client fetches them from there.
102
- // Content, name and summary were capped; nothing else was. One remote Question
103
- // with thousands of options, or thousands of emoji, mention or attachment
104
- // entries, put megabytes into a single statuses.json row — a document rewritten
105
- // whole on every change. These are display lists: past a few dozen, nothing can
106
- // render them and nobody meant them to be rendered.
107
- // http(s) only. An emoji, attachment or mention URL is written straight into
108
- // the client's markup, so `javascript:` and `data:` have no business in one.
109
- // The store has guarded avatars this way all along (safeUrl, lib/store.mjs);
110
- // these three lists were simply never put through it.
111
- const httpOnly = (u) => {
112
- if (!u) return null;
113
- try {
114
- const p = new URL(String(u));
115
- return (p.protocol === 'https:' || p.protocol === 'http:') ? String(u) : null;
116
- } catch { return null; }
117
- };
118
-
119
- const MAX_MODQUEUE = 200;
120
- const MAX_EMOJIS = 60;
121
- const MAX_POLL_OPTIONS = 50;
122
- const MAX_OPTION_CHARS = 200;
123
- const MAX_MENTIONS = 60;
124
- const MAX_URL_CHARS = 2048;
125
-
126
- function emojisOf(note) {
127
- return [].concat(note?.tag || [])
128
- .filter(t => t?.type === 'Emoji' && t.icon?.url && t.name)
129
- .slice(0, MAX_EMOJIS)
130
- .map(t => ({
131
- shortcode: String(t.name).replace(/^:|:$/g, '').slice(0, 64),
132
- url: httpOnly(String(t.icon.url).slice(0, MAX_URL_CHARS)),
133
- }))
134
- .filter(e => e.url);
135
- }
136
-
137
- function pollOf(note) {
138
- const opts = note?.oneOf || note?.anyOf;
139
- if (!Array.isArray(opts) || !opts.length) return null;
140
- return {
141
- multiple: !!note.anyOf,
142
- expiresAt: note.endTime || null,
143
- closed: !!note.closed,
144
- options: opts.slice(0, MAX_POLL_OPTIONS).map(o => ({
145
- title: String(o?.name ?? '').slice(0, MAX_OPTION_CHARS),
146
- votes: Number(o?.replies?.totalItems) || 0,
147
- })),
148
- };
149
- }
150
- // AS2 lets `type` be one string or a list, and implementations use both —
151
- // `["Person","Service"]` is an ordinary actor. Read either form.
152
- const typesOf = (t) => (Array.isArray(t) ? t : [t]).filter(x => typeof x === 'string');
153
- export const isContentType = (t) => typesOf(t).some(x => CONTENT_TYPES.has(x));
154
- const isActorType = (t) => typesOf(t).some(x => ACTOR_TYPES.has(x));
155
-
156
- // What we will carry to our followers on someone else's behalf (§7.1.2): the
157
- // activities a conversation is made of, and nothing else. A type this file
158
- // does not handle falls out of handle() with no rejection, and "no rejection"
159
- // is what qualifies an activity for forwarding — so without this gate a
160
- // stranger could have anything at all, of a type nothing here reads,
161
- // re-delivered to every follower over our signature.
162
- const FORWARDABLE = new Set(['Create', 'Update', 'Delete', 'Like', 'Announce', 'Undo']);
163
- // Of those, the ones we may re-deliver to our own followers over our own
164
- // signature (§7.1.2). Deliberately narrower than FORWARDABLE: these three are
165
- // the ones whose object this drain fetched from the author's origin and checked
166
- // before accepting. A Like, an Announce or an Undo is taken on the envelope's
167
- // word alone — relaying one is signing for a claim nothing corroborated.
168
- const FORWARD_TYPES = new Set(['Create', 'Update', 'Delete']);
169
- // How many forwards one drain may send. A reply into a busy thread of ours is a
170
- // handful; anything near this is a flood using us as an amplifier.
171
- const MAX_FORWARDS_PER_DRAIN = 20;
172
- const ACCEPT_AP = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams"';
173
-
174
- // What is worth keeping of an activity we are filing rather than acting on.
175
- //
176
- // A dead letter, a moderation entry and a waiting follow request each used to
177
- // hold the WHOLE delivered activity — up to the 512 KB item cap — in documents
178
- // that are serialized and PUT whole on every change. A stranger could inflate
179
- // deadletter.json to ~100 MB and requests.json to ~250 MB and make us rewrite
180
- // them on every batch.
181
- //
182
- // Everything the consumers need survives: `acceptActivity` wraps this as the
183
- // Accept's object (and {id, type, actor, object} IS the standard Follow shape),
184
- // `applyModeration` reads only `object`, and a dead letter is read by a human
185
- // who wants to know what arrived, not to replay it.
186
- export function trimActivity(a) {
187
- if (!a || typeof a !== 'object') return a ?? null;
188
- const idOf = (v) => (typeof v === 'string' ? v : v?.id ?? null);
189
- const out = {};
190
- for (const k of ['id', 'type', 'actor', 'target']) {
191
- const v = idOf(a[k]);
192
- if (v) out[k] = String(v).slice(0, 2048);
193
- }
194
- const obj = idOf(a.object);
195
- if (obj) out.object = String(obj).slice(0, 2048);
196
- // A typed object with no id still says what it was — a Block of nobody, an
197
- // Undo of a Follow — and that is the whole of what the queue reads.
198
- else if (a.object && typeof a.object === 'object' && a.object.type) {
199
- out.object = { type: String(a.object.type).slice(0, 64) };
200
- }
201
- return out;
202
- }
203
-
204
- // Same origin AND, where the origin holds more than one identity, the same
205
- // identity within it.
206
- //
207
- // Origin alone is the right test when an origin is one person — the ordinary
208
- // fediverse server, and a subdomain pod. It is the WRONG test on a path-based
209
- // host, and above all on a multi-tenant front: every tenant of fedipod.net has
210
- // ids under `https://fedipod.net/u/<name>/`, so origin-equality made every
211
- // tenant able to vouch for every other. One tenant could publish posts
212
- // attributed to another, Update a neighbour's post, or Delete it — and a
213
- // receiving Mastodon would believe it for the same reason we did.
214
- //
215
- // So when both ids carry an identity prefix, the prefixes must match too. The
216
- // two shapes that exist here are the front's `/u/<name>/` and a pod's own AP
217
- // root (`…/<root>/ap/…`); anything else has no prefix and falls back to origin,
218
- // which is what a plain remote server should be judged by.
219
- function identityPrefix(u) {
220
- const m = /^(https?:\/\/[^/]+\/u\/[^/]+\/)/u.exec(u);
221
- if (m) return m[1];
222
- const ap = /^(https?:\/\/[^/]+\/(?:[^/]+\/)*?)ap\//u.exec(u);
223
- return ap ? ap[1] : null;
224
- }
225
-
226
- export function sameIdentity(a, b) {
227
- if (!sameOrigin(a, b)) return false;
228
- const pa = identityPrefix(String(a));
229
- const pb = identityPrefix(String(b));
230
- if (!pa || !pb) return true; // no prefix to compare: origin is the answer
231
- return pa === pb;
232
- }
233
-
234
- export function sameOrigin(a, b) {
235
- try { return new URL(a).origin === new URL(b).origin; } catch { return false; }
236
- }
237
-
238
- // Is this socket URL the pod's own? The scheme has to be the socket form of the
239
- // pod's — wss for https, ws for http — so a downgrade to plaintext from an https
240
- // pod is somewhere else, not the same place unencrypted.
241
- //
242
- // The host may be the pod's, or a PARENT of it. Not a loosening for
243
- // convenience: a CSS server that gives every pod a subdomain answers
244
- // notifications from the server root, so jeff-zucker.teamid.live is served by
245
- // wss://teamid.live/.notifications/… — which is the deployment this project
246
- // actually runs on. Requiring an exact match dropped it to polling, and the
247
- // live agents are how that was found rather than the suite.
248
- //
249
- // A sibling subdomain is still refused: only a suffix of our own host passes,
250
- // and two labels minimum so `.live` cannot pose as everyone's parent. Not a
251
- // public-suffix list — that is a dependency and a data file to keep current,
252
- // and the party this guards against is the pod you already chose to trust.
253
- //
254
- // `localhost` is the one single-label parent allowed, because it is the one that
255
- // cannot be anybody else: it is reserved to the loopback interface (RFC 6761),
256
- // so `alice.localhost` and `localhost` are the same machine by definition and
257
- // there is no stranger for the rule to keep out. Without this a pod served from
258
- // a subdomain of localhost — which is how a Solid server with subdomain pods
259
- // runs on a developer's machine — refused its own socket and fell back to
260
- // polling, so every delivery waited up to two minutes.
261
- export function sameSocketOrigin(socketUrl, podBase) {
262
- let s, p;
263
- try { s = new URL(socketUrl); p = new URL(podBase); } catch { return false; }
264
- if (s.protocol !== (p.protocol === 'https:' ? 'wss:' : 'ws:')) return false;
265
- if (s.host === p.host) return true;
266
- const parent = s.hostname.toLowerCase();
267
- return (parent.split('.').length >= 2 || parent === 'localhost')
268
- && s.port === p.port
269
- && p.hostname.toLowerCase().endsWith('.' + parent);
270
- }
271
-
272
- export function httpUrl(u) {
273
- try {
274
- const p = new URL(String(u)).protocol;
275
- return p === 'https:' || p === 'http:';
276
- } catch { return false; }
277
- }
278
-
279
- // Who a note is BY. A document may only speak for an actor at its own origin.
280
- // `attributedTo` used to be taken at face value, so a note served anywhere
281
- // could name anyone: one at a host the attacker controls, claiming to be by
282
- // someone the owner follows, passed every check we had — the envelope's
283
- // sameOrigin compares the ACTIVITY to its object, never the object to its
284
- // author — and landed in the home timeline and in the pod as them. For a group
285
- // it went further still, because amplify() gates on the author's membership,
286
- // so the group signed an Announce of it and delivered it to every member.
287
- //
288
- // `delivered` is the actor that brought it, used only when the note names no
289
- // author of its own; it has to clear the same test, which is why a boost of an
290
- // unattributed note is refused rather than credited to the booster.
291
- //
292
- // Returns the author, or null when nothing at the note's origin vouches for one.
293
- export function authorOf(note, delivered = null) {
294
- const claimed = [].concat(note?.attributedTo || [])
295
- .map(a => (typeof a === 'string' ? a : a?.id)).find(Boolean) || null;
296
- const author = claimed || delivered;
297
- if (!author) return null;
298
- // sameIdentity, not sameOrigin: on a multi-tenant front every tenant shares
299
- // an origin, so origin-equality let any of them be credited with any other's
300
- // post. See sameIdentity.
301
- return sameIdentity(note?.id, author) ? author : null;
302
- }
303
-
304
- export class Intake {
305
- constructor({ config, urls, remote, store, deliverer, publisher, log = console.log, lease = null, archive = null, push = true, pollSeconds = null }) {
306
- Object.assign(this, { config, urls, remote, store, deliverer, publisher, log, lease, archive, push, pollSeconds });
307
- this.serial = Date.now();
308
- this.stopped = false;
309
- // (attempt counts are kept in pod state — see _bumpAttempt)
310
- this.lastDrain = null;
311
- this._forwardBudget = MAX_FORWARDS_PER_DRAIN;
312
- this.lastDrainAtMs = 0;
313
- this.reconnectTries = 0;
314
- this.drainCooldownUntil = 0;
315
- this.drainFailures = 0;
316
- this.wsState = 'never-connected';
317
- }
318
-
319
- // Draining is a destructive read — an item is gone from the pod once we
320
- // DELETE it — so the result of handling it must be on disk first.
321
- //
322
- // This used to be skipped whenever the state and the inbox shared an origin,
323
- // on the reasoning that a pod we cannot write to is a pod we cannot list
324
- // either, so the drain never starts. That covers the pod being unreachable
325
- // and nothing else: it does not cover a crash inside the 300ms debounce
326
- // window, and it does not cover a pod that refuses a write while still
327
- // serving reads and deletes — a quota, a 507, a 403 on one document. In
328
- // either case every item drained since the last successful write is gone,
329
- // and what goes is the mentions, replies, join requests and dead-letter
330
- // records that nothing else can rebuild.
331
- async _persisted() {
332
- return this.store.commit();
333
- }
334
-
335
- _backOff(why) {
336
- this.drainFailures++;
337
- const capped = Math.min(DRAIN_COOLDOWN_MIN_MS * 2 ** (this.drainFailures - 1), DRAIN_COOLDOWN_MAX_MS);
338
- this.drainCooldownUntil = Date.now() + Math.round(capped * (0.85 + Math.random() * 0.3));
339
- this.log(`${why} — next sweep in ${Math.round(capped / 1000)}s`);
340
- }
341
-
342
- // Sweep cadence: a configured interval wins, otherwise a live push channel —
343
- // a notification socket, or the store's own change events in-process — is
344
- // what makes the fallback poll a slow one.
345
- _pollMs() {
346
- if (this.pollSeconds) return this.pollSeconds * 1000;
347
- return this.wsState === 'open' || this.wsState === 'in-process' ? POLL_PUSH_OK_MS : POLL_MS;
348
- }
349
-
350
- async start() {
351
- this.stopped = false; // restartable across demote/takeover cycles
352
- await this.drain().catch(e => this.log(`drain: ${e.message}`));
353
- const tick = () => {
354
- this.pollTimer = setTimeout(() => {
355
- this.drain().catch(e => this.log(`drain: ${e.message}`)).finally(() => { if (!this.stopped) tick(); });
356
- }, Math.round(this._pollMs() * (0.85 + Math.random() * 0.3)));
357
- this.pollTimer.unref?.();
358
- };
359
- tick();
360
- // Embedded in the pod server, a notification socket back to that same
361
- // server buys nothing — the store's own change events wake the drain.
362
- if (this.push) this.subscribe().catch(e => this.log(`subscribe: ${e.message}`));
363
- else this.wsState = 'in-process';
364
- }
365
-
366
- stop() { this.stopped = true; clearTimeout(this.pollTimer); clearTimeout(this.resubTimer); this.ws?.close(); }
367
-
368
- // Attempt bookkeeping, persisted. Written only when an item fails, so a
369
- // healthy inbox never touches this document.
370
- _bumpAttempt(url, message) {
371
- const all = this.store.read(ATTEMPTS_DOC, {});
372
- const rec = all[url] || { n: 0 };
373
- rec.n += 1;
374
- rec.at = new Date().toISOString();
375
- rec.last = String(message || '').slice(0, 200);
376
- all[url] = rec;
377
- this.store.write(ATTEMPTS_DOC, all);
378
- return rec.n;
379
- }
380
-
381
- _clearAttempt(url) {
382
- const all = this.store.read(ATTEMPTS_DOC, {});
383
- if (!all[url]) return;
384
- delete all[url];
385
- this.store.write(ATTEMPTS_DOC, all);
386
- }
387
-
388
- // Items deleted long ago would otherwise accumulate here forever.
389
- _pruneAttempts() {
390
- const all = this.store.read(ATTEMPTS_DOC, {});
391
- const cutoff = Date.now() - ATTEMPTS_TTL_MS;
392
- let dropped = 0;
393
- for (const [url, rec] of Object.entries(all)) {
394
- if (!rec?.at || Date.parse(rec.at) < cutoff) { delete all[url]; dropped++; }
395
- }
396
- if (dropped) { this.store.write(ATTEMPTS_DOC, all); this.log(`pruned ${dropped} stale inbox attempt record(s)`); }
397
- }
398
-
399
- // Jittered exponential, floor to ceiling, reset by a successful open.
400
- _reconnectDelay() {
401
- const capped = Math.min(RECONNECT_MIN_MS * 2 ** this.reconnectTries, RECONNECT_MAX_MS);
402
- this.reconnectTries++;
403
- return Math.round(capped * (0.8 + Math.random() * 0.4));
404
- }
405
-
406
- // --- push ---
407
- // Any failure in here used to end push for the life of the process: the
408
- // retry lived only in the "server refused the subscription" branch, so a
409
- // network blip left wsState at never-connected and the agent silently on
410
- // polling. Every path now schedules a retry on the same backoff.
411
- async subscribe() {
412
- try {
413
- await this._subscribeOnce();
414
- } catch (e) {
415
- this.wsState = 'subscribe-error';
416
- const wait = this._reconnectDelay();
417
- this.log(`subscribe failed (${e.message}) — retrying in ${Math.round(wait / 1000)}s (polling meanwhile)`);
418
- if (!this.stopped) {
419
- this.resubTimer = setTimeout(() => this.subscribe().catch(() => {}), wait);
420
- this.resubTimer.unref?.();
421
- }
422
- }
423
- }
424
-
425
- /**
426
- * Where this pod describes the services it offers. The pod says so on any
427
- * response about one of its resources; the well-known path is only what a
428
- * pod that says nothing has always used.
429
- */
430
- async _storageDescriptionUrl() {
431
- return podNotifications.storageDescriptionUrl(this.urls.base,
432
- { headers: { 'user-agent': USER_AGENT }, timeoutMs: HTTP_TIMEOUT_MS });
433
- }
434
-
435
- async _subscribeOnce() {
436
- // Reuse a channel we already have rather than asking for another one.
437
- const saved = this.store.read(CHANNEL_DOC, null);
438
- if (saved?.receiveFrom && (!saved.endAt || Date.parse(saved.endAt) - Date.now() > 60_000)) {
439
- this._openSocket(saved.receiveFrom, true);
440
- return;
441
- }
442
- const descUrl = await this._storageDescriptionUrl();
443
- const { channel, error } = await podNotifications.readWebSocketChannel(descUrl,
444
- { headers: { 'user-agent': USER_AGENT }, timeoutMs: HTTP_TIMEOUT_MS });
445
- if (!channel) { this.wsState = 'unavailable'; this.log(`${error} — polling only`); return; }
446
- // The topic is a POD resource, and it travels in the BODY — so the url map
447
- // RemotePod applies to the request line never reaches it. A fronted
448
- // identity's inbox url names the front, which the pod cannot grant read on,
449
- // and the subscription came back 403. A no-op when unfronted.
450
- const topic = this.urls.toPod ? this.urls.toPod(this.urls.inbox) : this.urls.inbox;
451
- const sub = await podNotifications.subscribeToInbox(this.remote,
452
- { channelUrl: channel, podTopicUrl: topic });
453
- const body = await readCapped(sub).then(JSON.parse).catch(() => null);
454
- if (!body?.receiveFrom) {
455
- this.wsState = `subscribe-failed-${sub.status}`;
456
- const wait = this._reconnectDelay();
457
- this.log(`subscription failed (${sub.status}) — retrying in ${Math.round(wait / 1000)}s (polling meanwhile)`);
458
- if (!this.stopped) {
459
- this.resubTimer = setTimeout(() => this.subscribe().catch(e => this.log(`resubscribe: ${e.message}`)), wait);
460
- this.resubTimer.unref?.();
461
- }
462
- return;
463
- }
464
- this.store.write(CHANNEL_DOC, { receiveFrom: body.receiveFrom, endAt: body.endAt || null });
465
- this._openSocket(body.receiveFrom, false);
466
- }
467
-
468
- // The socket URL arrives in the pod's own subscription response, and it was
469
- // the one outbound address in the project that reached the network without
470
- // passing anything — safefetch guards every fetch, and `new WebSocket()` is
471
- // not a fetch. A pod that answered with somebody else's address had us open a
472
- // long-lived connection there and treat what came back as our inbox waking up.
473
- //
474
- // Same origin as the pod, not assertPublicUrl: a pod on this machine is a
475
- // documented setup and its socket is legitimately ws://localhost:3000, which
476
- // a public-address check would refuse.
477
- _openSocket(receiveFrom, reused) {
478
- if (!sameSocketOrigin(receiveFrom, this.urls.base)) {
479
- this.wsState = 'refused';
480
- this.log(`subscription named ${receiveFrom}, which is not this pod — polling only`);
481
- if (reused) this.store.write(CHANNEL_DOC, null);
482
- return;
483
- }
484
- this.ws = new WebSocket(receiveFrom);
485
- this.ws.onopen = () => {
486
- this.wsState = 'open';
487
- if (!this._announcedPush) { this.log('inbox push subscription active'); this._announcedPush = true; }
488
- this._openedAt = Date.now();
489
- // Anything that arrived while the socket was down is waiting — sweep it,
490
- // unless a sweep just ran: a flapping socket must not re-list the inbox
491
- // on every open.
492
- if (Date.now() - this.lastDrainAtMs > OPEN_DRAIN_MIN_GAP_MS) {
493
- this.drain().catch(e => this.log(`drain: ${e.message}`));
494
- }
495
- };
496
- this.ws.onmessage = () => this.drain().catch(e => this.log(`drain: ${e.message}`));
497
- this.ws.onclose = () => {
498
- this.wsState = 'closed';
499
- // Only a connection that STAYED up counts as a success. Resetting on open
500
- // alone meant the 2026-07-29 failure — a server that accepts the upgrade
501
- // and then drops the socket on a crash cycle — reconnected at the 2s
502
- // floor indefinitely: every cycle "succeeded", so the exponential cap was
503
- // never reached, and each open also drained the inbox.
504
- if (this._openedAt && Date.now() - this._openedAt >= RECONNECT_STABLE_MS) this.reconnectTries = 0;
505
- this._openedAt = 0;
506
- if (!this.stopped) {
507
- this.resubTimer = setTimeout(() => this.subscribe().catch(e => this.log(`resubscribe: ${e.message}`)), this._reconnectDelay());
508
- this.resubTimer.unref?.();
509
- }
510
- };
511
- this.ws.onerror = () => {
512
- this.wsState = 'error';
513
- // A channel we reused may simply be gone: forget it so the next attempt
514
- // asks for a fresh one instead of retrying a dead URL forever.
515
- if (reused) this.store.write(CHANNEL_DOC, null);
516
- };
517
- }
518
-
519
- // --- drain + dispatch ---
520
- // Serialized: push events, polls, and manual /drain calls can fire
521
- // concurrently, and overlapping sweeps double-process items (observed as
522
- // duplicate Accepts/timeline writes). One sweep at a time; callers that
523
- // arrive mid-sweep get one follow-up sweep.
524
- async drain() {
525
- if (this._draining) { this._drainAgain = true; return this._draining; }
526
- // Held for the whole sweep, released however it ends. The debounce cannot
527
- // coalesce a drain — every handler awaits somebody else's server first — so
528
- // the writes are left to the commit boundaries the drain already has. See
529
- // PodStore.hold.
530
- // Optional: this is a throughput hint, not part of the commit-before-delete
531
- // invariant — commit() flushes whatever is pending either way — so a store
532
- // that does not implement it behaves exactly as before.
533
- this._inSweep = true;
534
- // Fresh per sweep: the cap is on how much this drain may amplify, not a
535
- // lifetime total (see _maybeForward).
536
- this._forwardBudget = MAX_FORWARDS_PER_DRAIN;
537
- this.store.hold?.();
538
- this._draining = this._drainOnce().finally(async () => {
539
- this._inSweep = false;
540
- await this._publishPending();
541
- this.store.release?.();
542
- this._draining = null;
543
- if (this._drainAgain) {
544
- this._drainAgain = false;
545
- // Paced, not immediate: chained sweeps put a ceiling on work per unit
546
- // time, so a delivery flood cannot run the drain back-to-back.
547
- const t = setTimeout(() => this.drain().catch(e => this.log(`drain: ${e.message}`)), CHAIN_GAP_MS);
548
- t.unref?.();
549
- }
550
- });
551
- return this._draining;
552
- }
553
-
554
- // Discard the content waiting in the inbox from before `before`, on the
555
- // owner's say-so — the admin page asks, this does it. NOT a blind sweep:
556
- // every item is read, because the type is the only thing that decides its
557
- // fate and size does not predict it. A Follow, Undo, Accept or Delete is
558
- // APPLIED whatever it weighs, so the follow graph stays correct and a post
559
- // its author retracted still goes; only a Create is dropped. Judging by size
560
- // instead saved one request per item and lost any control activity that
561
- // happened to be large, silently and permanently.
562
- //
563
- // An item past the drain's own byte cap is deleted unread: it could not be
564
- // handled if it were read, so there is nothing to lose by not reading it.
565
- //
566
- // `keepConcerning` narrows the discard to noise: every item is passed to
567
- // handle(), which ingests a Create only when concernsUs passes — addressed to
568
- // us, a mention, a reply to ours, or from someone we follow — and drops the
569
- // rest.
570
- async prune({ before, keepConcerning = false } = {}) {
571
- const cutoff = Date.parse(before);
572
- if (!Number.isFinite(cutoff)) throw new Error(`"${before}" is not a date`);
573
- const all = await podInbox.list(this.remote, this.urls);
574
- const older = all.filter(e => !e.url.endsWith('.keep')
575
- && e.modified && Date.parse(e.modified) < cutoff);
576
- const out = { considered: older.length, applied: 0, dropped: 0, discarded: 0, failed: 0 };
577
-
578
- for (const item of older) {
579
- try {
580
- if (item.size > MAX_ITEM_BYTES) {
581
- await podInbox.dropHandledItem(this.remote, item.url); // unreadable by the drain either way
582
- out.discarded++;
583
- } else {
584
- // Same rule as the drain: a read we could not make is not a Create to
585
- // be dropped. readItem throws on anything but 404, so it counts as
586
- // failed and stays for the next pass.
587
- const got = await podInbox.readItem(this.remote, item.url, { maxBytes: MAX_ITEM_BYTES, readCapped });
588
- const activity = got.raw === null ? null : (() => { try { return JSON.parse(got.raw); } catch { return null; } })();
589
- // A Create is the content the owner just asked to be rid of. Anything
590
- // else changes state and is applied exactly as a drain would.
591
- if (keepConcerning) {
592
- const rejection = activity ? await this.handle(activity) : 'unparsable JSON';
593
- if (rejection) out.dropped++; else out.applied++;
594
- } else if (activity && activity.type !== 'Create') {
595
- await this.handle(activity);
596
- out.applied++;
597
- } else {
598
- out.dropped++;
599
- }
600
- if (!await this._persisted()) {
601
- this.log(`state not written — stopping the prune with ${older.length - out.applied - out.dropped - out.discarded} left`);
602
- break;
603
- }
604
- await podInbox.dropHandledItem(this.remote, item.url);
605
- }
606
- this._clearAttempt(item.url);
607
- await new Promise(r => setTimeout(r, DELETE_GAP_MS));
608
- } catch (e) {
609
- out.failed++;
610
- this.log(`prune ${item.url}: ${e.message}`);
611
- }
612
- }
613
- this.log(`pruned before ${before}: applied ${out.applied}, dropped ${out.dropped} `
614
- + `small Create(s), discarded ${out.discarded} unread${out.failed ? `, ${out.failed} failed` : ''}`);
615
- await this.store.flush();
616
- // Adjust the measurement in place rather than kicking a drain to re-take
617
- // it: an un-awaited drain would still be running when this returns, which
618
- // races whoever called us. The poll picks the rest up soon enough.
619
- const removed = out.applied + out.dropped + out.discarded;
620
- if (this.inboxStats && removed) {
621
- this.inboxStats = { ...this.inboxStats, count: Math.max(0, this.inboxStats.count - removed) };
622
- }
623
- return out;
624
- }
625
-
626
- // The sender's original bytes, kept after the activity was verified and
627
- // applied and before the pod DELETE erases the only other copy. The filename
628
- // is the content's own hash, so a re-delivered activity lands on the same
629
- // file instead of duplicating. Best-effort history: a failed write logs and
630
- // the drain goes on — mail must never stall on its own receipt. This is the
631
- // one category of account data that lives only in the private half; the pod
632
- // cannot rebuild it because the pod never kept it.
633
- //
634
- // The record is JSON-LD: a cnt:ContentAsText whose cnt:chars are the raw
635
- // bytes, stamped prov:generatedAtTime / prov:wasDerivedFrom / as:actor —
636
- // plain JSON to everything here, a graph to any RDF reader.
637
- async _archive(sourceUrl, raw, activity) {
638
- try {
639
- if (!this.archive || this.store.getConfig()?.archiveInbox === false) return;
640
- const { createHash } = await import('node:crypto');
641
- const hash = createHash('sha256').update(raw).digest('hex').slice(0, 16);
642
- const receivedAt = new Date().toISOString();
643
- const rec = {
644
- '@context': {
645
- prov: 'http://www.w3.org/ns/prov#',
646
- cnt: 'http://www.w3.org/2011/content#',
647
- as: 'https://www.w3.org/ns/activitystreams#',
648
- xsd: 'http://www.w3.org/2001/XMLSchema#',
649
- receivedAt: { '@id': 'prov:generatedAtTime', '@type': 'xsd:dateTime' },
650
- source: { '@id': 'prov:wasDerivedFrom', '@type': '@id' },
651
- actor: { '@id': 'as:actor', '@type': '@id' },
652
- raw: 'cnt:chars',
653
- },
654
- '@id': '',
655
- '@type': 'cnt:ContentAsText',
656
- receivedAt,
657
- actor: typeof activity?.actor === 'string' ? activity.actor : activity?.actor?.id || null,
658
- source: sourceUrl,
659
- raw,
660
- };
661
- const w = await this.archive.write(`${receivedAt.slice(0, 7)}/${hash}.json`,
662
- JSON.stringify(rec, null, 2), 'application/ld+json');
663
- if (!w.ok) this.log(`inbox archive: ${w.why || 'write failed'}`);
664
- } catch (e) {
665
- this.log(`inbox archive: ${e.message}`);
666
- }
667
- }
668
-
669
- async _drainOnce() {
670
- const cooling = this.drainCooldownUntil - Date.now();
671
- if (cooling > 0) {
672
- this.log(`inbox sweep skipped — backing off for another ${Math.ceil(cooling / 1000)}s`);
673
- return;
674
- }
675
- // Draining DELETES from the pod, so it must not run on a lease that has
676
- // quietly expired. renewOnce notices at its own cadence — up to ~117s — and
677
- // after the TTL another agent is entitled to start draining the same inbox.
678
- if (this.lease && !this.lease.stillHeld()) {
679
- this.log('lease is no longer held — not draining');
680
- return;
681
- }
682
- this.lastDrain = new Date().toISOString();
683
- this.lastDrainAtMs = Date.now();
684
- this._pruneAttempts();
685
- let all;
686
- try {
687
- all = await podInbox.list(this.remote, this.urls);
688
- this.drainFailures = 0;
689
- } catch (e) {
690
- this._backOff(`inbox unreadable (${e.message})`);
691
- return;
692
- }
693
- // What is waiting, measured from the listing we already fetched: no extra
694
- // request, and it is what /status reports and what the admin page prompts
695
- // on. The listing arrives oldest-first (lib/remote.mjs).
696
- const real = all.filter(e => !e.url.endsWith('.keep'));
697
- this.inboxStats = {
698
- count: real.length,
699
- bytes: real.reduce((n, e) => n + e.size, 0),
700
- oldest: real[0]?.modified || null,
701
- newest: real[real.length - 1]?.modified || null,
702
- at: new Date().toISOString(),
703
- };
704
- // The inbox is public-Append: a flood must not turn one sweep into an
705
- // unbounded run. But stopping there is why a backlog never cleared — 50
706
- // items every two minutes does not converge on an agent that is only
707
- // running while a laptop is open. So a sweep that made progress and left
708
- // work behind goes straight round again.
709
- const items = all.slice(0, MAX_ITEMS_PER_DRAIN);
710
- if (all.length > items.length) this.log(`inbox has ${all.length} items — processing ${items.length} this sweep`);
711
- let handled = 0;
712
- // Deletes are batched behind ONE commit rather than a commit per item.
713
- // Per-item, the 300ms debounce that coalesces a sweep's writes never gets
714
- // to do its job: on a flood of fast rejections that is fifty writes of
715
- // deadletter.json where one would do, and a flood is exactly when the pod
716
- // should be asked for less rather than more.
717
- const pending = [];
718
- const flush = async () => {
719
- if (!pending.length) return true;
720
- // Written down before any of them leaves the mailbox. A failure here
721
- // leaves them where they are: the next sweep sees them again, and a
722
- // re-delivered activity is handled idempotently.
723
- if (!await this._persisted()) {
724
- this._backOff(`state not written — ${pending.length} item(s) left in the inbox`);
725
- pending.length = 0;
726
- return false;
727
- }
728
- for (const url of pending.splice(0)) {
729
- if (!await podInbox.dropHandledItem(this.remote, url)) {
730
- // Still in the mailbox. Handling is idempotent so seeing it again is
731
- // harmless, but counting it would clear the attempt record and report
732
- // progress that did not happen.
733
- this.log(`inbox item ${url} was handled but NOT removed — it will be seen again`);
734
- continue;
735
- }
736
- this._clearAttempt(url);
737
- handled++;
738
- await new Promise(r => setTimeout(r, DELETE_GAP_MS));
739
- }
740
- return true;
741
- };
742
-
743
- for (const { url, size } of items) {
744
- if (url.endsWith('.keep')) continue;
745
- // The listing already carries every child's size, so this costs nothing
746
- // to ask. An activity is a few kB; anything of this order is not one, and
747
- // reading it with an unbounded res.text() buffers whatever a stranger
748
- // chose to Append into memory.
749
- // A cheap pre-filter only: listContainer coerces a missing posix:size to
750
- // 0, so a pod that does not publish sizes would wave everything through.
751
- // The real bound is readCapped on the body below.
752
- if (size > MAX_ITEM_BYTES) {
753
- this.store.addDeadLetter({ inboxUrl: url, reason: `oversized (${size} bytes)`, activity: null });
754
- pending.push(url);
755
- continue;
756
- }
757
- let activity = null;
758
- try {
759
- const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_ITEM_BYTES, readCapped });
760
- // readItem carries the rule that matters here: a pod that would not
761
- // GIVE us the item has told us nothing about it, so anything but a 404
762
- // throws rather than reading as an empty body. Reading a 500 as empty
763
- // made it "unparsable JSON" — a REJECTION, dead-lettered with both
764
- // `activity` and `raw` null and then DELETEd, destroying a delivery on
765
- // a transient fault with no record of what it had been.
766
- const raw = got.raw;
767
- try { activity = raw ? JSON.parse(raw) : null; } catch { /* kept raw for the dead letter */ }
768
- // A gateway that verified this delivery left a receipt beside it. Read
769
- // it only when a gateway is configured (no config → no fetch, so an
770
- // install with no gateway pays nothing); a missing or HMAC-invalid
771
- // receipt reads as null, which is exactly today's unverified behavior.
772
- const receipt = activity ? await this._readReceipt(url) : null;
773
- if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
774
- const rejection = activity ? await this.handle(activity, receipt) : 'unparsable JSON';
775
- if (!rejection && raw) await this._archive(url, raw, activity);
776
- if (!rejection) await this._maybeForward(activity); // §7.1.2, only what we accepted
777
- if (rejection) {
778
- this.store.addDeadLetter({
779
- inboxUrl: url, reason: rejection, activity: trimActivity(activity),
780
- ...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
781
- });
782
- this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
783
- }
784
- pending.push(url);
785
- } catch (e) {
786
- const n = this._bumpAttempt(url, e.message);
787
- this.log(`inbox item ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
788
- if (n >= MAX_ITEM_ATTEMPTS) {
789
- this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(activity) });
790
- // The dead letter IS the record of this item — deleting before it is
791
- // written down would lose the only evidence it ever arrived, so it
792
- // goes through the same commit-then-delete batch as everything else.
793
- pending.push(url);
794
- }
795
- }
796
- if (pending.length >= DELETE_BATCH && !await flush()) return;
797
- }
798
- if (!await this._finishSweep(flush)) return;
799
- // Made progress and there is more waiting: go straight round rather than
800
- // sleeping. Gated on progress so a sweep that achieved nothing — a
801
- // cooldown, an unwritable store, poison at the head — cannot spin.
802
- if (handled > 0 && all.length > items.length && !this.stopped) this._drainAgain = true;
803
- }
804
-
805
- // The end of a sweep: publish whatever the follow graph did ONCE, then flush.
806
- //
807
- // publishCollections used to run per handled item — every Follow, Undo,
808
- // Accept, Reject, admit and eject — and each one is a full GET of the pod's
809
- // followers collection plus a PUT of it. Fifty follows in a sweep were a
810
- // hundred requests where two would do, and the answer they arrive at is the
811
- // same either way, because it is built from contacts.json in memory.
812
- async _finishSweep(flush) {
813
- await this._publishPending();
814
- return flush();
815
- }
816
-
817
- // Idempotent: it clears what it takes, so the drain's own exit path calling
818
- // it again after a sweep that bailed early — an unwritable store, a delete
819
- // that failed — is a no-op in the ordinary case and the difference between
820
- // "published" and "waiting for a sweep that may never come" in the other.
821
- async _publishPending() {
822
- const want = this._republish;
823
- this._republish = null;
824
- if (!want) return;
825
- try { await this.publisher.publishCollections(want); }
826
- catch (e) { this.log(`publishing collections: ${e.message}`); }
827
- }
828
-
829
- // Ask for a collection to be republished at the end of this sweep. Outside a
830
- // sweep there is no boundary to wait for, so it happens now.
831
- async republish(which) {
832
- if (!this._inSweep) return this.publisher.publishCollections(which);
833
- this._republish = { ...(this._republish || {}), ...which };
834
- }
835
-
836
- async fetchAP(url) {
837
- const res = await this.deliverer.signedFetch(url, { headers: { accept: ACCEPT_AP } });
838
- if (res.status >= 400) return null;
839
- // Remote servers are untrusted: read with a byte budget rather than
840
- // letting res.json() buffer whatever they choose to send.
841
- const { readCapped } = await import('./safefetch.mjs');
842
- // Plenty of servers answer 200 text/html however politely we ask for AS2 —
843
- // people reply to ordinary web pages, and their id is that page. Say so,
844
- // rather than handing the HTML to JSON.parse and logging the parser's
845
- // complaint about an unexpected `<`.
846
- // Not logged: people reply to ordinary web pages, so the reply's object id
847
- // is that page and this is the expected answer, not a fault. Only a server
848
- // that CLAIMS to be sending JSON and then does not is worth a line.
849
- const ct = (res.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
850
- if (ct && !ct.endsWith('json')) return null;
851
- let doc = null;
852
- try { doc = JSON.parse(await readCapped(res)); }
853
- catch (e) { this.log(`fetch ${url}: unreadable as JSON — ${e.message}`); return null; }
854
- // A document is only evidence about its OWN origin.
855
- //
856
- // signedFetch follows redirects and hands back only the final response, and
857
- // every caller checks `doc.id` against the URL it ASKED for — so an open
858
- // redirector on a victim's origin let an attacker's server answer for a
859
- // victim's URL and be believed by that check.
860
- //
861
- // Only a redirect that CROSSED AN ORIGIN is refused here. A same-origin
862
- // redirect is ordinary (canonicalisation, trailing slashes), and a document
863
- // fetched with no redirect at all is still returned whatever it claims —
864
- // the callers reject it on id, and they say something more useful about it
865
- // than this could ("actor id mismatch", "object not verifiable content").
866
- const landed = res.finalUrl || url;
867
- if (!sameOrigin(landed, url) && doc && doc.id && !sameOrigin(doc.id, landed)) {
868
- this.log(`fetch ${url}: redirected to ${landed}, which is not where ${doc.id} lives — refused`);
869
- return null;
870
- }
871
- // Every actor type, not just Person. A Group was fetched, used and thrown
872
- // away, so nothing knew its preferredUsername — and a client rendering it
873
- // fell back to the last path segment of the actor URL, which is the literal
874
- // word `actor`. That is where @actor@host came from.
875
- // Under the id the document CLAIMS, but only when its own origin vouches
876
- // for that id. A stranger's actor document naming someone else's id used to
877
- // overwrite that actor's cached name, bio, avatar and Person/Group flag —
878
- // one appended Follow was enough, and the id-mismatch checks in onFollow
879
- // and ingestNote both run after this line and never undid it.
880
- //
881
- // Same origin rather than exact equality: signedFetch follows redirects
882
- // without reporting where it landed, so a server that redirects its own
883
- // canonical actor URL would otherwise stop being cached at all.
884
- if (isActorType(doc?.type) && doc.id && sameOrigin(doc.id, url)) {
885
- this.store.cacheActor(doc.id, doc);
886
- }
887
- return doc;
888
- }
889
-
890
- sameOrigin(a, b) { return sameOrigin(a, b); }
891
- // Overridable in tests the same way sameOrigin is.
892
- sameIdentity(a, b) { return sameIdentity(a, b); }
893
-
894
- // Have we ever heard of this actor or object? Answered entirely from local
895
- // state, so asking costs nothing. It is what stops a stranger's Delete or
896
- // Update — of which Mastodon broadcasts a great many, and of which anyone at
897
- // all can Append one — turning into a signed request to a host they chose.
898
- known(id) {
899
- const c = this.store.getContacts();
900
- return c.followers.some(f => f.actor === id)
901
- || c.following.some(f => f.actor === id)
902
- || this.store.getStatuses().some(s => s.noteId === id || s.actor === id)
903
- || !!this.store.getActors()[id];
904
- }
905
-
906
- // Returns a rejection reason string, or undefined when handled.
907
- // The gateway's shared HMAC secret, or null when no gateway is configured.
908
- // Its absence is what makes the whole receipt path dormant by default.
909
- gatewaySecret() {
910
- return this.store.getConfig()?.gateway?.hmacSecret || null;
911
- }
912
-
913
- // Shadow-mode measurement: how much real traffic actually verified. The one
914
- // number an operator needs before trusting receipts. Only touched while a
915
- // gateway is configured, so it costs a non-gateway install nothing.
916
- _bumpGatewayStat(verified) {
917
- const s = this.store.read('gateway-stats.json', { verified: 0, unverified: 0 });
918
- if (verified) s.verified++; else s.unverified++;
919
- s.lastAt = new Date().toISOString();
920
- this.store.write('gateway-stats.json', s);
921
- }
922
-
923
- // Read and authenticate the receipt a gateway wrote beside an inbox item.
924
- // Returns the receipt object only when its HMAC verifies against our secret;
925
- // null otherwise (no gateway, no receipt, a stranger's forged one, or a read
926
- // failure) — and null means "unverified", the pre-gateway behavior.
927
- async _readReceipt(itemUrl) {
928
- const secret = this.gatewaySecret();
929
- if (!secret) return null;
930
- try {
931
- const { readCapped: cap } = await import('./safefetch.mjs');
932
- const receipt = await podInbox.readDeliveryReceipt(this.remote, itemUrl, { maxBytes: 64 * 1024, readCapped: cap });
933
- if (!receipt) return null;
934
- const { verifyReceipt } = await import('./httpsig.mjs');
935
- return verifyReceipt(receipt, secret) ? receipt : null;
936
- } catch { return null; }
937
- }
938
-
939
- // Whether a receipt says anything about THIS actor. Verified-and-about-someone
940
- // -else is worth exactly as much as unverified, and is treated the same way:
941
- // the drain's verify-by-dereference still stands behind it.
942
- receiptVouchesFor(receipt, actor) {
943
- if (!receipt?.verified) return false;
944
- if (!receipt.actor || receipt.actor !== actor) return false;
945
- // A missing keyId with verified:true cannot come from our own door
946
- // (makeReceipt fills both from the same key), so refuse rather than waive.
947
- if (!receipt.keyId || !sameOrigin(receipt.keyId, actor)) return false;
948
- return true;
949
- }
950
-
951
- async handle(activity, receipt = null) { // eslint-disable-line no-unused-vars
952
- const actor = typeof activity.actor === 'string' ? activity.actor : activity.actor?.id;
953
- if (!actor) return 'no actor';
954
- // An actor is a URL that can be dereferenced. Most arms here go on to fetch
955
- // it, and safefetch refuses a bad scheme there — but Like and Announce on
956
- // one of our own notes record a notification without dereferencing
957
- // anything, so `javascript:` and `data:` reached the client as an account
958
- // url. The store already guards avatars this way (safeUrl); actors were
959
- // simply never put through it.
960
- if (!httpUrl(actor)) return `actor is not an http(s) URL (${actor})`;
961
- if (this.store.isBlocked(actor)) return `blocked sender (${actor})`;
962
-
963
- // Whether the door vouched for this sender — read by the moderation queue
964
- // just below as well as by the arms further down, so it is settled here,
965
- // before its first use.
966
- const trusted = this.receiptVouchesFor(receipt, actor)
967
- && this.store.getConfig()?.gateway?.mode === 'trust';
968
-
969
- // FEP-1b12 moderation from a LISTED moderator. A delivery proves nothing
970
- // about its sender, which is why these are QUEUED for the operator rather
971
- // than run on arrival — the queue is where a claimed moderator's word
972
- // waits for the one person who can vouch for it. Everything else about
973
- // the activity falls through to the ordinary arms.
974
- if (this.config.kind === 'group'
975
- && (this.config.moderators || []).includes(actor)
976
- && this.isModerationAsk(activity)) {
977
- return this.queueModeration(activity, actor, { trusted });
978
- }
979
-
980
- // `trusted` was settled above, before the moderation queue reads it. Why it
981
- // is not simply `receipt.verified`: that says the door checked a signature
982
- // and the signature was good; it does NOT say whose. The door reports the
983
- // signing key's owner separately, in `receipt.actor`, and nothing here used
984
- // to read it — so ANY valid fediverse signing key, over an activity whose
985
- // `actor` field named someone else entirely, arrived as trusted. In trust
986
- // mode that is one delivery to evict any of your followers, or to have a
987
- // Follow naming a third party auto-accepted. See receiptVouchesFor, which
988
- // also holds the keyId to the actor's origin: a key document is fetched
989
- // from wherever its id points, so one hosted elsewhere that merely CLAIMS
990
- // `owner: <you>` would otherwise bind.
991
-
992
- switch (activity.type) {
993
- case 'Follow': return this.onFollow(activity, actor, { trusted });
994
- case 'Undo': return this.onUndo(activity, actor, { trusted });
995
- case 'Create': return this.onCreate(activity, actor);
996
- case 'Accept': return this.onAccept(activity, actor, { trusted });
997
- case 'Like': case 'Announce': {
998
- // FEP-1b12: a group Announces the member's whole Create, not the note.
999
- // Without unwrapping we try to ingest a Create as if it were a Note and
1000
- // dead-letter every post a group ever carries — including our own.
1001
- const wrapped = activity.object;
1002
- // FEP-1b12: a group announces its moderation too. The one act a
1003
- // follower can honor without trusting anyone new is a Delete of a
1004
- // post that same group carried to us — the carrier unsaying its carry.
1005
- if (activity.type === 'Announce' && wrapped && typeof wrapped === 'object'
1006
- && wrapped.type === 'Delete') {
1007
- return this.onAnnouncedDelete(actor, wrapped);
1008
- }
1009
- const inner = (wrapped && typeof wrapped === 'object'
1010
- && (wrapped.type === 'Create' || wrapped.type === 'Update')) ? wrapped.object : wrapped;
1011
- const objectId = typeof inner === 'string' ? inner : inner?.id;
1012
- this.log(`${activity.type} from ${actor} on ${objectId}`);
1013
- if (objectId && objectId.startsWith(this.urls.notes)) {
1014
- // Nothing vouches for this actor: a Like carries no signature and,
1015
- // unlike a Create, has no object at the sender's origin to re-read.
1016
- // `known()` is answered from local state and costs nothing — a
1017
- // stranger's favourite is still recorded, it is just the first thing
1018
- // the cap evicts, so a flood cannot push out real history.
1019
- this.store.addNotification({
1020
- type: activity.type === 'Like' ? 'favourite' : 'reblog', actor, noteId: objectId,
1021
- ...(this.known(actor) ? {} : { unverified: true }),
1022
- });
1023
- return;
1024
- }
1025
- if (activity.type === 'Announce') return this.onAnnounce(activity, actor, objectId);
1026
- return;
1027
- }
1028
- case 'Delete': return this.onDelete(activity, actor);
1029
- case 'Update': return this.onUpdate(activity, actor);
1030
- case 'Reject': return this.onReject(activity, actor, { trusted });
1031
- case 'Move': return this.onMove(activity, actor);
1032
- case 'Add': case 'Remove': return this.onAddRemove(activity, actor);
1033
- default: this.log(`ignored ${activity.type} from ${actor}`);
1034
- }
1035
- }
1036
-
1037
- // Which inbound activities count as a moderator's ask: a ban, an unban, a
1038
- // post removal, or a roster change naming OUR moderators collection. A
1039
- // moderator's ordinary traffic (their posts, likes, follows) is not
1040
- // moderation and takes the normal arms.
1041
- isModerationAsk(activity) {
1042
- if (activity.type === 'Block') return true;
1043
- if (activity.type === 'Undo') {
1044
- return typeof activity.object === 'object' && activity.object?.type === 'Block';
1045
- }
1046
- if (activity.type === 'Delete') {
1047
- const id = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1048
- const s = id && this.store.getStatuses().find(x => x.noteId === id);
1049
- // Only a post the group holds and did not author — removing those is
1050
- // moderation; everything else is the author's own Delete.
1051
- return !!s && s.kind !== 'post';
1052
- }
1053
- if (activity.type === 'Add' || activity.type === 'Remove') {
1054
- const target = typeof activity.target === 'string' ? activity.target : activity.target?.id;
1055
- return target === this.urls.moderators;
1056
- }
1057
- return false;
1058
- }
1059
-
1060
- // Held, not run: one entry per distinct ask, capped, waiting for the
1061
- // operator to apply or dismiss it (social.applyModeration).
1062
- // A moderator's WORD, not their proof. `actor` is a field in an unsigned
1063
- // body and a moderator's URL is public, so anyone can claim to be one — which
1064
- // is exactly why these are QUEUED for the operator rather than run. What was
1065
- // missing is that the queue did not say which is which, and a stranger could
1066
- // fill all 200 slots and push the real asks out.
1067
- //
1068
- // So: the entry records whether the door vouched for the sender, and when the
1069
- // queue is full the UNVERIFIED entries are what get dropped. A real
1070
- // moderator's ask cannot be crowded out by someone impersonating them.
1071
- queueModeration(activity, actor, { trusted = false } = {}) {
1072
- const q = this.store.read('modqueue.json', []);
1073
- const objectId = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1074
- const key = [activity.type, actor, objectId || JSON.stringify(activity.object || null)].join(' ');
1075
- const seen = q.find(e => e.key === key);
1076
- if (seen) {
1077
- // The same ask arriving verified is worth more than the copy we hold.
1078
- if (trusted && !seen.verified) {
1079
- seen.verified = true;
1080
- this.store.write('modqueue.json', q);
1081
- }
1082
- return;
1083
- }
1084
- q.unshift({
1085
- key, id: (this.serial++).toString(36) + '-' + q.length,
1086
- type: activity.type, moderator: actor, activity: trimActivity(activity),
1087
- verified: !!trusted, at: new Date().toISOString(),
1088
- });
1089
- let kept = q;
1090
- if (kept.length > MAX_MODQUEUE) {
1091
- const verified = kept.filter(e => e.verified);
1092
- const rest = kept.filter(e => !e.verified);
1093
- // Verified first, then the newest unverified up to the cap.
1094
- kept = [...verified, ...rest].slice(0, MAX_MODQUEUE);
1095
- }
1096
- this.store.write('modqueue.json', kept);
1097
- this.log(`moderation queued from ${actor}${trusted ? '' : ' (unverified)'}: ${activity.type} ${objectId || ''}`);
1098
- }
1099
-
1100
- // §7.6 Add / §7.9 Remove. The side effect would be to add or remove the object
1101
- // to/from the collection named in `target` — but only a collection we own AND
1102
- // that the sender is authorised to modify. No remote is granted write to our
1103
- // collections (membership is Follow/Undo, pins are ours to set), so there is
1104
- // nothing an inbound Add or Remove may change here. It is a valid activity,
1105
- // not garbage: acknowledge it, make no change, and never dead-letter it.
1106
- onAddRemove(activity, actor) {
1107
- const target = typeof activity.target === 'string' ? activity.target : activity.target?.id;
1108
- const ours = target && [this.urls.followers, this.urls.following, this.urls.featured]
1109
- .filter(Boolean).includes(target);
1110
- this.log(ours
1111
- ? `${activity.type} from ${actor} targets our ${target} — no remote may modify it; acknowledged`
1112
- : `${activity.type} from ${actor} targets ${target || 'no collection of ours'}; nothing here to change`);
1113
- return; // accepted, no side effect
1114
- }
1115
-
1116
- async onFollow(activity, actor, { trusted = false } = {}) {
1117
- const doc = await this.fetchAP(actor); // origin must vouch for the actor
1118
- if (!doc) return `actor fetch failed (${actor})`;
1119
- if (doc.id !== actor) return `actor id mismatch (${actor} vs ${doc.id})`;
1120
- if (!doc.inbox) return `actor has no inbox (${actor})`;
1121
- const contacts = this.store.getContacts();
1122
- const existing = contacts.followers.find(f => f.actor === actor);
1123
- // NOTHING binds a delivered Follow to the actor it names. LDN bodies carry
1124
- // no signature, and unlike Create, Delete and Update there is no object at
1125
- // the origin to re-fetch and compare — dereferencing the actor proves only
1126
- // that the actor EXISTS. So anyone at all could Append a Follow naming
1127
- // anyone at all, and we would sign an Accept, deliver it to that person,
1128
- // and send them everything published from then on.
1129
- //
1130
- // Until deliveries terminate somewhere their signature survives, a follow
1131
- // we cannot verify is a REQUEST, waiting in the same queue a gated group
1132
- // uses. The requester's client shows "Requested", which is the ordinary
1133
- // locked-account state that manuallyApprovesFollowers tells it to expect.
1134
- // `autoAcceptFollows: true` in config restores the old behaviour.
1135
- // A GROUP is left alone: `approveJoins: false` is its operator saying, in
1136
- // as many words, that anyone may join, and mute/eject are the remedy there.
1137
- // A person has no such setting, so this is their default.
1138
- //
1139
- // A gateway-verified Follow (trust mode) is no longer unverifiable — the
1140
- // door proved the sender — so it does not need the OK that unverifiability
1141
- // alone demanded. An explicit `approveJoins` still holds: verified or not,
1142
- // the operator asked to see joins.
1143
- const unverifiedNeedsOk = this.config.kind !== 'group' && !this.config.autoAcceptFollows && !trusted;
1144
- const mustApprove = this.config.approveJoins || unverifiedNeedsOk;
1145
- if (mustApprove && !existing) {
1146
- const reqs = this.store.getRequests();
1147
- if (!reqs.some(r => r.actor === actor)) {
1148
- reqs.unshift({
1149
- actor, inbox: doc.inbox, sharedInbox: doc.endpoints?.sharedInbox,
1150
- activity: trimActivity(activity), at: new Date().toISOString(),
1151
- });
1152
- this.store.setRequests(reqs.slice(0, 500));
1153
- this.store.addNotification({ type: 'follow-request', actor });
1154
- await this.republish({ pending: true });
1155
- }
1156
- this.log(`join requested: ${actor}`);
1157
- return;
1158
- }
1159
- if (existing) {
1160
- // Deliberately NOT updating followId. An inbound Follow is unverifiable —
1161
- // that is what the queue above exists for — so letting one rewrite the id
1162
- // of a follower we already hold hands an attacker the exact value onUndo
1163
- // matches on: POST a Follow naming any follower in the published
1164
- // collection, then POST an Undo naming the id you just chose, and they are
1165
- // gone permanently. A genuine refollow needs nothing from us but the
1166
- // Accept below, which is idempotent.
1167
- } else {
1168
- // A Bluesky member who bridges later arrives here as a second, different
1169
- // actor: the bridge follows on their behalf from bsky.brid.gy/ap/<did>,
1170
- // while the native join is recorded under bsky.app/profile/<did>. Left
1171
- // alone that is one person listed twice, carried twice, and ejectable
1172
- // only half at a time. The bridged record supersedes the native one —
1173
- // it reaches the fediverse side, which the native one never could.
1174
- const bridgedDid = /^https:\/\/bsky\.brid\.gy\/ap\/(did:[^/]+)$/.exec(actor)?.[1];
1175
- if (bridgedDid) {
1176
- const before = contacts.followers.length;
1177
- contacts.followers = contacts.followers.filter(f => f.bsky?.did !== bridgedDid);
1178
- if (contacts.followers.length < before) {
1179
- this.log(`bluesky member ${bridgedDid} is bridged now — the native record gives way to it`);
1180
- }
1181
- }
1182
- contacts.followers.push({
1183
- actor, inbox: doc.inbox, sharedInbox: doc.endpoints?.sharedInbox, followId: activity.id,
1184
- ...(bridgedDid ? { bsky: { did: bridgedDid, bridged: true } } : {}),
1185
- });
1186
- this.store.setContacts(contacts);
1187
- this.store.addNotification({ type: 'follow', actor });
1188
- await this.republish({ followers: true });
1189
- this.log(`new follower: ${actor}`);
1190
- }
1191
- const { acceptActivity } = await import('./wire.mjs');
1192
- await this.deliverer.deliver(doc.inbox,
1193
- acceptActivity({ urls: this.urls, followActivity: activity, serial: this.serial++ }));
1194
- this.log(`Accept sent → ${doc.inbox}`);
1195
- }
1196
-
1197
- async onUndo(activity, actor, { trusted = false } = {}) {
1198
- // AS2 allows `object` to be a bare IRI, and that IRI is exactly the Follow
1199
- // id we stored. Reading `.type` off a string gives undefined, so the whole
1200
- // Undo was dropped — silently, since handle() reads that as handled, so no
1201
- // dead letter was kept and the item was DELETEd. The follower stayed, we
1202
- // kept delivering to them, and their server had recorded the unfollow as
1203
- // done and would never resend. Only a TYPED non-Follow is not ours.
1204
- if (typeof activity.object === 'object' && activity.object?.type
1205
- && activity.object.type !== 'Follow') return;
1206
- // And it must NAME something. Widening the type test to admit a bare IRI
1207
- // also admitted `object: undefined`, `null` and `{}` — which land on the
1208
- // no-followId carve-out below and evict, which is the very hole the
1209
- // followId check was added to close. An Undo that identifies nothing is
1210
- // not an Undo of ours.
1211
- const named = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1212
- // A gateway-verified Undo need not name a stored id — the door proved the
1213
- // sender, so an Undo{Follow} that names us as its object is enough. An
1214
- // unverified one must still identify something (the eviction-hole guard).
1215
- if (!named && !trusted) return;
1216
- // Deliveries arrive unordered: an Undo may land AFTER the refollow it
1217
- // predates. It names the Follow id it revokes — only honor it when it
1218
- // matches the follow we currently hold for that actor.
1219
- const undoneId = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1220
- const contacts = this.store.getContacts();
1221
- const rec = contacts.followers.find(f => f.actor === actor);
1222
- // Withdrawing a request that was never answered: drop it, or it sits in the
1223
- // operator's queue forever asking about someone who left.
1224
- if (!rec) {
1225
- const reqs = this.store.getRequests();
1226
- const pending = reqs.find(r => r.actor === actor);
1227
- if (pending) {
1228
- // Bound the same way the follower eviction below is, and for the same
1229
- // reason: `actor` is a field in an unsigned body, so without a match
1230
- // anyone could withdraw anyone else's waiting request. The victim's
1231
- // server believes the Follow is still pending and never resends, so
1232
- // they simply never get followed and nobody sees why.
1233
- const theirs = pending.activity?.id;
1234
- if (!trusted && (!theirs || named !== theirs)) {
1235
- this.log(`Undo from ${actor} does not name the request we hold — ignored`);
1236
- return;
1237
- }
1238
- this.store.setRequests(reqs.filter(r => r.actor !== actor));
1239
- await this.republish({ pending: true });
1240
- this.log(`join request withdrawn: ${actor}`);
1241
- }
1242
- return;
1243
- }
1244
- // An Undo must NAME the Follow it revokes, and name the one we hold.
1245
- //
1246
- // The follow id is the ONLY thing binding an Undo to the follower. LDN
1247
- // bodies carry no signature, and unlike every other inbound type this path
1248
- // dereferences nothing, so there is no origin to disagree. Matching works
1249
- // because the id was chosen by their server and delivered in a Follow we
1250
- // accepted: we publish the followers collection, but never the ids.
1251
- //
1252
- // Which means a record with NO id cannot be matched at all — and the
1253
- // carve-out that used to let those through turned "we cannot tell" into
1254
- // "anyone may evict". reconcileFollowers writes exactly such records when a
1255
- // restored machine recovers its followers from the pod, so after a restore
1256
- // every follower could be removed by one unauthenticated POST, permanently:
1257
- // dropFollower leaves a mark and the next reconcile will not bring them
1258
- // back, their server recorded no unfollow so it never resends, and neither
1259
- // side has anything to notice.
1260
- //
1261
- // Unmatchable is refused now. The cost is a follower who really did leave
1262
- // staying on the list until the operator ejects them, which is the right way
1263
- // round: `eject` is one command, and the alternative was silent, permanent,
1264
- // and available to anyone.
1265
- // A gateway-verified Undo carries the sender's proof, so it is honored on
1266
- // its own — the followId match below exists only because an UNVERIFIED Undo
1267
- // is otherwise unbindable. A verified one needs no such crutch.
1268
- if (trusted) {
1269
- dropFollower(contacts, actor, 'undo-follow');
1270
- this.store.setContacts(contacts);
1271
- await this.republish({ followers: true });
1272
- this.log(`unfollowed by ${actor} (gateway-verified)`);
1273
- return;
1274
- }
1275
- if (!rec.followId) {
1276
- this.log(`Undo from ${actor} cannot be matched — this follower was `
1277
- + `${rec.recovered ? 'recovered from the pod' : 'recorded before follow ids were kept'}, `
1278
- + `so its follow id is unknown. Ignored; \`fedipod eject ${actor}\` if they did leave.`);
1279
- return;
1280
- }
1281
- if (undoneId !== rec.followId) {
1282
- this.log(`Undo from ${actor} does not name the follow we hold `
1283
- + `(revokes ${undoneId || 'nothing'}, current is ${rec.followId}) — ignored`);
1284
- return;
1285
- }
1286
- dropFollower(contacts, actor, 'undo-follow');
1287
- this.store.setContacts(contacts);
1288
- await this.republish({ followers: true });
1289
- this.log(`unfollowed by ${actor}`);
1290
- }
1291
-
1292
- // Does this activity/note concern us at all? Either it comes from someone
1293
- // we follow, or it names us (to/cc, mention tag) or replies to one of our
1294
- // notes. Anything else is a stranger blasting inboxes — refuse it before
1295
- // spending a dereference on it.
1296
- concernsUs(doc, actor) {
1297
- if (this.store.getContacts().following.some(f => f.actor === actor && f.accepted)) return true;
1298
- const audience = []
1299
- .concat(doc?.to || [], doc?.cc || [], doc?.bto || [], doc?.bcc || [], doc?.audience || [])
1300
- .map(v => (typeof v === 'string' ? v : v?.id)).filter(Boolean);
1301
- if (audience.includes(this.urls.actor) || audience.includes(this.urls.followers)) return true;
1302
- const tagged = [].concat(doc?.tag || [])
1303
- .some(t => t?.type === 'Mention' && (t.href === this.urls.actor || t.name?.includes(this.urls.actor)));
1304
- if (tagged) return true;
1305
- const inReplyTo = typeof doc?.inReplyTo === 'string' ? doc.inReplyTo : doc?.inReplyTo?.id;
1306
- if (!inReplyTo) return false;
1307
- if (String(inReplyTo).startsWith(this.urls.notes)) return true;
1308
- // A group also owns the conversation under anything it carried. Without
1309
- // this, a reply that lost the group's mention on its way round the
1310
- // fediverse is refused, and the thread breaks for everyone who was only
1311
- // ever following the group.
1312
- return this.config.kind === 'group'
1313
- && this.store.getStatuses().some(s => s.noteId === String(inReplyTo));
1314
- }
1315
-
1316
- // §7.1.2 Forwarding from the inbox. A reply into one of our threads reaches
1317
- // only the servers the replier's server chose to deliver to — never our
1318
- // followers on servers it has never heard of. As the actor those followers
1319
- // follow, WE close that gap: an activity addressed to our followers collection
1320
- // that names one of our objects is re-delivered to our followers' inboxes.
1321
- //
1322
- // Only what was addressed to the followers COLLECTION is carried — bto/bcc are
1323
- // never read here, so a direct message (addressed to a person) never qualifies
1324
- // and is never rebroadcast. And this runs only after handle() accepted the
1325
- // activity, so anything blocked or muted was already refused upstream and is
1326
- // never forwarded.
1327
- async _maybeForward(activity) {
1328
- if (!activity || typeof activity !== 'object') return;
1329
- if (!FORWARDABLE.has(activity.type)) return; // see FORWARDABLE
1330
- // Forwarding signs somebody else's activity with OUR key and pushes it to
1331
- // every follower we have. That is a lot to do on the word of an unsigned
1332
- // POST, so it is narrowed three ways:
1333
- //
1334
- // 1. Only the types whose object this drain actually DEREFERENCED at the
1335
- // author's own origin. A Create/Update/Delete went through onCreate /
1336
- // onUpdate / onDelete, which fetch and check the author; a Like, an
1337
- // Announce or an Undo is believed on the envelope alone, so relaying
1338
- // one made us a signed relay for anything a stranger cared to write.
1339
- // 2. The actor has to be someone we already know of. A complete stranger
1340
- // addressing our followers collection is not a conversation we are
1341
- // party to.
1342
- // 3. A budget per drain, so a flood cannot turn one sweep into thousands
1343
- // of outbound deliveries under our signature.
1344
- if (!FORWARD_TYPES.has(activity.type)) return;
1345
- try {
1346
- const audience = []
1347
- .concat(activity.to || [], activity.cc || [], activity.audience || [])
1348
- .map(v => (typeof v === 'string' ? v : v?.id)).filter(Boolean);
1349
- if (!audience.includes(this.urls.followers)) return; // not for our followers
1350
- if (!this._referencesOurObject(activity)) return; // not into a thread of ours
1351
- const actor = typeof activity.actor === 'string' ? activity.actor : activity.actor?.id;
1352
- if (actor === this.urls.actor) return; // our own; nothing to forward
1353
- if (!this.known(actor)) {
1354
- this.log(`not forwarding ${activity.type} from ${actor}: nobody we know of`);
1355
- return;
1356
- }
1357
- if (this._forwardBudget <= 0) {
1358
- this.log(`not forwarding ${activity.type}: this drain's forwarding budget is spent`);
1359
- return;
1360
- }
1361
- const id = typeof activity.id === 'string' ? activity.id : null;
1362
- if (!id) return;
1363
- const forwarded = this.store.read('forwarded.json', []);
1364
- if (forwarded.includes(id)) return; // already done
1365
-
1366
- const inboxes = [...new Set(this.store.getContacts().followers
1367
- .filter(f => !f.bsky) // Bluesky members are not AP inboxes
1368
- .map(f => f.sharedInbox || f.inbox)
1369
- .filter(Boolean))];
1370
- if (!inboxes.length) return;
1371
- this._forwardBudget -= 1;
1372
- await this.deliverer.deliverToAll(inboxes, activity);
1373
- this.store.write('forwarded.json', [...forwarded, id].slice(-MAX_FORWARDED));
1374
- this.log(`forwarded ${activity.type} ${id} to ${inboxes.length} follower inbox(es)`);
1375
- } catch (e) {
1376
- this.log(`inbox forwarding: ${e.message}`); // never stalls the drain
1377
- }
1378
- }
1379
-
1380
- // The "objects owned by the server" half of §7.1.2: does the activity reply
1381
- // to, like, boost or otherwise name one of our own objects?
1382
- _referencesOurObject(activity) {
1383
- const refs = [];
1384
- const add = (v) => { const id = typeof v === 'string' ? v : v?.id; if (id) refs.push(String(id)); };
1385
- const obj = activity.object;
1386
- if (obj && typeof obj === 'object') add(obj.inReplyTo); // a reply's parent
1387
- add(activity.inReplyTo);
1388
- add(activity.object); // a Like/Announce of our note
1389
- add(activity.target);
1390
- if (refs.some(r => r.startsWith(this.urls.notes))) return true;
1391
- return this.config.kind === 'group'
1392
- && refs.some(r => this.store.getStatuses().some(s => s.noteId === r));
1393
- }
1394
-
1395
- async onCreate(activity, actor) {
1396
- const objectId = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1397
- if (!objectId) return 'Create without object id';
1398
- if (this.store.isBlocked(objectId)) return `blocked domain (${objectId})`;
1399
- if (!this.sameIdentity(objectId, actor)) return `object/actor identity mismatch (${objectId})`;
1400
- // The delivered copy is untrusted for CONTENT, but its addressing is
1401
- // enough to decide whether to bother fetching the origin's copy.
1402
- const envelope = typeof activity.object === 'object' ? { ...activity, ...activity.object } : activity;
1403
- if (!this.concernsUs(envelope, actor)) return `not addressed to us (${objectId})`;
1404
- // The check onAnnounce has had all along. A re-delivered Create — a remote
1405
- // retry, a group fan-out, or our own sweep seeing an item whose DELETE was
1406
- // refused — cost a fresh signed GET to the origin and rewrote the private
1407
- // RDF note every time, because addStatus only dedupes AFTER the deref.
1408
- // Gated around the INGEST alone: a group must still reach amplify below,
1409
- // which is separately idempotent on announcedAt.
1410
- // Only a status that came from an INGEST counts as already done. TagFeed
1411
- // writes a bare `kind:'tag'` row straight into the index — no pod RDF note,
1412
- // no mention notification, no replies-collection entry — so treating that
1413
- // as ingested loses all three when the same note is then delivered to us.
1414
- const ingested = this.store.getStatuses()
1415
- .some(x => x.noteId === objectId && (x.kind === 'timeline' || x.kind === 'mention'));
1416
- if (!ingested) {
1417
- const rejected = await this.ingestNote(objectId, actor);
1418
- if (rejected) return rejected;
1419
- }
1420
- // A group carries its members' posts onward. Only reached from Create, so an
1421
- // inbound Announce is never re-announced. The activity is passed through
1422
- // untouched — FEP-1b12 wants the original wrapped, not a summary of it.
1423
- if (this.config.kind === 'group') await this.amplify(objectId, { activity });
1424
- }
1425
-
1426
- // Anyone can Append to a public inbox, so arriving is not the same as being
1427
- // carried to every follower. Membership is the gate: you cannot post to a
1428
- // group you have not joined, and declining to carry a member is the only
1429
- // moderation a group can actually enforce.
1430
- async amplify(noteId, { approved = false, activity = null } = {}) {
1431
- const s = this.store.getStatuses().find(x => x.noteId === noteId);
1432
- if (!s) return;
1433
- if (s.announcedAt) return; // a re-delivered Create announces once
1434
- // A DM to the group, or a followers-only post it happened to receive, was
1435
- // addressed to less than the world — carrying it would widen the author's
1436
- // audience for them. A group only ever amplifies public posts.
1437
- if (s.direct || s.nonPublic) {
1438
- this.log(`not amplified — ${noteId} was not addressed publicly, and a group never widens a post's audience`);
1439
- return;
1440
- }
1441
- const contacts = this.store.getContacts();
1442
- if (!contacts.followers.some(f => f.actor === s.actor)) {
1443
- this.log(`not amplified — ${s.actor} is not a member`);
1444
- return;
1445
- }
1446
- if (this.store.getMuted().actors.includes(s.actor)) {
1447
- this.log(`not amplified — ${s.actor} is muted`);
1448
- return;
1449
- }
1450
- // A reviewed group carries nothing until its operator says so.
1451
- if (this.config.review && !approved) {
1452
- const pending = this.store.getPending();
1453
- if (!pending.some(p => p.noteId === noteId)) {
1454
- // Full means refuse the new one, not evict the oldest. `slice(0, 500)`
1455
- // dropped from the tail, so one member posting 500 notes silently
1456
- // discarded everything the operator was still deciding about — the
1457
- // posts were never carried, never refused, and left no record that they
1458
- // had ever arrived. Becoming a member costs one Follow when joins are
1459
- // unmoderated, which is the default.
1460
- //
1461
- // Not carrying it is what a reviewed group does with anything it has
1462
- // not approved, so refusing is the same outcome the queue was for.
1463
- if (pending.length >= MAX_PENDING_REVIEW) {
1464
- this.log(`review queue is full (${MAX_PENDING_REVIEW}) — ${noteId} not held. `
1465
- + 'Approve or decline what is waiting and it will be carried on redelivery.');
1466
- return;
1467
- }
1468
- // The activity rides along: approving later still has to wrap the one
1469
- // the member actually sent, not a reconstruction of it.
1470
- pending.unshift({ noteId, actor: s.actor, activity, at: new Date().toISOString() });
1471
- this.store.setPending(pending);
1472
- }
1473
- this.log(`held for review: ${noteId}`);
1474
- return;
1475
- }
1476
- // A member's Bluesky post: the carry is a native repost by the group's
1477
- // account. It reaches AP followers only through the author's own bridge —
1478
- // the group never fabricates an AP object for someone else's words.
1479
- if (s.kind === 'bsky') {
1480
- if (!this.bskyGroup) { this.log(`not amplified — ${noteId} is a bluesky post and no account is connected`); return; }
1481
- return this.bskyGroup.carry(s);
1482
- }
1483
- const held = this.store.getPending().find(p => p.noteId === noteId);
1484
- const inboxes = this.announceTargets(s.actor);
1485
- const { announceActivity } = await import('./wire.mjs');
1486
- // Wrap the member's own activity when we have it; a bare note URL is the
1487
- // fallback, and renders as a plain boost rather than a group carry. The
1488
- // group names itself as the audience (FEP-1b12).
1489
- //
1490
- // `activity` is the envelope as DELIVERED — a document the sender wrote,
1491
- // which the group would otherwise re-sign and hand to every follower with
1492
- // whatever addressing, tags and object body it carried. Only the note id was
1493
- // ever verified (ingestNote fetched it from the author's origin and checked
1494
- // the attribution), so only the note id is safe to pass on: send the bare
1495
- // id unless the wrapper's own object id agrees with what we verified.
1496
- const wrapperObject = (a) => {
1497
- const inner = a?.object;
1498
- const id = typeof inner === 'string' ? inner : inner?.id;
1499
- return id === noteId ? a : null;
1500
- };
1501
- const act = announceActivity({
1502
- urls: this.urls,
1503
- object: wrapperObject(activity) || wrapperObject(held?.activity) || noteId,
1504
- serial: this.serial++,
1505
- audience: this.urls.actor,
1506
- });
1507
- await this.deliverer.deliverToAll(inboxes, act);
1508
- // Marked carried before recorded: a failed outbox write costs one missing
1509
- // entry, a failed status write would carry the same post twice.
1510
- this.store.updateStatus(noteId, { announcedAt: new Date().toISOString(), announceActivity: act });
1511
- await this.publisher.recordOutbox(act);
1512
- this.store.setPending(this.store.getPending().filter(p => p.noteId !== noteId));
1513
- this.log(`amplified ${noteId} → ${inboxes.length} inbox(es)`);
1514
- // The same carry, shown natively to the group's Bluesky followers.
1515
- await this.bskyGroup?.mirrorCarry(s)
1516
- .catch(e => this.log(`bluesky mirror of the carry failed: ${e.message}`));
1517
- }
1518
-
1519
- // Is this actor in a group we are in? Each followed Group's membership is a
1520
- // public collection, read at most once a day and cached — a membership list
1521
- // is slow-moving, and this runs on arriving mail.
1522
- async isCoMember(actor) {
1523
- if (this.config.kind === 'group') return false; // a group has members, not peers
1524
- const groups = this.store.getContacts().following
1525
- .filter(f => f.accepted && this.store.getActors()[f.actor]?.type === 'Group')
1526
- .map(f => f.actor);
1527
- if (!groups.length) return false;
1528
- const cache = this.store.read('comembers.json', {});
1529
- const fresh = Date.now() - CO_MEMBER_TTL_MS;
1530
- let changed = false;
1531
- for (const g of groups) {
1532
- const held = cache[g];
1533
- if (held && Date.parse(held.at || 0) > fresh) continue;
1534
- const doc = await this.fetchAP(g).catch(() => null);
1535
- const list = doc?.followers ? await this.collectionMembers(doc.followers) : null;
1536
- // A list we could not read keeps whatever we had: losing it would demote
1537
- // every co-member to a stranger for a day because one fetch failed.
1538
- if (!list) continue;
1539
- cache[g] = { at: new Date().toISOString(), members: list };
1540
- changed = true;
1541
- }
1542
- if (changed) this.store.write('comembers.json', cache);
1543
- return groups.some(g => cache[g]?.members?.includes(actor));
1544
- }
1545
-
1546
- // The actor ids in a (possibly paged) public collection, capped.
1547
- async collectionMembers(url) {
1548
- const out = [];
1549
- let next = url;
1550
- for (let page = 0; next && page < 10 && out.length < CO_MEMBER_MAX; page++) {
1551
- const doc = await this.fetchAP(next).catch(() => null);
1552
- if (!doc) return out.length ? out : null;
1553
- for (const item of doc.orderedItems || doc.items || []) {
1554
- if (typeof item === 'string') out.push(item);
1555
- }
1556
- next = doc.first && page === 0 ? doc.first : doc.next;
1557
- if (typeof next === 'object') next = next?.id;
1558
- }
1559
- return out;
1560
- }
1561
-
1562
- // Who an Announce for `author` goes to. Shared with the retract path: an Undo
1563
- // that reached a different set than the Announce did would leave the post
1564
- // standing for whoever the two sets disagreed about.
1565
- // The author's own target is dropped only when it serves nobody else — a
1566
- // shared inbox carries the whole server's members.
1567
- announceTargets(author) {
1568
- const byTarget = new Map();
1569
- for (const f of this.store.getContacts().followers) {
1570
- const t = f.sharedInbox || f.inbox;
1571
- if (!t) continue;
1572
- if (!byTarget.has(t)) byTarget.set(t, new Set());
1573
- byTarget.get(t).add(f.actor);
1574
- }
1575
- return [...byTarget]
1576
- .filter(([, who]) => !(who.size === 1 && who.has(author)))
1577
- .map(([t]) => t);
1578
- }
1579
-
1580
- // A boost: ingest the boosted note when the booster is someone we follow —
1581
- // that's what following means, their boosts widen the timeline. Anything
1582
- // else is unsolicited and only logged.
1583
- // A group we follow announces a Delete: the carrier moderating away a post
1584
- // it carried. Honored only within what the carry itself established — the
1585
- // announcer is a group we follow AND the post reached us via that same
1586
- // group — so no new party is trusted and nothing is dereferenced. Our own
1587
- // posts are never removed by anyone's moderation.
1588
- async onAnnouncedDelete(actor, del) {
1589
- const followed = this.store.getContacts().following.some(f => f.actor === actor && f.accepted);
1590
- if (!followed) { this.log(`announced Delete from unfollowed ${actor} — ignored`); return; }
1591
- const targetId = typeof del.object === 'string' ? del.object : del.object?.id;
1592
- if (!targetId) return 'announced Delete without an object';
1593
- const s = this.store.getStatuses().find(x => x.noteId === targetId);
1594
- if (!s || s.kind === 'post' || s.via !== actor) return;
1595
- await this.forget(s);
1596
- this.log(`moderated away by ${actor}: ${targetId}`);
1597
- }
1598
-
1599
- async onAnnounce(activity, actor, objectId) {
1600
- if (!objectId) return 'Announce without object id';
1601
- const followed = this.store.getContacts().following.some(f => f.actor === actor && f.accepted);
1602
- if (!followed) { this.log(`Announce from unfollowed ${actor} — ignored`); return; }
1603
- if (this.store.isBlocked(objectId)) return `blocked domain (${objectId})`;
1604
- const existing = this.store.getStatuses().find(s => s.noteId === objectId);
1605
- if (existing) {
1606
- // Known, but possibly as a lesser kind — a stranger's mention, a tag or
1607
- // search mirror — none of which the home timeline shows. A carry from
1608
- // someone we follow is exactly what promotes it there.
1609
- if (!['timeline', 'post'].includes(existing.kind)) {
1610
- this.store.updateStatus(objectId, { kind: 'timeline', via: actor });
1611
- this.log(`promoted to timeline (carried by ${actor}): ${objectId}`);
1612
- }
1613
- return;
1614
- }
1615
- return this.ingestNote(objectId, actor, { via: actor });
1616
- }
1617
-
1618
- // Shared tail of Create/Announce: deref the note at its origin (never trust
1619
- // the delivered copy), mirror it into pod RDF + statuses, notify on replies
1620
- // to our own notes. Returns a rejection reason string, or undefined.
1621
- async ingestNote(objectId, actor, { via } = {}) {
1622
- const note = await this.fetchAP(objectId);
1623
- if (!note) return `object fetch failed (${objectId})`;
1624
- if (note.id !== objectId || !isContentType(note.type)) return `object not verifiable content (${objectId}, ${note.type})`;
1625
- const { attachmentsOf, titledContent } = await import('./wire.mjs');
1626
- const attachments = attachmentsOf(note);
1627
- const content = titledContent(note); // hostile markup never reaches pod or client
1628
- // The delivering actor was checked on arrival; the author is only known once
1629
- // the note is dereferenced. authorOf refuses an author the note's own origin
1630
- // does not vouch for — see its comment; this is where a forged attribution
1631
- // would otherwise become a timeline entry, a pod document, and for a group a
1632
- // signed Announce to every member.
1633
- const author = authorOf(note, actor);
1634
- if (!author) return `object names an author its origin does not vouch for (${objectId})`;
1635
- // This is the check that catches a blocked actor reaching us through
1636
- // somebody else's boost, or through a hashtag feed.
1637
- if (this.store.isBlocked(author)) return `blocked author (${author})`;
1638
-
1639
- // The ENVELOPE said this concerns us. The envelope is a document a stranger
1640
- // wrote: onCreate reads addressing off the delivered copy to decide whether
1641
- // fetching is worth it, and nothing re-asked the question of the copy that
1642
- // came back from the author's own server. So anyone could take any public
1643
- // post, address the delivery to us, and have it filed as "X mentioned you"
1644
- // — or, to a group, have a member's post carried to every follower when the
1645
- // member never sent it there.
1646
- //
1647
- // A boost is exempt (`via`): someone we follow deliberately putting a post
1648
- // in front of us is the whole point, and the note will not address us.
1649
- if (!via && !this.concernsUs(note, author)) {
1650
- return `the note its own server serves does not address us (${objectId})`;
1651
- }
1652
-
1653
- // An answer to one of our polls is a number on a document, not a post. It
1654
- // arrives as an ordinary reply naming an option and carrying nothing else,
1655
- // so filing it as one would put a blank entry in the thread and ring the
1656
- // owner once per voter. Counted or refused — a second answer, an option we
1657
- // do not offer, a poll already shut — it stops here either way.
1658
- const asked = note.inReplyTo && this.store.getStatuses()
1659
- .find(x => x.noteId === String(note.inReplyTo) && x.kind === 'post' && x.poll);
1660
- if (asked && polls.isVoteShape(note)) {
1661
- const counted = await this.publisher.recordVote(asked.noteId, author, note.name)
1662
- .catch(e => { this.log(`vote on ${asked.noteId}: ${e.message}`); return false; });
1663
- this.log(counted
1664
- ? `vote counted (${note.name}): ${asked.noteId}`
1665
- : `vote not counted (${note.name}) from ${author}: ${asked.noteId}`);
1666
- return;
1667
- }
1668
-
1669
- // Anyone can Append to a public inbox, so arriving is not the same as
1670
- // belonging in the home timeline. Follow Mastodon's split: people you
1671
- // follow (and their boosts) are HOME; anyone else is a MENTION — kept,
1672
- // notified, readable in the Mentions view, but out of the timeline, and
1673
- // mirror-only so unsolicited content never accumulates in the pod.
1674
- // A group's people are its FOLLOWERS — it follows nobody. Reading the
1675
- // following list for one filed every member's post as a stranger's mention,
1676
- // so nothing reached the pod RDF and each post raised a notification.
1677
- const contacts = this.store.getContacts();
1678
- const known = this.config.kind === 'group'
1679
- ? contacts.followers.some(f => f.actor === author)
1680
- : contacts.following.some(f => f.actor === author && f.accepted);
1681
- // Someone in a group you are in is not a stranger: their reply belongs in
1682
- // the room, not in the drawer of unsolicited mail. Whose word this is on
1683
- // is the group's — its published membership — so the group's own door
1684
- // decides who gets in.
1685
- const followed = via || known || (!known && await this.isCoMember(author));
1686
- const kind = followed ? 'timeline' : 'mention';
1687
-
1688
- // Mastodon carries a thread's mentions into every reply, which is the only
1689
- // reason a reply ever reaches a group. Keep them so our composer can too.
1690
- const mentions = [].concat(note.tag || [])
1691
- .filter(t => t?.type === 'Mention' && t.href && t.name)
1692
- .slice(0, MAX_MENTIONS)
1693
- .map(t => ({ href: httpOnly(String(t.href).slice(0, MAX_URL_CHARS)), name: String(t.name).slice(0, 256) }))
1694
- .filter(m => m.href);
1695
- const emojis = emojisOf(note);
1696
- const poll = pollOf(note);
1697
- // Explicitly addressed, but to nobody public and to no followers
1698
- // collection: a direct message, which belongs to the conversations view
1699
- // rather than a timeline. A note with no addressing at all is NOT direct —
1700
- // some servers omit to/cc, and vanishing from home is the wrong reading.
1701
- const audience = [].concat(note.to || [], note.cc || []).map(String);
1702
- const direct = audience.length > 0
1703
- && !audience.includes(PUBLIC) && !audience.some(a => a.endsWith('/followers'));
1704
- // Addressed to less than the world: whatever else happens to it, a group
1705
- // must never widen its audience by carrying it.
1706
- const nonPublic = audience.length > 0 && !audience.includes(PUBLIC);
1707
- this.store.addStatus({
1708
- noteId: note.id, actor: author, content,
1709
- published: note.published, inReplyTo: note.inReplyTo, kind,
1710
- ...(direct ? { direct: true } : {}),
1711
- ...(nonPublic ? { nonPublic: true } : {}),
1712
- // The author's content warning, shown as one: plain text only.
1713
- ...(note.summary ? { spoiler: String(note.summary).replace(/<[^>]*>/g, '') } : {}),
1714
- ...(poll ? { poll } : {}),
1715
- ...(emojis.length ? { emojis } : {}),
1716
- ...(mentions.length ? { mentions } : {}),
1717
- ...(attachments.length ? { attachments } : {}),
1718
- ...(via ? { via } : {}),
1719
- });
1720
- if (!followed || (note.inReplyTo && String(note.inReplyTo).startsWith(this.urls.notes))) {
1721
- this.store.addNotification({ type: 'mention', actor: author, noteId: note.id });
1722
- }
1723
- if (note.inReplyTo && String(note.inReplyTo).startsWith(this.urls.notes)) {
1724
- await this.addReply(String(note.inReplyTo), note.id)
1725
- .catch(e => this.log(`replies collection: ${e.message}`));
1726
- }
1727
- this.log(`${kind}: ${note.id}${via ? ` (boosted by ${via})` : ''}`);
1728
- }
1729
-
1730
- // Is this really gone at its origin? true / false / null when the origin
1731
- // could not be asked. Delivered bodies carry no signature, so this is how a
1732
- // Delete is verified — the same verify-by-dereference the rest of intake uses.
1733
- async isGone(url) {
1734
- let res;
1735
- try { res = await this.deliverer.signedFetch(url, { headers: { accept: ACCEPT_AP } }); }
1736
- catch { return null; }
1737
- if (res.status === 404 || res.status === 410) return true;
1738
- if (res.status < 400) {
1739
- // A Tombstone answers 200 and still means deleted.
1740
- try {
1741
- const { readCapped } = await import('./safefetch.mjs');
1742
- return JSON.parse(await readCapped(res))?.type === 'Tombstone';
1743
- } catch { return false; }
1744
- }
1745
- return null; // 401/403/5xx — no answer, not a denial
1746
- }
1747
-
1748
- // Mastodon sends these constantly; ignoring them left deleted posts standing
1749
- // for good. Two guards, because a forged Delete would otherwise erase anyone's
1750
- // content: it must come from the object's own origin, and the object must
1751
- // really be gone there. An origin we cannot reach is a retry, never a delete.
1752
- async onDelete(activity, actor) {
1753
- const objectId = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1754
- if (!objectId) return 'Delete without object id';
1755
- if (!this.sameIdentity(objectId, actor)) return `Delete crosses identities (${objectId})`;
1756
- // `objectId === actor` is NOT evidence we care: it is true of EVERY account
1757
- // deletion, and Mastodon broadcasts those constantly. Taking it as known
1758
- // meant a signed dereference to a stranger's server for each one.
1759
- if (!this.known(objectId)) return; // nothing of ours to remove
1760
- const gone = await this.isGone(objectId);
1761
- if (gone === null) throw new Error(`cannot confirm ${objectId} is gone — will retry`);
1762
- if (!gone) return `Delete for something still published (${objectId})`;
1763
-
1764
- if (objectId === actor) { // the account itself
1765
- const contacts = this.store.getContacts();
1766
- dropFollower(contacts, actor, 'account-deleted');
1767
- contacts.following = contacts.following.filter(f => f.actor !== actor);
1768
- this.store.setContacts(contacts);
1769
- // One publish for the lot. Each forget() used to run its own
1770
- // unrecordOutbox, and each of those republished the outbox — so a group
1771
- // that had carried M of this actor's posts paid M full page sweeps for a
1772
- // single inbox item. The Undo deliveries stay per-Announce, because each
1773
- // Announce needs its own; only the pod write is collected.
1774
- const retracted = [];
1775
- for (const s of this.store.getStatuses().filter(s => s.actor === actor)) {
1776
- await this.forget(s, { collect: retracted });
1777
- }
1778
- if (retracted.length) {
1779
- const gone = new Set(retracted);
1780
- await this.publisher.unrecordOutbox(i => gone.has(i?.id));
1781
- }
1782
- // Both: an account deletion drops them from followers AND following.
1783
- await this.republish({ followers: true, following: true });
1784
- this.log(`account deleted upstream: ${actor}`);
1785
- return;
1786
- }
1787
- const s = this.store.getStatuses().find(x => x.noteId === objectId);
1788
- if (s) await this.forget(s);
1789
- this.log(`deleted upstream: ${objectId}`);
1790
- }
1791
-
1792
- // Drop a post we were holding. A group that carried it also unsays its own
1793
- // Announce — forwarding the author's Delete would be signed by us and not by
1794
- // them, which receivers are right to refuse.
1795
- // `collect` batches the outbox side: retract pushes the Announce id onto it
1796
- // instead of republishing, and the caller writes once for all of them.
1797
- async forget(s, { collect = null } = {}) {
1798
- if (s.announceActivity) {
1799
- await this.retract(s.noteId, { collect }).catch(e => this.log(`retract: ${e.message}`));
1800
- }
1801
- this.store.removeStatus(s.noteId);
1802
- }
1803
-
1804
- // Undo an Announce this group made. Shared with the operator's `retract`.
1805
- async retract(noteId, { collect = null } = {}) {
1806
- const s = this.store.getStatuses().find(x => x.noteId === noteId);
1807
- if (!s) throw new Error('no such post');
1808
- // A Bluesky carry is a repost, and unsaying it is deleting the repost.
1809
- if (s.repostUri) {
1810
- if (!this.bskyGroup) throw new Error('no bluesky account connected');
1811
- return this.bskyGroup.retract(s);
1812
- }
1813
- if (!s.announceActivity) throw new Error('that post was never carried');
1814
- const { undoActivity } = await import('./wire.mjs');
1815
- const inboxes = this.announceTargets(s.actor);
1816
- await this.deliverer.deliverToAll(inboxes,
1817
- undoActivity({ urls: this.urls, activity: s.announceActivity, serial: this.serial++ }));
1818
- if (collect) collect.push(s.announceActivity.id);
1819
- else await this.publisher.unrecordOutbox(i => i?.id === s.announceActivity.id);
1820
- this.store.updateStatus(noteId, {
1821
- announcedAt: undefined, announceActivity: undefined, retractedAt: new Date().toISOString(),
1822
- });
1823
- return { ok: true, noteId, inboxes: inboxes.length };
1824
- }
1825
-
1826
- // An edited post, or a changed profile. Verified the only way we can: by
1827
- // refetching at the origin and believing that, not the delivered copy.
1828
- async onUpdate(activity, actor) {
1829
- const objectId = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1830
- if (!objectId) return 'Update without object id';
1831
- if (!this.sameIdentity(objectId, actor)) return `Update crosses identities (${objectId})`;
1832
- if (objectId === actor) { // display name, avatar, bio
1833
- // The same guard onDelete has, for the same reason: the inbox is
1834
- // public-Append, so without it anyone can name any host and make us spend
1835
- // a signed GET on it — and because the failure below THROWS rather than
1836
- // returning a rejection, one planted item buys five of them, plus five
1837
- // pod reads and the head of the inbox held for five sweeps.
1838
- if (!this.known(actor)) return; // nothing of ours to update
1839
- const doc = await this.fetchAP(actor);
1840
- if (!doc) throw new Error(`cannot refetch ${actor} — will retry`);
1841
- this.store.cacheActor(actor, doc); // fetchAP caches Persons; Groups too
1842
- this.log(`profile updated: ${actor}`);
1843
- return;
1844
- }
1845
- const s = this.store.getStatuses().find(x => x.noteId === objectId);
1846
- if (!s) return; // not one we hold
1847
- const note = await this.fetchAP(objectId);
1848
- if (!note) throw new Error(`cannot refetch ${objectId} — will retry`);
1849
- if (note.id !== objectId || !isContentType(note.type)) return `object not verifiable content (${objectId}, ${note.type})`;
1850
- const { attachmentsOf, titledContent } = await import('./wire.mjs');
1851
- const content = titledContent(note);
1852
- const attachments = attachmentsOf(note);
1853
- const freshPoll = pollOf(note);
1854
- const freshEmojis = emojisOf(note);
1855
- this.store.updateStatus(objectId, {
1856
- content, ...(attachments.length ? { attachments } : {}),
1857
- emojis: freshEmojis.length ? freshEmojis : undefined,
1858
- // The edit's own stamp when the note carries one; tallies and the
1859
- // content warning follow the edit too. A poll refresh keeps our vote.
1860
- editedAt: note.updated || new Date().toISOString(),
1861
- spoiler: note.summary ? String(note.summary).replace(/<[^>]*>/g, '') : undefined,
1862
- ...(freshPoll ? {
1863
- poll: { ...freshPoll, voted: !!s.poll?.voted, ownVotes: s.poll?.ownVotes || [] },
1864
- } : {}),
1865
- });
1866
- this.log(`edited upstream: ${objectId}`);
1867
- }
1868
-
1869
- // Read-modify-write, and the drain is serialized, so two replies in one sweep
1870
- // do not race. Nothing else writes this document.
1871
- // MAX_REPLIES_RECORDED caps the collection. It is a discovery aid — a client
1872
- // reading a thread — and the statuses index is what actually holds the
1873
- // replies, so dropping the oldest costs a hop, not a record.
1874
- async addReply(parentId, replyId) {
1875
- // The parent has to be a post we actually made. The only check used to be
1876
- // that the id started with our notes prefix, and `inReplyTo` is read off a
1877
- // document at the sender's own origin — so a stranger could name a note we
1878
- // never wrote, and we would CREATE a document on the pod at a URL of their
1879
- // choosing and then grow it, one whole re-PUT per reply, with no cap. Four
1880
- // pod requests each and bytes quadratic in the number of replies.
1881
- if (!this.store.getStatuses().some(s => s.noteId === parentId && s.kind === 'post')) {
1882
- this.log(`reply names ${parentId}, which is not a post of ours — not recorded`);
1883
- return;
1884
- }
1885
- const { repliesId, collection } = await import('./wire.mjs');
1886
- const url = repliesId(parentId);
1887
- // Deliberately NOT caught: a read we could not make is not an empty
1888
- // collection, and rewriting on top of one erases every reply already
1889
- // recorded. getJson returns null only for a genuine 404 — the document does
1890
- // not exist yet — and throws otherwise, which the caller logs and retries.
1891
- const cur = await podNotes.readReplies(this.remote, url);
1892
- const items = Array.isArray(cur?.items) ? cur.items : [];
1893
- if (items.includes(replyId)) return;
1894
- items.push(replyId);
1895
- await podNotes.writeReplies(this.remote, url, collection(url, items.slice(-MAX_REPLIES_RECORDED)));
1896
- this.log(`reply recorded on ${parentId}`);
1897
- }
1898
-
1899
- // The other answer to a Follow, and it was dropped on the floor. Their server
1900
- // has recorded that we do not follow them; ours went on saying we did, and
1901
- // published it — so the two disagreed permanently, and a retry would never
1902
- // come because as far as they are concerned the question was answered.
1903
- //
1904
- // It has to answer the Follow we actually SENT. Only the type was checked, so
1905
- // one Append per account you follow — from anyone, naming no particular
1906
- // follow — severed every one of them at once, and silently: their server
1907
- // never hears about it, so nothing ever retries and nothing looks wrong until
1908
- // the timeline goes quiet. `followActivity` is stored by followActor
1909
- // (lib/social.mjs) for exactly this kind of comparison.
1910
- async onReject(activity, actor, { trusted = false } = {}) {
1911
- if (activity.object?.type && activity.object.type !== 'Follow') return;
1912
- const contacts = this.store.getContacts();
1913
- const rec = contacts.following.find(f => f.actor === actor);
1914
- if (!rec) return; // nothing of ours to undo
1915
- const named = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1916
- const ours = rec.followActivity?.id;
1917
- // A gateway receipt bound to this actor is the other way to believe it —
1918
- // the door checked a signature and said whose (see receiptVouchesFor).
1919
- if (!trusted) {
1920
- if (!ours) {
1921
- this.log(`Reject from ${actor}: no follow id on record to match it against — ignored`);
1922
- return;
1923
- }
1924
- if (named !== ours) {
1925
- this.log(`Reject from ${actor} answers ${named || 'nothing'}, not the follow we sent — ignored`);
1926
- return;
1927
- }
1928
- }
1929
- contacts.following = contacts.following.filter(f => f.actor !== actor);
1930
- this.store.setContacts(contacts);
1931
- await this.republish({ following: true, pending: true });
1932
- this.log(`follow rejected by ${actor}`);
1933
- }
1934
-
1935
- // Someone we follow has moved. Their server will stop delivering from the old
1936
- // actor, so without this we keep an entry that can never produce another post
1937
- // and never learn where they went. The new account is not followed
1938
- // automatically — that is a Follow only the owner should send — but it is
1939
- // recorded and raised, so it can be acted on.
1940
- async onMove(activity, actor) {
1941
- const target = typeof activity.target === 'string' ? activity.target : activity.target?.id;
1942
- if (!target) return 'Move without a target';
1943
- const contacts = this.store.getContacts();
1944
- const rec = contacts.following.find(f => f.actor === actor);
1945
- if (!rec) return; // not someone we follow
1946
- // Believed only if the actor we follow says so at its OWN origin: a Move is
1947
- // otherwise a redirect anyone could Append.
1948
- const doc = await this.fetchAP(actor);
1949
- if (!doc) throw new Error(`cannot confirm ${actor} moved — will retry`);
1950
- const movedTo = typeof doc.movedTo === 'string' ? doc.movedTo : doc.movedTo?.id;
1951
- if (movedTo !== target) return `Move not corroborated by ${actor} (says ${movedTo || 'nothing'})`;
1952
- rec.movedTo = target;
1953
- this.store.setContacts(contacts);
1954
- this.store.addNotification({ type: 'move', actor, target });
1955
- this.log(`${actor} moved to ${target} — follow the new account to keep seeing them`);
1956
- }
1957
-
1958
- async onAccept(activity, actor, { trusted = false } = {}) {
1959
- const contacts = this.store.getContacts();
1960
- const rec = contacts.following.find(f => f.actor === actor);
1961
- // It has to answer the Follow we actually sent. followActor stores that
1962
- // activity for the later Undo, so the id is here to compare against; without
1963
- // the check any Accept from an actor we happen to follow flips the flag,
1964
- // including one answering a Follow we never made.
1965
- const named = typeof activity.object === 'string' ? activity.object : activity.object?.id;
1966
- const ours = rec?.followActivity?.id;
1967
- // `named &&` used to be part of this, so an Accept naming NOTHING sailed
1968
- // past — which is the easy one to send, and it marks a request to a locked
1969
- // account as accepted when it is still sitting in their queue.
1970
- if (ours && named !== ours && !trusted) {
1971
- this.log(`Accept from ${actor} answers ${named || 'nothing'}, not the follow we sent — ignored`);
1972
- return;
1973
- }
1974
- if (rec && !rec.accepted) {
1975
- rec.accepted = true;
1976
- this.store.setContacts(contacts);
1977
- await this.republish({ following: true, pending: true });
1978
- this.log(`follow accepted by ${actor}`);
1979
- }
1980
- }
1981
- }