fedipod-server 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/README.md +22 -6
  2. package/dist/claims.d.ts +8 -0
  3. package/dist/claims.js +10 -0
  4. package/dist/handler.d.ts +13 -0
  5. package/dist/handler.js +56 -11
  6. package/dist/handler.jsonld +8 -0
  7. package/dist/store-pod.js +18 -4
  8. package/lib/{c2s.mjs → client/c2s.mjs} +10 -3
  9. package/lib/{localapi.mjs → client/localapi.mjs} +2 -2
  10. package/lib/client/masto/accounts.mjs +264 -0
  11. package/lib/client/masto/body.mjs +69 -0
  12. package/lib/client/masto/index.mjs +183 -0
  13. package/lib/client/masto/instance.mjs +104 -0
  14. package/lib/client/masto/media.mjs +133 -0
  15. package/lib/client/masto/oauth.mjs +599 -0
  16. package/lib/client/masto/render.mjs +459 -0
  17. package/lib/client/masto/statuses.mjs +331 -0
  18. package/lib/client/masto/timelines.mjs +316 -0
  19. package/lib/{streaming.mjs → client/streaming.mjs} +1 -1
  20. package/lib/{acctfeed.mjs → connections/acctfeed.mjs} +1 -1
  21. package/lib/{atproto.mjs → connections/atproto.mjs} +15 -16
  22. package/lib/{bskygroup.mjs → connections/bskygroup.mjs} +1 -1
  23. package/lib/{fediacct.mjs → connections/fediacct.mjs} +31 -35
  24. package/lib/{import.mjs → connections/import.mjs} +1 -1
  25. package/lib/{tagfeed.mjs → connections/tagfeed.mjs} +3 -3
  26. package/lib/connections/vault.mjs +114 -0
  27. package/lib/core/as2.mjs +170 -0
  28. package/lib/core/contexts/activitystreams.json +379 -0
  29. package/lib/core/contexts/did-v1.json +57 -0
  30. package/lib/core/contexts/fep-5711.json +36 -0
  31. package/lib/core/contexts/gotosocial.json +86 -0
  32. package/lib/core/contexts/identity-v1.json +152 -0
  33. package/lib/core/contexts/index.mjs +45 -0
  34. package/lib/core/contexts/join-lemmy.json +33 -0
  35. package/lib/core/contexts/joinmastodon.json +28 -0
  36. package/lib/core/contexts/map.json +16 -0
  37. package/lib/core/contexts/miscellany.json +19 -0
  38. package/lib/core/contexts/schemaorg.json +8845 -0
  39. package/lib/core/contexts/security-data-integrity-v1.json +78 -0
  40. package/lib/core/contexts/security-data-integrity-v2.json +81 -0
  41. package/lib/core/contexts/security-multikey-v1.json +35 -0
  42. package/lib/core/contexts/security-v1.json +74 -0
  43. package/lib/core/contexts/webfinger.json +10 -0
  44. package/lib/{deliver.mjs → core/deliver.mjs} +2 -2
  45. package/lib/core/graphview.mjs +269 -0
  46. package/lib/core/intake/activities.mjs +437 -0
  47. package/lib/core/intake/activity.mjs +240 -0
  48. package/lib/core/intake/channel.mjs +144 -0
  49. package/lib/core/intake/group.mjs +222 -0
  50. package/lib/core/intake/index.mjs +629 -0
  51. package/lib/core/intake/notes.mjs +288 -0
  52. package/lib/core/intake/verify.mjs +142 -0
  53. package/lib/{keys.mjs → core/keys.mjs} +1 -1
  54. package/lib/core/publisher/collections.mjs +229 -0
  55. package/lib/core/publisher/index.mjs +421 -0
  56. package/lib/core/publisher/notes.mjs +188 -0
  57. package/lib/core/publisher/questions.mjs +233 -0
  58. package/lib/core/publisher/restore.mjs +199 -0
  59. package/lib/core/shapes/activitystreams.ttl +129 -0
  60. package/lib/core/shapes/index.mjs +107 -0
  61. package/lib/core/shapes/shapes-text.mjs +13 -0
  62. package/lib/{social.mjs → core/social.mjs} +2 -2
  63. package/lib/{store.mjs → core/store.mjs} +4 -0
  64. package/lib/{wire.mjs → core/wire.mjs} +2 -2
  65. package/lib/device/admin/index.mjs +13 -0
  66. package/lib/device/admin/origins.mjs +35 -0
  67. package/lib/device/admin/routes/connections.mjs +144 -0
  68. package/lib/device/admin/routes/gateway.mjs +199 -0
  69. package/lib/device/admin/routes/lifecycle.mjs +191 -0
  70. package/lib/device/admin/routes/owner.mjs +322 -0
  71. package/lib/device/admin/routes/setup.mjs +393 -0
  72. package/lib/device/admin/routes/social.mjs +188 -0
  73. package/lib/device/admin/server.mjs +95 -0
  74. package/lib/device/admin/static.mjs +244 -0
  75. package/lib/device/admin/surface.mjs +274 -0
  76. package/lib/device/cli/commands/account.mjs +586 -0
  77. package/lib/device/cli/commands/run.mjs +278 -0
  78. package/lib/device/cli/commands/service.mjs +221 -0
  79. package/lib/device/cli/commands/setup.mjs +410 -0
  80. package/lib/device/cli/commands/state.mjs +559 -0
  81. package/lib/device/cli/context.mjs +288 -0
  82. package/lib/{migrate.mjs → device/migrate.mjs} +1 -1
  83. package/lib/{remote.mjs → device/remote.mjs} +3 -3
  84. package/lib/{setup.mjs → device/setup.mjs} +3 -3
  85. package/lib/{update.mjs → device/update.mjs} +1 -1
  86. package/lib/{directory.mjs → gateway/directory.mjs} +1 -1
  87. package/lib/{front-core.mjs → gateway/front-core.mjs} +3 -3
  88. package/lib/{gateway-core.mjs → gateway/gateway-core.mjs} +1 -1
  89. package/lib/{httpsig.mjs → gateway/httpsig.mjs} +1 -1
  90. package/lib/server/embed.mjs +405 -0
  91. package/lib/{links.mjs → shared/links.mjs} +1 -1
  92. package/lib/{ua.mjs → shared/ua.mjs} +1 -1
  93. package/package.json +1 -1
  94. package/run-agent.mjs +33 -25
  95. package/web/admin/actors.js +145 -0
  96. package/web/admin/common.js +23 -0
  97. package/web/admin/connections.js +112 -0
  98. package/web/admin/gateway.js +111 -0
  99. package/web/admin/group.js +258 -0
  100. package/web/admin/index.html +7 -1
  101. package/web/admin/record.js +378 -0
  102. package/web/admin/setup/index.html +1 -0
  103. package/web/admin/setup/setup.js +2 -13
  104. package/web/admin/upkeep.js +170 -0
  105. package/web/app/README.md +6 -6
  106. package/web/app/admin-facade.mjs +3 -3
  107. package/web/app/agent.mjs +14 -16
  108. package/web/app/atproto-browser.mjs +1 -1
  109. package/web/app/boot.mjs +2 -3
  110. package/web/app/deliver-relay.mjs +1 -1
  111. package/web/app/dist/boot.js +22 -3
  112. package/web/app/dist/boot.js.map +2 -2
  113. package/web/app/dist/sw.js +21913 -5446
  114. package/web/app/dist/sw.js.map +4 -4
  115. package/web/app/fediacct-browser.mjs +1 -1
  116. package/web/app/keys-browser.mjs +27 -4
  117. package/web/app/shims/shapes-text.mjs +8 -0
  118. package/web/app/signup.mjs +2 -3
  119. package/web/app/site/admin/actors.js +145 -0
  120. package/web/app/site/admin/common.js +23 -0
  121. package/web/app/site/admin/connections.js +112 -0
  122. package/web/app/site/admin/gateway.js +111 -0
  123. package/web/app/site/admin/group.js +258 -0
  124. package/web/app/site/admin/index.html +7 -1
  125. package/web/app/site/admin/record.js +378 -0
  126. package/web/app/site/admin/setup/index.html +1 -0
  127. package/web/app/site/admin/setup/setup.js +2 -13
  128. package/web/app/site/admin/upkeep.js +170 -0
  129. package/web/app/site/boot.js +22 -3
  130. package/web/app/site/sw.js +21913 -5446
  131. package/web/app/sw-src.mjs +17 -2
  132. package/lib/admin.mjs +0 -1913
  133. package/lib/embed.mjs +0 -220
  134. package/lib/intake.mjs +0 -1981
  135. package/lib/mastoapi.mjs +0 -2284
  136. package/lib/publisher.mjs +0 -1192
  137. package/web/admin/admin.js +0 -1181
  138. package/web/app/site/admin/admin.js +0 -1181
  139. /package/lib/{oidc-auth.mjs → client/oidc-auth.mjs} +0 -0
  140. /package/lib/{webpush.mjs → client/webpush.mjs} +0 -0
  141. /package/lib/{bskyfeed.mjs → connections/bskyfeed.mjs} +0 -0
  142. /package/lib/{lease.mjs → core/lease.mjs} +0 -0
  143. /package/lib/{polls.mjs → core/polls.mjs} +0 -0
  144. /package/lib/{proof.mjs → core/proof.mjs} +0 -0
  145. /package/lib/{storage.mjs → core/storage.mjs} +0 -0
  146. /package/lib/{account.mjs → device/account.mjs} +0 -0
  147. /package/lib/{certs.mjs → device/certs.mjs} +0 -0
  148. /package/lib/{export-collections.mjs → device/export-collections.mjs} +0 -0
  149. /package/lib/{home.mjs → device/home.mjs} +0 -0
  150. /package/lib/{ports.mjs → device/ports.mjs} +0 -0
  151. /package/lib/{guard.mjs → shared/guard.mjs} +0 -0
  152. /package/lib/{safefetch.mjs → shared/safefetch.mjs} +0 -0
@@ -0,0 +1,629 @@
1
+ // intake.mjs — drains the remote pod's public-append inbox and applies side
2
+ // effects. Inbound authenticity: LDN bodies don't carry the delivery's
3
+ // HTTP-Signature headers, so instead of verifying signatures we VERIFY BY
4
+ // DEREFERENCING — re-fetch the claimed object/actor from its origin (signed
5
+ // GET, so authorized-fetch instances answer) and trust only what the origin
6
+ // itself serves.
7
+ //
8
+ // Failure policy: a REJECTED item (verification says no) goes to the
9
+ // dead-letter store and leaves the inbox; a FAILING item (exception —
10
+ // network, remote 5xx) stays in the inbox for the next drain, and moves to
11
+ // the dead-letter store after MAX_ITEM_ATTEMPTS. Nothing is silently
12
+ // destroyed.
13
+ //
14
+ // Wake-up: WebSocketChannel2023 push on the inbox container (probe P4), plus
15
+ // a poll every POLL_MS as fallback, plus a drain at startup.
16
+ //
17
+ // This file is the drain and the dispatcher. What it does with an activity
18
+ // once it has one is in the modules beside it — activity.mjs (what an activity
19
+ // is), channel.mjs (the push socket), verify.mjs (what to believe),
20
+ // activities.mjs (one handler per type), group.mjs (FEP-1b12), notes.mjs (a
21
+ // note on its way in) — each a set of functions taking the Intake as their
22
+ // first argument, reached here through one-line delegations so that every
23
+ // caller, and every test that overrides a method, sees one object.
24
+
25
+ import * as podInbox from '../../pod/inbox.mjs';
26
+ import { readCapped } from '../../shared/safefetch.mjs';
27
+ import { trimActivity, sameIdentity, sameOrigin, httpUrl, MAX_ITEM_BYTES, MAX_FORWARDS_PER_DRAIN } from './activity.mjs';
28
+ import * as channel from './channel.mjs';
29
+ import * as verify from './verify.mjs';
30
+ import * as group from './group.mjs';
31
+ import * as activities from './activities.mjs';
32
+ import * as notes from './notes.mjs';
33
+ import { readLenient } from '../as2.mjs';
34
+ import { checkShapes, describeShapeFailure } from '../shapes/index.mjs';
35
+ export { isContentType, trimActivity, sameIdentity, sameOrigin, sameSocketOrigin, httpUrl, authorOf } from './activity.mjs';
36
+
37
+ const POLL_MS = 2 * 60_000; // fallback cadence when there is no push
38
+ // With a live socket the poll is pure redundancy: it exists for the case where
39
+ // push is down, so it slows right down while push is up.
40
+ const POLL_PUSH_OK_MS = 10 * 60_000;
41
+ // A container that times out will time out again in two minutes, and each
42
+ // attempt holds one of the pod's workers for the full timeout. Sweeping stops
43
+ // for a while instead, doubling up to half an hour.
44
+ const DRAIN_COOLDOWN_MIN_MS = 2 * 60_000;
45
+ const DRAIN_COOLDOWN_MAX_MS = 30 * 60_000;
46
+ // Our DELETEs take the same container write lock as the deliveries arriving
47
+ // into it — a gap between them keeps a sweep from convoying against inbound.
48
+ const DELETE_GAP_MS = 150;
49
+ const CHAIN_GAP_MS = 5_000; // pause between chained backlog sweeps
50
+ // How many handled items ride on one commit before they are deleted. Small
51
+ // enough that a crash re-does little, large enough that a flood of fast
52
+ // rejections does not become a pod write per item.
53
+ const DELETE_BATCH = 10;
54
+ // Attempt counts live in pod state, not in memory: a restart used to hand every
55
+ // poison item five fresh tries, and under a crash loop that is unbounded.
56
+ const ATTEMPTS_DOC = 'intake-attempts.json';
57
+ const ATTEMPTS_TTL_MS = 7 * 24 * 60 * 60_000;
58
+ const MAX_ITEM_ATTEMPTS = 5;
59
+ const MAX_ITEMS_PER_DRAIN = 50;
60
+
61
+ export class Intake {
62
+ constructor({ config, urls, remote, store, deliverer, publisher, log = console.log, lease = null, archive = null, push = true, pollSeconds = null }) {
63
+ Object.assign(this, { config, urls, remote, store, deliverer, publisher, log, lease, archive, push, pollSeconds });
64
+ this.serial = Date.now();
65
+ this.stopped = false;
66
+ // (attempt counts are kept in pod state — see _bumpAttempt)
67
+ this.lastDrain = null;
68
+ this._forwardBudget = MAX_FORWARDS_PER_DRAIN;
69
+ this.lastDrainAtMs = 0;
70
+ this.reconnectTries = 0;
71
+ this.drainCooldownUntil = 0;
72
+ this.drainFailures = 0;
73
+ this.wsState = 'never-connected';
74
+ }
75
+
76
+ // Draining is a destructive read — an item is gone from the pod once we
77
+ // DELETE it — so the result of handling it must be on disk first.
78
+ //
79
+ // This used to be skipped whenever the state and the inbox shared an origin,
80
+ // on the reasoning that a pod we cannot write to is a pod we cannot list
81
+ // either, so the drain never starts. That covers the pod being unreachable
82
+ // and nothing else: it does not cover a crash inside the 300ms debounce
83
+ // window, and it does not cover a pod that refuses a write while still
84
+ // serving reads and deletes — a quota, a 507, a 403 on one document. In
85
+ // either case every item drained since the last successful write is gone,
86
+ // and what goes is the mentions, replies, join requests and dead-letter
87
+ // records that nothing else can rebuild.
88
+ async _persisted() {
89
+ return this.store.commit();
90
+ }
91
+
92
+ _backOff(why) {
93
+ this.drainFailures++;
94
+ const capped = Math.min(DRAIN_COOLDOWN_MIN_MS * 2 ** (this.drainFailures - 1), DRAIN_COOLDOWN_MAX_MS);
95
+ this.drainCooldownUntil = Date.now() + Math.round(capped * (0.85 + Math.random() * 0.3));
96
+ this.log(`${why} — next sweep in ${Math.round(capped / 1000)}s`);
97
+ }
98
+
99
+ // Sweep cadence: a configured interval wins, otherwise a live push channel —
100
+ // a notification socket, or the store's own change events in-process — is
101
+ // what makes the fallback poll a slow one.
102
+ _pollMs() {
103
+ if (this.pollSeconds) return this.pollSeconds * 1000;
104
+ return this.wsState === 'open' || this.wsState === 'in-process' ? POLL_PUSH_OK_MS : POLL_MS;
105
+ }
106
+
107
+ async start() {
108
+ this.stopped = false; // restartable across demote/takeover cycles
109
+ await this.drain().catch(e => this.log(`drain: ${e.message}`));
110
+ const tick = () => {
111
+ this.pollTimer = setTimeout(() => {
112
+ this.drain().catch(e => this.log(`drain: ${e.message}`)).finally(() => { if (!this.stopped) tick(); });
113
+ }, Math.round(this._pollMs() * (0.85 + Math.random() * 0.3)));
114
+ this.pollTimer.unref?.();
115
+ };
116
+ tick();
117
+ // Embedded in the pod server, a notification socket back to that same
118
+ // server buys nothing — the store's own change events wake the drain.
119
+ if (this.push) this.subscribe().catch(e => this.log(`subscribe: ${e.message}`));
120
+ else this.wsState = 'in-process';
121
+ }
122
+
123
+ stop() { this.stopped = true; clearTimeout(this.pollTimer); clearTimeout(this.resubTimer); this.ws?.close(); }
124
+
125
+ // Attempt bookkeeping, persisted. Written only when an item fails, so a
126
+ // healthy inbox never touches this document.
127
+ _bumpAttempt(url, message) {
128
+ const all = this.store.read(ATTEMPTS_DOC, {});
129
+ const rec = all[url] || { n: 0 };
130
+ rec.n += 1;
131
+ rec.at = new Date().toISOString();
132
+ rec.last = String(message || '').slice(0, 200);
133
+ all[url] = rec;
134
+ this.store.write(ATTEMPTS_DOC, all);
135
+ return rec.n;
136
+ }
137
+
138
+ _clearAttempt(url) {
139
+ const all = this.store.read(ATTEMPTS_DOC, {});
140
+ if (!all[url]) return;
141
+ delete all[url];
142
+ this.store.write(ATTEMPTS_DOC, all);
143
+ }
144
+
145
+ // Items deleted long ago would otherwise accumulate here forever.
146
+ _pruneAttempts() {
147
+ const all = this.store.read(ATTEMPTS_DOC, {});
148
+ const cutoff = Date.now() - ATTEMPTS_TTL_MS;
149
+ let dropped = 0;
150
+ for (const [url, rec] of Object.entries(all)) {
151
+ if (!rec?.at || Date.parse(rec.at) < cutoff) { delete all[url]; dropped++; }
152
+ }
153
+ if (dropped) { this.store.write(ATTEMPTS_DOC, all); this.log(`pruned ${dropped} stale inbox attempt record(s)`); }
154
+ }
155
+
156
+ // --- drain + dispatch ---
157
+ // Serialized: push events, polls, and manual /drain calls can fire
158
+ // concurrently, and overlapping sweeps double-process items (observed as
159
+ // duplicate Accepts/timeline writes). One sweep at a time; callers that
160
+ // arrive mid-sweep get one follow-up sweep.
161
+ async drain() {
162
+ if (this._draining) { this._drainAgain = true; return this._draining; }
163
+ // Held for the whole sweep, released however it ends. The debounce cannot
164
+ // coalesce a drain — every handler awaits somebody else's server first — so
165
+ // the writes are left to the commit boundaries the drain already has. See
166
+ // PodStore.hold.
167
+ // Optional: this is a throughput hint, not part of the commit-before-delete
168
+ // invariant — commit() flushes whatever is pending either way — so a store
169
+ // that does not implement it behaves exactly as before.
170
+ this._inSweep = true;
171
+ // Fresh per sweep: the cap is on how much this drain may amplify, not a
172
+ // lifetime total (see _maybeForward).
173
+ this._forwardBudget = MAX_FORWARDS_PER_DRAIN;
174
+ this.store.hold?.();
175
+ this._draining = this._drainOnce().finally(async () => {
176
+ this._inSweep = false;
177
+ await this._publishPending();
178
+ this.store.release?.();
179
+ this._draining = null;
180
+ if (this._drainAgain) {
181
+ this._drainAgain = false;
182
+ // Paced, not immediate: chained sweeps put a ceiling on work per unit
183
+ // time, so a delivery flood cannot run the drain back-to-back.
184
+ const t = setTimeout(() => this.drain().catch(e => this.log(`drain: ${e.message}`)), CHAIN_GAP_MS);
185
+ t.unref?.();
186
+ }
187
+ });
188
+ return this._draining;
189
+ }
190
+
191
+ // Discard the content waiting in the inbox from before `before`, on the
192
+ // owner's say-so — the admin page asks, this does it. NOT a blind sweep:
193
+ // every item is read, because the type is the only thing that decides its
194
+ // fate and size does not predict it. A Follow, Undo, Accept or Delete is
195
+ // APPLIED whatever it weighs, so the follow graph stays correct and a post
196
+ // its author retracted still goes; only a Create is dropped. Judging by size
197
+ // instead saved one request per item and lost any control activity that
198
+ // happened to be large, silently and permanently.
199
+ //
200
+ // An item past the drain's own byte cap is deleted unread: it could not be
201
+ // handled if it were read, so there is nothing to lose by not reading it.
202
+ //
203
+ // `keepConcerning` narrows the discard to noise: every item is passed to
204
+ // handle(), which ingests a Create only when concernsUs passes — addressed to
205
+ // us, a mention, a reply to ours, or from someone we follow — and drops the
206
+ // rest.
207
+ async prune({ before, keepConcerning = false } = {}) {
208
+ const cutoff = Date.parse(before);
209
+ if (!Number.isFinite(cutoff)) throw new Error(`"${before}" is not a date`);
210
+ const all = await podInbox.list(this.remote, this.urls);
211
+ const older = all.filter(e => !e.url.endsWith('.keep')
212
+ && e.modified && Date.parse(e.modified) < cutoff);
213
+ const out = { considered: older.length, applied: 0, dropped: 0, discarded: 0, failed: 0 };
214
+
215
+ for (const item of older) {
216
+ try {
217
+ if (item.size > MAX_ITEM_BYTES) {
218
+ await podInbox.dropHandledItem(this.remote, item.url); // unreadable by the drain either way
219
+ out.discarded++;
220
+ } else {
221
+ // Same rule as the drain: a read we could not make is not a Create to
222
+ // be dropped. readItem throws on anything but 404, so it counts as
223
+ // failed and stays for the next pass.
224
+ const got = await podInbox.readItem(this.remote, item.url, { maxBytes: MAX_ITEM_BYTES, readCapped });
225
+ const read = got.raw === null ? null : await readLenient(got.raw);
226
+ if (read?.degraded) this.log(`inbox item ${item.url} grounded to read: ${read.degraded}`);
227
+ const activity = read?.view ?? read?.doc ?? null;
228
+ // A Create is the content the owner just asked to be rid of. Anything
229
+ // else changes state and is applied exactly as a drain would.
230
+ if (keepConcerning) {
231
+ const rejection = activity ? await this.handle(activity) : 'unparsable JSON';
232
+ if (rejection) out.dropped++; else out.applied++;
233
+ } else if (activity && activity.type !== 'Create') {
234
+ await this.handle(activity);
235
+ out.applied++;
236
+ } else {
237
+ out.dropped++;
238
+ }
239
+ if (!await this._persisted()) {
240
+ this.log(`state not written — stopping the prune with ${older.length - out.applied - out.dropped - out.discarded} left`);
241
+ break;
242
+ }
243
+ await podInbox.dropHandledItem(this.remote, item.url);
244
+ }
245
+ this._clearAttempt(item.url);
246
+ await new Promise(r => setTimeout(r, DELETE_GAP_MS));
247
+ } catch (e) {
248
+ out.failed++;
249
+ this.log(`prune ${item.url}: ${e.message}`);
250
+ }
251
+ }
252
+ this.log(`pruned before ${before}: applied ${out.applied}, dropped ${out.dropped} `
253
+ + `small Create(s), discarded ${out.discarded} unread${out.failed ? `, ${out.failed} failed` : ''}`);
254
+ await this.store.flush();
255
+ // Adjust the measurement in place rather than kicking a drain to re-take
256
+ // it: an un-awaited drain would still be running when this returns, which
257
+ // races whoever called us. The poll picks the rest up soon enough.
258
+ const removed = out.applied + out.dropped + out.discarded;
259
+ if (this.inboxStats && removed) {
260
+ this.inboxStats = { ...this.inboxStats, count: Math.max(0, this.inboxStats.count - removed) };
261
+ }
262
+ return out;
263
+ }
264
+
265
+ // The sender's original bytes, kept after the activity was verified and
266
+ // applied and before the pod DELETE erases the only other copy. The filename
267
+ // is the content's own hash, so a re-delivered activity lands on the same
268
+ // file instead of duplicating. Best-effort history: a failed write logs and
269
+ // the drain goes on — mail must never stall on its own receipt. This is the
270
+ // one category of account data that lives only in the private half; the pod
271
+ // cannot rebuild it because the pod never kept it.
272
+ //
273
+ // The record is JSON-LD: a cnt:ContentAsText whose cnt:chars are the raw
274
+ // bytes, stamped prov:generatedAtTime / prov:wasDerivedFrom / as:actor —
275
+ // plain JSON to everything here, a graph to any RDF reader.
276
+ async _archive(sourceUrl, raw, activity) {
277
+ try {
278
+ if (!this.archive || this.store.getConfig()?.archiveInbox === false) return;
279
+ const { createHash } = await import('node:crypto');
280
+ const hash = createHash('sha256').update(raw).digest('hex').slice(0, 16);
281
+ const receivedAt = new Date().toISOString();
282
+ const rec = {
283
+ '@context': {
284
+ prov: 'http://www.w3.org/ns/prov#',
285
+ cnt: 'http://www.w3.org/2011/content#',
286
+ as: 'https://www.w3.org/ns/activitystreams#',
287
+ xsd: 'http://www.w3.org/2001/XMLSchema#',
288
+ receivedAt: { '@id': 'prov:generatedAtTime', '@type': 'xsd:dateTime' },
289
+ source: { '@id': 'prov:wasDerivedFrom', '@type': '@id' },
290
+ actor: { '@id': 'as:actor', '@type': '@id' },
291
+ raw: 'cnt:chars',
292
+ },
293
+ '@id': '',
294
+ '@type': 'cnt:ContentAsText',
295
+ receivedAt,
296
+ actor: typeof activity?.actor === 'string' ? activity.actor : activity?.actor?.id || null,
297
+ source: sourceUrl,
298
+ raw,
299
+ };
300
+ const w = await this.archive.write(`${receivedAt.slice(0, 7)}/${hash}.json`,
301
+ JSON.stringify(rec, null, 2), 'application/ld+json');
302
+ if (!w.ok) this.log(`inbox archive: ${w.why || 'write failed'}`);
303
+ } catch (e) {
304
+ this.log(`inbox archive: ${e.message}`);
305
+ }
306
+ }
307
+
308
+ async _drainOnce() {
309
+ const cooling = this.drainCooldownUntil - Date.now();
310
+ if (cooling > 0) {
311
+ this.log(`inbox sweep skipped — backing off for another ${Math.ceil(cooling / 1000)}s`);
312
+ return;
313
+ }
314
+ // Draining DELETES from the pod, so it must not run on a lease that has
315
+ // quietly expired. renewOnce notices at its own cadence — up to ~117s — and
316
+ // after the TTL another agent is entitled to start draining the same inbox.
317
+ if (this.lease && !this.lease.stillHeld()) {
318
+ this.log('lease is no longer held — not draining');
319
+ return;
320
+ }
321
+ this.lastDrain = new Date().toISOString();
322
+ this.lastDrainAtMs = Date.now();
323
+ this._pruneAttempts();
324
+ let all;
325
+ try {
326
+ all = await podInbox.list(this.remote, this.urls);
327
+ this.drainFailures = 0;
328
+ } catch (e) {
329
+ this._backOff(`inbox unreadable (${e.message})`);
330
+ return;
331
+ }
332
+ // What is waiting, measured from the listing we already fetched: no extra
333
+ // request, and it is what /status reports and what the admin page prompts
334
+ // on. The listing arrives oldest-first (lib/remote.mjs).
335
+ const real = all.filter(e => !e.url.endsWith('.keep'));
336
+ this.inboxStats = {
337
+ count: real.length,
338
+ bytes: real.reduce((n, e) => n + e.size, 0),
339
+ oldest: real[0]?.modified || null,
340
+ newest: real[real.length - 1]?.modified || null,
341
+ at: new Date().toISOString(),
342
+ };
343
+ // The inbox is public-Append: a flood must not turn one sweep into an
344
+ // unbounded run. But stopping there is why a backlog never cleared — 50
345
+ // items every two minutes does not converge on an agent that is only
346
+ // running while a laptop is open. So a sweep that made progress and left
347
+ // work behind goes straight round again.
348
+ const items = all.slice(0, MAX_ITEMS_PER_DRAIN);
349
+ if (all.length > items.length) this.log(`inbox has ${all.length} items — processing ${items.length} this sweep`);
350
+ let handled = 0;
351
+ // Deletes are batched behind ONE commit rather than a commit per item.
352
+ // Per-item, the 300ms debounce that coalesces a sweep's writes never gets
353
+ // to do its job: on a flood of fast rejections that is fifty writes of
354
+ // deadletter.json where one would do, and a flood is exactly when the pod
355
+ // should be asked for less rather than more.
356
+ const pending = [];
357
+ const flush = async () => {
358
+ if (!pending.length) return true;
359
+ // Written down before any of them leaves the mailbox. A failure here
360
+ // leaves them where they are: the next sweep sees them again, and a
361
+ // re-delivered activity is handled idempotently.
362
+ if (!await this._persisted()) {
363
+ this._backOff(`state not written — ${pending.length} item(s) left in the inbox`);
364
+ pending.length = 0;
365
+ return false;
366
+ }
367
+ for (const url of pending.splice(0)) {
368
+ if (!await podInbox.dropHandledItem(this.remote, url)) {
369
+ // Still in the mailbox. Handling is idempotent so seeing it again is
370
+ // harmless, but counting it would clear the attempt record and report
371
+ // progress that did not happen.
372
+ this.log(`inbox item ${url} was handled but NOT removed — it will be seen again`);
373
+ continue;
374
+ }
375
+ this._clearAttempt(url);
376
+ handled++;
377
+ await new Promise(r => setTimeout(r, DELETE_GAP_MS));
378
+ }
379
+ return true;
380
+ };
381
+
382
+ for (const { url, size } of items) {
383
+ if (url.endsWith('.keep')) continue;
384
+ // The listing already carries every child's size, so this costs nothing
385
+ // to ask. An activity is a few kB; anything of this order is not one, and
386
+ // reading it with an unbounded res.text() buffers whatever a stranger
387
+ // chose to Append into memory.
388
+ // A cheap pre-filter only: listContainer coerces a missing posix:size to
389
+ // 0, so a pod that does not publish sizes would wave everything through.
390
+ // The real bound is readCapped on the body below.
391
+ if (size > MAX_ITEM_BYTES) {
392
+ this.store.addDeadLetter({ inboxUrl: url, reason: `oversized (${size} bytes)`, activity: null });
393
+ pending.push(url);
394
+ continue;
395
+ }
396
+ let activity = null;
397
+ try {
398
+ const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_ITEM_BYTES, readCapped });
399
+ // readItem carries the rule that matters here: a pod that would not
400
+ // GIVE us the item has told us nothing about it, so anything but a 404
401
+ // throws rather than reading as an empty body. Reading a 500 as empty
402
+ // made it "unparsable JSON" — a REJECTION, dead-lettered with both
403
+ // `activity` and `raw` null and then DELETEd, destroying a delivery on
404
+ // a transient fault with no record of what it had been.
405
+ const raw = got.raw;
406
+ // Read as JSON-LD, so a document whose terms are aliased means what it
407
+ // says. A document we cannot read that way is read the way it always
408
+ // was rather than lost, and the reason is logged.
409
+ const read = raw ? await readLenient(raw) : null;
410
+ if (read?.degraded) this.log(`inbox item ${url} grounded to read: ${read.degraded}`);
411
+ activity = read?.view ?? read?.doc ?? null;
412
+ // What the shapes make of it. This decides NOTHING — the activity is
413
+ // handled either way. It is written down so that which shapes fire on
414
+ // real traffic is a question with an answer, rather than a guess made
415
+ // in advance about implementations we have not met.
416
+ if (read?.graph) {
417
+ const failure = await checkShapes(read.graph);
418
+ if (failure) {
419
+ const said = describeShapeFailure(failure);
420
+ this.log(`inbox item ${url} does not fit its shape: ${said}`);
421
+ this.store.addDeadLetter({
422
+ inboxUrl: url, reason: `shape: ${said}`, shapeOnly: true, activity: trimActivity(activity),
423
+ });
424
+ }
425
+ }
426
+ // A gateway that verified this delivery left a receipt beside it. Read
427
+ // it only when a gateway is configured (no config → no fetch, so an
428
+ // install with no gateway pays nothing); a missing or HMAC-invalid
429
+ // receipt reads as null, which is exactly today's unverified behavior.
430
+ const receipt = activity ? await this._readReceipt(url) : null;
431
+ if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
432
+ const rejection = activity ? await this.handle(activity, receipt) : 'unparsable JSON';
433
+ if (!rejection && raw) await this._archive(url, raw, activity);
434
+ if (!rejection) await this._maybeForward(activity); // §7.1.2, only what we accepted
435
+ if (rejection) {
436
+ this.store.addDeadLetter({
437
+ inboxUrl: url, reason: rejection, activity: trimActivity(activity),
438
+ ...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
439
+ });
440
+ this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
441
+ }
442
+ pending.push(url);
443
+ } catch (e) {
444
+ const n = this._bumpAttempt(url, e.message);
445
+ this.log(`inbox item ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
446
+ if (n >= MAX_ITEM_ATTEMPTS) {
447
+ this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(activity) });
448
+ // The dead letter IS the record of this item — deleting before it is
449
+ // written down would lose the only evidence it ever arrived, so it
450
+ // goes through the same commit-then-delete batch as everything else.
451
+ pending.push(url);
452
+ }
453
+ }
454
+ if (pending.length >= DELETE_BATCH && !await flush()) return;
455
+ }
456
+ if (!await this._finishSweep(flush)) return;
457
+ // Made progress and there is more waiting: go straight round rather than
458
+ // sleeping. Gated on progress so a sweep that achieved nothing — a
459
+ // cooldown, an unwritable store, poison at the head — cannot spin.
460
+ if (handled > 0 && all.length > items.length && !this.stopped) this._drainAgain = true;
461
+ }
462
+
463
+ // The end of a sweep: publish whatever the follow graph did ONCE, then flush.
464
+ //
465
+ // publishCollections used to run per handled item — every Follow, Undo,
466
+ // Accept, Reject, admit and eject — and each one is a full GET of the pod's
467
+ // followers collection plus a PUT of it. Fifty follows in a sweep were a
468
+ // hundred requests where two would do, and the answer they arrive at is the
469
+ // same either way, because it is built from contacts.json in memory.
470
+ async _finishSweep(flush) {
471
+ await this._publishPending();
472
+ return flush();
473
+ }
474
+
475
+ // Idempotent: it clears what it takes, so the drain's own exit path calling
476
+ // it again after a sweep that bailed early — an unwritable store, a delete
477
+ // that failed — is a no-op in the ordinary case and the difference between
478
+ // "published" and "waiting for a sweep that may never come" in the other.
479
+ async _publishPending() {
480
+ const want = this._republish;
481
+ this._republish = null;
482
+ if (!want) return;
483
+ try { await this.publisher.publishCollections(want); }
484
+ catch (e) { this.log(`publishing collections: ${e.message}`); }
485
+ }
486
+
487
+ // Ask for a collection to be republished at the end of this sweep. Outside a
488
+ // sweep there is no boundary to wait for, so it happens now.
489
+ async republish(which) {
490
+ if (!this._inSweep) return this.publisher.publishCollections(which);
491
+ this._republish = { ...(this._republish || {}), ...which };
492
+ }
493
+
494
+ sameOrigin(a, b) { return sameOrigin(a, b); }
495
+ // Overridable in tests the same way sameOrigin is.
496
+ sameIdentity(a, b) { return sameIdentity(a, b); }
497
+
498
+ async handle(activity, receipt = null) { // eslint-disable-line no-unused-vars
499
+ const actor = typeof activity.actor === 'string' ? activity.actor : activity.actor?.id;
500
+ if (!actor) return 'no actor';
501
+ // An actor is a URL that can be dereferenced. Most arms here go on to fetch
502
+ // it, and safefetch refuses a bad scheme there — but Like and Announce on
503
+ // one of our own notes record a notification without dereferencing
504
+ // anything, so `javascript:` and `data:` reached the client as an account
505
+ // url. The store already guards avatars this way (safeUrl); actors were
506
+ // simply never put through it.
507
+ if (!httpUrl(actor)) return `actor is not an http(s) URL (${actor})`;
508
+ if (this.store.isBlocked(actor)) return `blocked sender (${actor})`;
509
+
510
+ // Whether the door vouched for this sender — read by the moderation queue
511
+ // just below as well as by the arms further down, so it is settled here,
512
+ // before its first use.
513
+ const trusted = this.receiptVouchesFor(receipt, actor)
514
+ && this.store.getConfig()?.gateway?.mode === 'trust';
515
+
516
+ // FEP-1b12 moderation from a LISTED moderator. A delivery proves nothing
517
+ // about its sender, which is why these are QUEUED for the operator rather
518
+ // than run on arrival — the queue is where a claimed moderator's word
519
+ // waits for the one person who can vouch for it. Everything else about
520
+ // the activity falls through to the ordinary arms.
521
+ if (this.config.kind === 'group'
522
+ && (this.config.moderators || []).includes(actor)
523
+ && this.isModerationAsk(activity)) {
524
+ return this.queueModeration(activity, actor, { trusted });
525
+ }
526
+
527
+ // `trusted` was settled above, before the moderation queue reads it. Why it
528
+ // is not simply `receipt.verified`: that says the door checked a signature
529
+ // and the signature was good; it does NOT say whose. The door reports the
530
+ // signing key's owner separately, in `receipt.actor`, and nothing here used
531
+ // to read it — so ANY valid fediverse signing key, over an activity whose
532
+ // `actor` field named someone else entirely, arrived as trusted. In trust
533
+ // mode that is one delivery to evict any of your followers, or to have a
534
+ // Follow naming a third party auto-accepted. See receiptVouchesFor, which
535
+ // also holds the keyId to the actor's origin: a key document is fetched
536
+ // from wherever its id points, so one hosted elsewhere that merely CLAIMS
537
+ // `owner: <you>` would otherwise bind.
538
+
539
+ switch (activity.type) {
540
+ case 'Follow': return this.onFollow(activity, actor, { trusted });
541
+ case 'Undo': return this.onUndo(activity, actor, { trusted });
542
+ case 'Create': return this.onCreate(activity, actor);
543
+ case 'Accept': return this.onAccept(activity, actor, { trusted });
544
+ case 'Like': case 'Announce': {
545
+ // FEP-1b12: a group Announces the member's whole Create, not the note.
546
+ // Without unwrapping we try to ingest a Create as if it were a Note and
547
+ // dead-letter every post a group ever carries — including our own.
548
+ const wrapped = activity.object;
549
+ // FEP-1b12: a group announces its moderation too. The one act a
550
+ // follower can honor without trusting anyone new is a Delete of a
551
+ // post that same group carried to us — the carrier unsaying its carry.
552
+ if (activity.type === 'Announce' && wrapped && typeof wrapped === 'object'
553
+ && wrapped.type === 'Delete') {
554
+ return this.onAnnouncedDelete(actor, wrapped);
555
+ }
556
+ const inner = (wrapped && typeof wrapped === 'object'
557
+ && (wrapped.type === 'Create' || wrapped.type === 'Update')) ? wrapped.object : wrapped;
558
+ const objectId = typeof inner === 'string' ? inner : inner?.id;
559
+ this.log(`${activity.type} from ${actor} on ${objectId}`);
560
+ if (objectId && objectId.startsWith(this.urls.notes)) {
561
+ // Nothing vouches for this actor: a Like carries no signature and,
562
+ // unlike a Create, has no object at the sender's origin to re-read.
563
+ // `known()` is answered from local state and costs nothing — a
564
+ // stranger's favourite is still recorded, it is just the first thing
565
+ // the cap evicts, so a flood cannot push out real history.
566
+ this.store.addNotification({
567
+ type: activity.type === 'Like' ? 'favourite' : 'reblog', actor, noteId: objectId,
568
+ ...(this.known(actor) ? {} : { unverified: true }),
569
+ });
570
+ return;
571
+ }
572
+ if (activity.type === 'Announce') return this.onAnnounce(activity, actor, objectId);
573
+ return;
574
+ }
575
+ case 'Delete': return this.onDelete(activity, actor);
576
+ case 'Update': return this.onUpdate(activity, actor);
577
+ case 'Reject': return this.onReject(activity, actor, { trusted });
578
+ case 'Move': return this.onMove(activity, actor);
579
+ case 'Add': case 'Remove': return this.onAddRemove(activity, actor);
580
+ default: this.log(`ignored ${activity.type} from ${actor}`);
581
+ }
582
+ }
583
+
584
+ // channel.mjs
585
+ _reconnectDelay(...a) { return channel.reconnectDelay(this, ...a); }
586
+ subscribe(...a) { return channel.subscribe(this, ...a); }
587
+ _storageDescriptionUrl(...a) { return channel.storageDescriptionUrl(this, ...a); }
588
+ _subscribeOnce(...a) { return channel.subscribeOnce(this, ...a); }
589
+ _openSocket(...a) { return channel.openSocket(this, ...a); }
590
+
591
+ // verify.mjs
592
+ fetchAP(...a) { return verify.fetchAP(this, ...a); }
593
+ known(...a) { return verify.known(this, ...a); }
594
+ gatewaySecret(...a) { return verify.gatewaySecret(this, ...a); }
595
+ _bumpGatewayStat(...a) { return verify.bumpGatewayStat(this, ...a); }
596
+ _readReceipt(...a) { return verify.readReceipt(this, ...a); }
597
+ receiptVouchesFor(...a) { return verify.receiptVouchesFor(this, ...a); }
598
+ isGone(...a) { return verify.isGone(this, ...a); }
599
+
600
+ // group.mjs
601
+ isModerationAsk(...a) { return group.isModerationAsk(this, ...a); }
602
+ queueModeration(...a) { return group.queueModeration(this, ...a); }
603
+ amplify(...a) { return group.amplify(this, ...a); }
604
+ isCoMember(...a) { return group.isCoMember(this, ...a); }
605
+ collectionMembers(...a) { return group.collectionMembers(this, ...a); }
606
+ announceTargets(...a) { return group.announceTargets(this, ...a); }
607
+
608
+ // activities.mjs
609
+ onAddRemove(...a) { return activities.onAddRemove(this, ...a); }
610
+ onFollow(...a) { return activities.onFollow(this, ...a); }
611
+ onUndo(...a) { return activities.onUndo(this, ...a); }
612
+ onCreate(...a) { return activities.onCreate(this, ...a); }
613
+ onAnnouncedDelete(...a) { return activities.onAnnouncedDelete(this, ...a); }
614
+ onAnnounce(...a) { return activities.onAnnounce(this, ...a); }
615
+ onDelete(...a) { return activities.onDelete(this, ...a); }
616
+ onUpdate(...a) { return activities.onUpdate(this, ...a); }
617
+ onReject(...a) { return activities.onReject(this, ...a); }
618
+ onMove(...a) { return activities.onMove(this, ...a); }
619
+ onAccept(...a) { return activities.onAccept(this, ...a); }
620
+
621
+ // notes.mjs
622
+ concernsUs(...a) { return notes.concernsUs(this, ...a); }
623
+ _maybeForward(...a) { return notes.maybeForward(this, ...a); }
624
+ _referencesOurObject(...a) { return notes.referencesOurObject(this, ...a); }
625
+ ingestNote(...a) { return notes.ingestNote(this, ...a); }
626
+ forget(...a) { return notes.forget(this, ...a); }
627
+ retract(...a) { return notes.retract(this, ...a); }
628
+ addReply(...a) { return notes.addReply(this, ...a); }
629
+ }