fedipod 1.36.6 → 1.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +29 -8
  2. package/architecture.md +8 -0
  3. package/bin/fedipod.mjs +6 -0
  4. package/cli.md +7 -1
  5. package/device-agent.md +3 -3
  6. package/gateway.md +69 -8
  7. package/groups.md +2 -1
  8. package/gui.md +3 -1
  9. package/lib/client/c2s.mjs +97 -43
  10. package/lib/client/masto/accounts.mjs +1 -0
  11. package/lib/client/masto/bridge.mjs +61 -0
  12. package/lib/client/masto/index.mjs +4 -1
  13. package/lib/client/masto/oauth.mjs +1 -1
  14. package/lib/client/masto/statuses.mjs +3 -0
  15. package/lib/connections/acctfeed.mjs +13 -9
  16. package/lib/connections/bskyfeed.mjs +13 -9
  17. package/lib/connections/tagfeed.mjs +14 -9
  18. package/lib/core/deliver.mjs +17 -1
  19. package/lib/core/intake/activities.mjs +18 -2
  20. package/lib/core/intake/group.mjs +3 -1
  21. package/lib/core/intake/index.mjs +226 -48
  22. package/lib/core/intake/notes.mjs +12 -5
  23. package/lib/core/intake/verify.mjs +11 -0
  24. package/lib/core/lease.mjs +15 -1
  25. package/lib/core/place.mjs +82 -0
  26. package/lib/core/pod-only.mjs +4 -0
  27. package/lib/core/publisher/collections.mjs +37 -2
  28. package/lib/core/publisher/index.mjs +25 -1
  29. package/lib/core/publisher/notes.mjs +73 -7
  30. package/lib/core/publisher/own.mjs +143 -0
  31. package/lib/core/publisher/questions.mjs +5 -3
  32. package/lib/core/scheduled.mjs +39 -0
  33. package/lib/core/social.mjs +62 -32
  34. package/lib/core/storage.mjs +67 -0
  35. package/lib/core/store.mjs +77 -5
  36. package/lib/core/wire.mjs +51 -15
  37. package/lib/device/admin/routes/lifecycle.mjs +1 -1
  38. package/lib/device/admin/routes/setup.mjs +17 -1
  39. package/lib/device/cli/commands/setup.mjs +50 -8
  40. package/lib/device/cli/context.mjs +1 -1
  41. package/lib/device/migrate.mjs +1 -1
  42. package/lib/device/setup.mjs +45 -8
  43. package/lib/gateway/account-agent.mjs +111 -0
  44. package/lib/gateway/copy.mjs +381 -0
  45. package/lib/gateway/front-core.mjs +74 -63
  46. package/lib/gateway/gateway-core.mjs +107 -9
  47. package/lib/gateway/held-mail.mjs +198 -0
  48. package/lib/gateway/keeper-due.mjs +39 -0
  49. package/lib/gateway/keeper-session.mjs +10 -0
  50. package/lib/gateway/keeper.mjs +72 -0
  51. package/lib/gateway/masto-gateway.mjs +510 -0
  52. package/lib/gateway/quiet.mjs +7 -2
  53. package/lib/gateway/relay-extras.mjs +89 -0
  54. package/lib/gateway/state-api.mjs +207 -0
  55. package/lib/gateway/token-claims.mjs +16 -0
  56. package/lib/pod/containers.mjs +17 -0
  57. package/lib/pod/location.mjs +52 -0
  58. package/lib/pod/notes.mjs +2 -4
  59. package/lib/pod/transport.mjs +214 -24
  60. package/lib/pod/type-index.mjs +101 -0
  61. package/lib/pod/urls.mjs +6 -0
  62. package/lib/server/embed.mjs +7 -6
  63. package/lib/session/README.md +5 -5
  64. package/lib/session/demo.html +1 -1
  65. package/lib/session/fedi-account.mjs +19 -10
  66. package/lib/session/package.json +2 -2
  67. package/package.json +2 -2
  68. package/run-agent.mjs +6 -16
  69. package/scripts/stage-site.mjs +17 -4
  70. package/web/admin/actors.js +2 -0
  71. package/web/admin/gateway.js +14 -1
  72. package/web/admin/index.html +13 -0
  73. package/web/admin/oauth-signin.mjs +1 -1
  74. package/web/admin/record.js +4 -1
  75. package/web/admin/setup/index.html +17 -1
  76. package/web/admin/setup/setup.js +18 -5
  77. package/web/app/README.md +2 -2
  78. package/web/app/admin-facade.mjs +12 -2
  79. package/web/app/agent.mjs +228 -51
  80. package/web/app/boot.mjs +75 -32
  81. package/web/app/copy-mode.mjs +223 -0
  82. package/web/app/dist/boot.js +498 -88
  83. package/web/app/dist/boot.js.map +4 -4
  84. package/web/app/dist/sw.js +4016 -2539
  85. package/web/app/dist/sw.js.map +4 -4
  86. package/web/app/index.html +16 -0
  87. package/web/app/signup.mjs +65 -26
  88. package/web/app/sw-src.mjs +15 -53
  89. package/web/app/update.js +3 -2
  90. package/web/app/warm-start.mjs +115 -0
  91. package/web/app-signin/app-signin.mjs +78 -0
  92. package/web/app-signin/index.html +41 -0
  93. package/web/front/run.html +7 -1
  94. package/web/front/run.js +30 -4
@@ -149,7 +149,10 @@ export class MastoApi {
149
149
  // A viewer-mode agent (another agent holds the drain lease) may not act —
150
150
  // but a user acting HERE outranks the idle active agent elsewhere, so a
151
151
  // write attempt claims the lease and proceeds. Only a failed claim 503s.
152
- if (this.agent.viewer && req.method !== 'GET' && req.method !== 'HEAD') {
152
+ // An agent working from a copy its gateway shares (the browser build,
153
+ // web/app/copy-mode.mjs) asks every time, since an app at the gateway may
154
+ // have taken the lease without it hearing yet.
155
+ if ((this.agent.viewer || this.agent.copy) && req.method !== 'GET' && req.method !== 'HEAD') {
153
156
  const took = await this.agent.requestTakeover?.();
154
157
  if (!took) return send(503, { error: 'another agent is active for this pod — takeover failed, try again' });
155
158
  }
@@ -45,7 +45,7 @@ const parseRedirects = (v) => (Array.isArray(v) ? v : String(v || '').split(/\s+
45
45
  // The sign-in page for an account whose owner signs in at their pod: no
46
46
  // password of ours exists, and none is asked for. The page carries the
47
47
  // client's request in its address, signs the owner in at the pod with the
48
- // fediverse-account library, and proves that sign-in back to /oauth/authorize.
48
+ // fediverse-session library, and proves that sign-in back to /oauth/authorize.
49
49
  function sendPodSigninPage(res, client, webId, mount = '') {
50
50
  let asking = '';
51
51
  if (client && (client.name || client.redirect)) {
@@ -112,6 +112,9 @@ export async function handle(api, ctx) {
112
112
  };
113
113
  sched.push(entry);
114
114
  api.store.setScheduled(sched);
115
+ // Whatever keeps the time for this agent hears of it (a browser app
116
+ // tells its gateway, which publishes it if the app has closed by then).
117
+ api.agent.onScheduled?.();
115
118
  return send(200, api.scheduledJson(entry));
116
119
  }
117
120
  let note;
@@ -55,18 +55,22 @@ export class AcctFeed {
55
55
  });
56
56
  }
57
57
 
58
- start() {
58
+ // `lastSweptAt` and `onSwept`: see TagFeed.start — a restarted browser
59
+ // worker sweeps only when the interval is due.
60
+ start({ lastSweptAt = 0, onSwept = null } = {}) {
59
61
  this.stopped = false;
60
- this.sweep().catch(e => this.log(`acctfeed: ${e.message}`));
61
- const tick = () => {
62
- this.timer = setTimeout(() => {
63
- this.sweep()
64
- .catch(e => this.log(`acctfeed: ${e.message}`))
65
- .finally(() => { if (!this.stopped) tick(); });
66
- }, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
62
+ const sweep = () => this.sweep()
63
+ .catch(e => this.log(`acctfeed: ${e.message}`))
64
+ .finally(() => onSwept?.(Date.now()));
65
+ const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
66
+ const tick = (ms) => {
67
+ this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
67
68
  this.timer.unref?.();
68
69
  };
69
- tick();
70
+ const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
71
+ if (due > 0) { tick(due); return; }
72
+ sweep();
73
+ tick(every());
70
74
  }
71
75
 
72
76
  stop() { this.stopped = true; clearTimeout(this.timer); }
@@ -52,18 +52,22 @@ export class BskyFeed {
52
52
  return this.config();
53
53
  }
54
54
 
55
- start() {
55
+ // `lastSweptAt` and `onSwept`: see TagFeed.start — a restarted browser
56
+ // worker sweeps only when the interval is due.
57
+ start({ lastSweptAt = 0, onSwept = null } = {}) {
56
58
  this.stopped = false;
57
- this.sweep().catch(e => this.log(`bskyfeed: ${e.message}`));
58
- const tick = () => {
59
- this.timer = setTimeout(() => {
60
- this.sweep()
61
- .catch(e => this.log(`bskyfeed: ${e.message}`))
62
- .finally(() => { if (!this.stopped) tick(); });
63
- }, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
59
+ const sweep = () => this.sweep()
60
+ .catch(e => this.log(`bskyfeed: ${e.message}`))
61
+ .finally(() => onSwept?.(Date.now()));
62
+ const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
63
+ const tick = (ms) => {
64
+ this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
64
65
  this.timer.unref?.();
65
66
  };
66
- tick();
67
+ const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
68
+ if (due > 0) { tick(due); return; }
69
+ sweep();
70
+ tick(every());
67
71
  }
68
72
 
69
73
  stop() { this.stopped = true; clearTimeout(this.timer); }
@@ -72,20 +72,25 @@ export class TagFeed {
72
72
  return this.config();
73
73
  }
74
74
 
75
- start() {
75
+ // `lastSweptAt` and `onSwept` are for a browser worker the browser stops and
76
+ // restarts: it keeps the time of its last sweep, so a restart sweeps only
77
+ // when the interval is due rather than on every wake.
78
+ start({ lastSweptAt = 0, onSwept = null } = {}) {
76
79
  this.stopped = false; // restartable, the same way Intake.start is
77
- this.sweep().catch(e => this.log(`tagfeed: ${e.message}`));
80
+ const sweep = () => this.sweep()
81
+ .catch(e => this.log(`tagfeed: ${e.message}`))
82
+ .finally(() => onSwept?.(Date.now()));
78
83
  // Jittered and self-scheduling: every agent polling the same instance on
79
84
  // the same 15-minute boundary is a beat nobody asked for.
80
- const tick = () => {
81
- this.timer = setTimeout(() => {
82
- this.sweep()
83
- .catch(e => this.log(`tagfeed: ${e.message}`))
84
- .finally(() => { if (!this.stopped) tick(); });
85
- }, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
85
+ const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
86
+ const tick = (ms) => {
87
+ this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
86
88
  this.timer.unref?.();
87
89
  };
88
- tick();
90
+ const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
91
+ if (due > 0) { tick(due); return; }
92
+ sweep();
93
+ tick(every());
89
94
  }
90
95
 
91
96
  // The flag is what makes this stick. Clearing the timer only cancels a sweep
@@ -187,7 +187,21 @@ export class Deliverer {
187
187
  return out;
188
188
  }
189
189
 
190
+ // A fresh activity of this actor's, for the outbox its owner reads. Never
191
+ // allowed to stop a delivery; a retry from the queue is not fresh.
192
+ _sent(activity) {
193
+ try { this.onSent?.(activity); } catch (e) { this.log(`owner's outbox record: ${e.message}`); }
194
+ }
195
+
196
+ // An inbox on a blocked domain is never delivered to, whoever still lists it.
197
+ _blockedInbox(inbox) {
198
+ try { return !!this.store.getBlocklist?.().domains.length && this.store.isBlocked?.(new URL(inbox).origin + '/'); }
199
+ catch { return false; }
200
+ }
201
+
190
202
  async deliver(inbox, activity) {
203
+ this._sent(activity);
204
+ if (this._blockedInbox(inbox)) return;
191
205
  // Proved before anything else, so the copy that goes on the queue is the
192
206
  // copy that was signed — a retry days later must not post a bare activity.
193
207
  const signed = await this.proofed(activity);
@@ -250,13 +264,15 @@ export class Deliverer {
250
264
  }
251
265
 
252
266
  async deliverToAll(inboxes, activity) {
267
+ this._sent(activity);
253
268
  // Proved once for the whole fan-out: every recipient gets the same bytes,
254
269
  // and one signature is computed rather than one per follower.
255
270
  const signed = await this.proofed(activity);
256
271
  // Shared inboxes deduplicate fan-out to the same server. Sent in batches
257
272
  // of batchSize, and a host found cooling by one batch is not asked again
258
273
  // by the next.
259
- const targets = [...new Set(inboxes)].map((inbox) => ({ inbox, activity: signed }));
274
+ const targets = [...new Set(inboxes)].filter((inbox) => !this._blockedInbox(inbox))
275
+ .map((inbox) => ({ inbox, activity: signed }));
260
276
  for (let i = 0; i < targets.length; i += this.batchSize) {
261
277
  const chunk = targets.slice(i, i + this.batchSize).filter((t) => !this._queueIfCooling(t.inbox, signed));
262
278
  if (!chunk.length) continue;
@@ -5,6 +5,9 @@
5
5
  import { dropFollower } from '../store.mjs';
6
6
  import { emojisOf, pollOf, isContentType, trimActivity, reactionOf, quoteOf, quotePolicyOf, authorOf } from './activity.mjs';
7
7
 
8
+ // How long a follower's refollow waits before it is answered again.
9
+ const REACCEPT_MS = 60 * 60 * 1000;
10
+
8
11
  const idOf = (v) => (typeof v === 'string' ? v : v?.id);
9
12
 
10
13
  // §7.6 Add / §7.9 Remove. The side effect would be to add or remove the object
@@ -98,6 +101,18 @@ export async function onFollow(intake, activity, actor, { trusted = false } = {}
98
101
  await intake.republish({ followers: true });
99
102
  intake.log(`new follower: ${actor}`);
100
103
  }
104
+ // A refollow from someone already following is answered again — their server
105
+ // may have lost our Accept — but not every time: the same Follow sent over and
106
+ // over would otherwise make us send, and record, an Accept for each one.
107
+ if (existing) {
108
+ const c = intake.store.getContacts();
109
+ const f = c.followers.find(x => x.actor === actor);
110
+ if (f?.acceptedAt && Date.now() - Date.parse(f.acceptedAt) < REACCEPT_MS) {
111
+ intake.log(`refollow from ${actor} — already accepted within the hour, not answered again`);
112
+ return;
113
+ }
114
+ if (f) { f.acceptedAt = new Date().toISOString(); intake.store.setContacts(c); }
115
+ }
101
116
  const { acceptActivity } = await import('../wire.mjs');
102
117
  await intake.deliverer.deliver(doc.inbox,
103
118
  acceptActivity({ urls: intake.urls, followActivity: activity, serial: intake.serial++ }));
@@ -370,8 +385,9 @@ export async function onDelete(intake, activity, actor) {
370
385
  await intake.forget(s, { collect: retracted });
371
386
  }
372
387
  if (retracted.length) {
373
- const gone = new Set(retracted);
374
- await intake.publisher.unrecordOutbox(i => gone.has(i?.id));
388
+ const gone = new Set(retracted.map(u => u.object.id));
389
+ await intake.publisher.unrecordOutbox(i => gone.has(i?.id),
390
+ { record: (was) => (intake.config.private ? null : retracted.filter(u => was.includes(u.object.id))) });
375
391
  }
376
392
  // Both: an account deletion drops them from followers AND following.
377
393
  await intake.republish({ followers: true, following: true });
@@ -175,7 +175,9 @@ export async function amplify(intake, noteId, { approved = false, activity = nul
175
175
  // Marked carried before recorded: a failed outbox write costs one missing
176
176
  // entry, a failed status write would carry the same post twice.
177
177
  intake.store.updateStatus(noteId, { announcedAt: new Date().toISOString(), announceActivity: act });
178
- await intake.publisher.recordOutbox(act);
178
+ // A private group's carries are written for its members alone, so they go on
179
+ // its owner's full outbox (recorded as sent) and never on the public one.
180
+ if (!intake.config.private) await intake.publisher.recordOutbox(act);
179
181
  intake.store.setPending(intake.store.getPending().filter(p => p.noteId !== noteId));
180
182
  intake.log(`amplified ${noteId} → ${inboxes.length} inbox(es)`);
181
183
  // What the group carried is what the group's conversation holds. A forum
@@ -58,6 +58,11 @@ const ATTEMPTS_DOC = 'intake-attempts.json';
58
58
  const ATTEMPTS_TTL_MS = 7 * 24 * 60 * 60_000;
59
59
  const MAX_ITEM_ATTEMPTS = 5;
60
60
  const MAX_ITEMS_PER_DRAIN = 50;
61
+ // A batch the gateway wrote while the owner's app was closed: at most this many
62
+ // deliveries, read under this cap (lib/gateway/held-mail.mjs writes 100).
63
+ const MAX_BATCH_ENTRIES = 100;
64
+ const MAX_BATCH_BYTES = 8 * 1024 * 1024;
65
+ const isBatch = (url) => /\/batch-[^/]*\.json$/u.test(url);
61
66
 
62
67
  export class Intake {
63
68
  // `ownerPost(activity, { raw, slug })` is the client-to-server dispatcher,
@@ -88,8 +93,10 @@ export class Intake {
88
93
  // either case every item drained since the last successful write is gone,
89
94
  // and what goes is the mentions, replies, join requests and dead-letter
90
95
  // records that nothing else can rebuild.
96
+ // Against the store's generation when this sweep began: results dropped
97
+ // since (another agent acted, web/app/copy-mode.mjs) are not written down.
91
98
  async _persisted() {
92
- return this.store.commit();
99
+ return this.store.commit({ since: this._sweepGen });
93
100
  }
94
101
 
95
102
  _backOff(why) {
@@ -107,9 +114,11 @@ export class Intake {
107
114
  return this.wsState === 'open' || this.wsState === 'in-process' ? POLL_PUSH_OK_MS : POLL_MS;
108
115
  }
109
116
 
110
- async start() {
117
+ // `drainNow: false` and `subscribe: false` are for a browser worker the
118
+ // browser restarted: it drained moments ago, and a socket would die with it.
119
+ async start({ drainNow = true, subscribe = true } = {}) {
111
120
  this.stopped = false; // restartable across demote/takeover cycles
112
- await this.drain().catch(e => this.log(`drain: ${e.message}`));
121
+ if (drainNow) await this.drain().catch(e => this.log(`drain: ${e.message}`));
113
122
  const tick = () => {
114
123
  this.pollTimer = setTimeout(() => {
115
124
  this.drain().catch(e => this.log(`drain: ${e.message}`)).finally(() => { if (!this.stopped) tick(); });
@@ -119,8 +128,8 @@ export class Intake {
119
128
  tick();
120
129
  // Embedded in the pod server, a notification socket back to that same
121
130
  // server buys nothing — the store's own change events wake the drain.
122
- if (this.push) this.subscribe().catch(e => this.log(`subscribe: ${e.message}`));
123
- else this.wsState = 'in-process';
131
+ if (!this.push) this.wsState = 'in-process';
132
+ else if (subscribe) this.subscribe().catch(e => this.log(`subscribe: ${e.message}`));
124
133
  }
125
134
 
126
135
  stop() { this.stopped = true; clearTimeout(this.pollTimer); clearTimeout(this.resubTimer); this.ws?.close(); }
@@ -208,6 +217,7 @@ export class Intake {
208
217
  // us, a mention, a reply to ours, or from someone we follow — and drops the
209
218
  // rest.
210
219
  async prune({ before, keepConcerning = false } = {}) {
220
+ this._sweepGen = this.store.generation;
211
221
  const cutoff = Date.parse(before);
212
222
  if (!Number.isFinite(cutoff)) throw new Error(`"${before}" is not a date`);
213
223
  const all = await podInbox.list(this.remote, this.urls);
@@ -217,7 +227,29 @@ export class Intake {
217
227
 
218
228
  for (const item of older) {
219
229
  try {
220
- if (item.size > MAX_ITEM_BYTES) {
230
+ if (isBatch(item.url)) {
231
+ // Mail the gateway held while the app was closed: the same rule for
232
+ // each delivery in it, then the batch goes.
233
+ for (const e of (await this._batchEntries(item.url)) || []) {
234
+ const read = typeof e?.body === 'string' ? await readLenient(e.body) : null;
235
+ const activity = read?.view ?? read?.doc ?? null;
236
+ const receipt = activity ? await this._checkReceipt(e?.receipt) : null;
237
+ if (keepConcerning) {
238
+ const rejection = activity ? await this.handle(activity, receipt) : 'unparsable JSON';
239
+ if (rejection) out.dropped++; else out.applied++;
240
+ } else if (activity && activity.type !== 'Create') {
241
+ await this.handle(activity, receipt);
242
+ out.applied++;
243
+ } else {
244
+ out.dropped++;
245
+ }
246
+ }
247
+ if (!await this._persisted()) {
248
+ this.log('state not written — stopping the prune with a batch left');
249
+ break;
250
+ }
251
+ await podInbox.dropHandledItem(this.remote, item.url);
252
+ } else if (item.size > MAX_ITEM_BYTES) {
221
253
  await podInbox.dropHandledItem(this.remote, item.url); // unreadable by the drain either way
222
254
  out.discarded++;
223
255
  } else {
@@ -309,6 +341,7 @@ export class Intake {
309
341
  }
310
342
 
311
343
  async _drainOnce() {
344
+ this._sweepGen = this.store.generation;
312
345
  const cooling = this.drainCooldownUntil - Date.now();
313
346
  if (cooling > 0) {
314
347
  this.log(`inbox sweep skipped — backing off for another ${Math.ceil(cooling / 1000)}s`);
@@ -389,8 +422,29 @@ export class Intake {
389
422
  return true;
390
423
  };
391
424
 
425
+ let spent = 0; // deliveries taken this sweep, a batch counting each of its own
392
426
  for (const { url, size, receipt: hasReceipt } of items) {
393
427
  if (url.endsWith('.keep')) continue;
428
+ if (spent >= MAX_ITEMS_PER_DRAIN) { this._drainAgain = handled > 0 || pending.length > 0; break; }
429
+ if (isBatch(url)) {
430
+ try {
431
+ const took = await this._takeBatch(url);
432
+ spent += took.entries;
433
+ if (took.complete) pending.push(url);
434
+ } catch (e) {
435
+ // The batch itself could not be read: tried again next sweep, like
436
+ // any item, and dead-lettered only after as many tries.
437
+ const n = this._bumpAttempt(url, e.message);
438
+ this.log(`inbox batch ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
439
+ if (n >= MAX_ITEM_ATTEMPTS) {
440
+ this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: null });
441
+ pending.push(url);
442
+ }
443
+ }
444
+ if (pending.length >= DELETE_BATCH && !await flush()) return;
445
+ continue;
446
+ }
447
+ spent++;
394
448
  if (hasReceipt) withReceipt.add(url);
395
449
  // The listing already carries every child's size, so this costs nothing
396
450
  // to ask. An activity is a few kB; anything of this order is not one, and
@@ -404,7 +458,7 @@ export class Intake {
404
458
  pending.push(url);
405
459
  continue;
406
460
  }
407
- let activity = null;
461
+ const at = {};
408
462
  try {
409
463
  const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_ITEM_BYTES, readCapped });
410
464
  // readItem carries the rule that matters here: a pod that would not
@@ -413,56 +467,19 @@ export class Intake {
413
467
  // made it "unparsable JSON" — a REJECTION, dead-lettered with both
414
468
  // `activity` and `raw` null and then DELETEd, destroying a delivery on
415
469
  // a transient fault with no record of what it had been.
416
- const raw = got.raw;
417
- // Read as JSON-LD, so a document whose terms are aliased means what it
418
- // says. A document we cannot read that way is read the way it always
419
- // was rather than lost, and the reason is logged.
420
- const read = raw ? await readLenient(raw) : null;
421
- if (read?.degraded) this.log(`inbox item ${url} grounded to read: ${read.degraded}`);
422
- activity = read?.view ?? read?.doc ?? null;
423
- // What the shapes make of it. This decides NOTHING — the activity is
424
- // handled either way. It is written down so that which shapes fire on
425
- // real traffic is a question with an answer, rather than a guess made
426
- // in advance about implementations we have not met.
427
- if (read?.graph) {
428
- const failure = await checkShapes(read.graph);
429
- if (failure) {
430
- const said = describeShapeFailure(failure);
431
- this.log(`inbox item ${url} does not fit its shape: ${said}`);
432
- this.store.addDeadLetter({
433
- inboxUrl: url, reason: `shape: ${said}`, shapeOnly: true, activity: trimActivity(activity),
434
- });
435
- }
436
- }
437
470
  // A gateway that verified this delivery left a receipt beside it. Read
438
471
  // it only when a gateway is configured (no config → no fetch, so an
439
472
  // install with no gateway pays nothing) and the listing showed one
440
473
  // beside this item (`false` is a definite no; a listing that does not
441
474
  // say leaves it to the read); a missing or HMAC-invalid receipt reads
442
475
  // as null, which is exactly today's unverified behavior.
443
- const receipt = activity && hasReceipt !== false ? await this._readReceipt(url) : null;
444
- if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
445
- // The owner's own post, taken at the outbox door: not mail to read
446
- // but a write to make. Never archived or forwarded as if received.
447
- const owned = activity && this.isOwnerPost(receipt);
448
- const rejection = !activity ? 'unparsable JSON'
449
- : owned ? await this.ownerPostFrom(activity, raw, receipt)
450
- : await this.handle(activity, receipt);
451
- if (!rejection && raw && !owned) await this._archive(url, raw, activity);
452
- if (!rejection && !owned) await this._maybeForward(activity); // §7.1.2, only what we accepted
453
- if (rejection) {
454
- this.store.addDeadLetter({
455
- inboxUrl: url, reason: rejection, activity: trimActivity(activity),
456
- ...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
457
- });
458
- this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
459
- }
476
+ await this._take(url, got.raw, () => (hasReceipt !== false ? this._readReceipt(url) : null), at);
460
477
  pending.push(url);
461
478
  } catch (e) {
462
479
  const n = this._bumpAttempt(url, e.message);
463
480
  this.log(`inbox item ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
464
481
  if (n >= MAX_ITEM_ATTEMPTS) {
465
- this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(activity) });
482
+ this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(at.activity) });
466
483
  // The dead letter IS the record of this item — deleting before it is
467
484
  // written down would lose the only evidence it ever arrived, so it
468
485
  // goes through the same commit-then-delete batch as everything else.
@@ -483,6 +500,135 @@ export class Intake {
483
500
  if (handled > 0 && all.length > items.length && !this.stopped) this._drainAgain = true;
484
501
  }
485
502
 
503
+ // One delivery, alone in the inbox or in a batch: read as JSON-LD, checked
504
+ // against its receipt, handled, and dead-lettered if refused. Throws when it
505
+ // could not be handled this time; `at.activity` then says what it was.
506
+ async _take(url, raw, receiptOf, at = {}) {
507
+ // Read as JSON-LD, so a document whose terms are aliased means what it
508
+ // says. A document we cannot read that way is read the way it always
509
+ // was rather than lost, and the reason is logged.
510
+ const read = raw ? await readLenient(raw) : null;
511
+ if (read?.degraded) this.log(`inbox item ${url} grounded to read: ${read.degraded}`);
512
+ const activity = at.activity = read?.view ?? read?.doc ?? null;
513
+ // What the shapes make of it. This decides NOTHING — the activity is
514
+ // handled either way. It is written down so that which shapes fire on
515
+ // real traffic is a question with an answer, rather than a guess made
516
+ // in advance about implementations we have not met.
517
+ if (read?.graph) {
518
+ const failure = await checkShapes(read.graph);
519
+ if (failure) {
520
+ const said = describeShapeFailure(failure);
521
+ this.log(`inbox item ${url} does not fit its shape: ${said}`);
522
+ this.store.addDeadLetter({
523
+ inboxUrl: url, reason: `shape: ${said}`, shapeOnly: true, activity: trimActivity(activity),
524
+ });
525
+ }
526
+ }
527
+ const receipt = activity ? await receiptOf() : null;
528
+ if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
529
+ // The owner's own post, taken at the outbox door: not mail to read
530
+ // but a write to make. Never archived or forwarded as if received.
531
+ const owned = activity && this.isOwnerPost(receipt);
532
+ const rejection = !activity ? 'unparsable JSON'
533
+ : owned ? await this.ownerPostFrom(activity, raw, receipt)
534
+ : await this.handle(activity, receipt);
535
+ if (!rejection && raw && !owned) await this._archive(url, raw, activity);
536
+ if (!rejection && !owned) await this._maybeForward(activity); // §7.1.2, only what we accepted
537
+ if (rejection) {
538
+ this.store.addDeadLetter({
539
+ inboxUrl: url, reason: rejection, activity: trimActivity(activity),
540
+ ...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
541
+ });
542
+ this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
543
+ }
544
+ }
545
+
546
+ // The deliveries in a batch the gateway wrote while the owner's app was
547
+ // closed (lib/gateway/held-mail.mjs), or null when it holds none. Throws when
548
+ // the pod would not give it, like readItem.
549
+ async _batchEntries(url) {
550
+ const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_BATCH_BYTES, readCapped });
551
+ if (got.raw === null) return [];
552
+ let batch = null;
553
+ try { batch = JSON.parse(got.raw)?.batch; } catch { batch = null; }
554
+ return Array.isArray(batch) ? batch.slice(0, MAX_BATCH_ENTRIES) : null;
555
+ }
556
+
557
+ // Each delivery in a batch, taken as if it had arrived alone. `complete` is
558
+ // false while some entry is to be tried again: the batch then stays, and the
559
+ // entries already handled are handled again next time, harmlessly, as a
560
+ // re-delivered activity is.
561
+ async _takeBatch(url) {
562
+ const batch = await this._batchEntries(url);
563
+ if (!batch) {
564
+ this.store.addDeadLetter({ inboxUrl: url, reason: 'unreadable batch', activity: null });
565
+ return { entries: 1, complete: true };
566
+ }
567
+ let complete = true;
568
+ for (const [i, e] of batch.entries()) {
569
+ const key = `${url}#${e?.name || i}`;
570
+ const body = typeof e?.body === 'string' ? e.body : null;
571
+ if (body !== null && Buffer.byteLength(body) > MAX_ITEM_BYTES) {
572
+ this.store.addDeadLetter({ inboxUrl: key, reason: `oversized (${Buffer.byteLength(body)} bytes)`, activity: null });
573
+ continue;
574
+ }
575
+ const at = {};
576
+ try {
577
+ await this._take(key, body, () => this._checkReceipt(e?.receipt), at);
578
+ this._clearAttempt(key);
579
+ } catch (err) {
580
+ const n = this._bumpAttempt(key, err.message);
581
+ this.log(`inbox batch entry ${key} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${err.message}`);
582
+ if (n >= MAX_ITEM_ATTEMPTS) {
583
+ this.store.addDeadLetter({ inboxUrl: key, reason: `failed ${n}x: ${err.message}`, activity: trimActivity(at.activity) });
584
+ } else {
585
+ complete = false;
586
+ }
587
+ }
588
+ }
589
+ return { entries: batch.length, complete };
590
+ }
591
+
592
+ /**
593
+ * Deliveries handed straight in rather than read from the pod's inbox: mail
594
+ * the gateway held for an account whose copy it keeps (lib/gateway/keeper.mjs).
595
+ * `entries` are { name, body, receipt }, each taken as if it had arrived
596
+ * alone. Returns the names that are done with — handled, or refused for good
597
+ * — once what they changed is written; one to be tried again is left out, so
598
+ * whoever holds it keeps it for next time.
599
+ */
600
+ async takeHeld(entries) {
601
+ const since = this.store.generation;
602
+ const done = [];
603
+ this.store.hold?.();
604
+ try {
605
+ for (const e of entries) {
606
+ const key = `${this.urls.inbox}${e.name}`;
607
+ const body = typeof e.body === 'string' ? e.body : null;
608
+ if (body !== null && Buffer.byteLength(body) > MAX_ITEM_BYTES) {
609
+ this.store.addDeadLetter({ inboxUrl: key, reason: `oversized (${Buffer.byteLength(body)} bytes)`, activity: null });
610
+ done.push(e.name);
611
+ continue;
612
+ }
613
+ const at = {};
614
+ try {
615
+ await this._take(key, body, () => this._checkReceipt(e.receipt), at);
616
+ this._clearAttempt(key);
617
+ done.push(e.name);
618
+ } catch (err) {
619
+ const n = this._bumpAttempt(key, err.message);
620
+ this.log(`held delivery ${key} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${err.message}`);
621
+ if (n >= MAX_ITEM_ATTEMPTS) {
622
+ this.store.addDeadLetter({ inboxUrl: key, reason: `failed ${n}x: ${err.message}`, activity: trimActivity(at.activity) });
623
+ done.push(e.name);
624
+ }
625
+ }
626
+ }
627
+ await this._publishPending();
628
+ } finally { this.store.release?.(); }
629
+ return (await this.store.commit({ since })) ? done : [];
630
+ }
631
+
486
632
  // The end of a sweep: publish whatever the follow graph did ONCE, then flush.
487
633
  //
488
634
  // publishCollections used to run per handled item — every Follow, Undo,
@@ -643,6 +789,7 @@ export class Intake {
643
789
  gatewaySecret(...a) { return verify.gatewaySecret(this, ...a); }
644
790
  _bumpGatewayStat(...a) { return verify.bumpGatewayStat(this, ...a); }
645
791
  _readReceipt(...a) { return verify.readReceipt(this, ...a); }
792
+ _checkReceipt(...a) { return verify.checkReceipt(this, ...a); }
646
793
 
647
794
  // A receipt the door stamped `c2s` for THIS actor, and only a receipt whose
648
795
  // HMAC verified (readReceipt returns nothing else). A stranger appending an
@@ -656,10 +803,33 @@ export class Intake {
656
803
  // document, not our reading of it); the graph is what it decides from.
657
804
  // Returns null when published, else the reason it was not — a dead letter.
658
805
  async ownerPostFrom(activity, raw, receipt) {
806
+ // The receipt names the body it vouches for; a body it does not name is
807
+ // not the owner's. A body already handled — a replayed request, or an item
808
+ // the pod would not let us delete — is not published twice.
809
+ const hash = raw ? await sha256Hex(raw) : null;
810
+ if (receipt.hash && receipt.hash !== hash) return 'owner post refused: the body is not the one its receipt names';
811
+ const seen = this.store.read('c2s-seen.json', []);
812
+ if (hash && seen.includes(hash)) {
813
+ this.log('owner post already handled — not published again');
814
+ return null;
815
+ }
659
816
  let asSent = null;
660
817
  try { asSent = raw ? JSON.parse(raw) : null; } catch { asSent = null; }
661
- const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null });
662
- if (!r || r.status >= 300) return `owner post refused (${r?.status || '?'}): ${r?.body?.error || ''}`;
818
+ const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null,
819
+ // What the door told the client this will be called, when it said.
820
+ ...(receipt.serial ? { serial: receipt.serial } : {}), ...(receipt.at ? { at: receipt.at } : {}) });
821
+ // Handled, one way or the other: a second sight of it changes nothing. A
822
+ // throw above is not handled, and is tried again.
823
+ if (hash) this.store.write('c2s-seen.json', [hash, ...this.store.read('c2s-seen.json', [])].slice(0, C2S_SEEN_MAX));
824
+ if (!r || r.status >= 300) {
825
+ const why = r?.body?.error || 'no reason was given';
826
+ // The door already answered the app "created"; the owner is the one
827
+ // left to tell.
828
+ await this.publisher.noteToSelf?.(`An app asked your account to publish ${article(activity.type)}, `
829
+ + `and your account refused it: ${why}. Nothing was sent.`)
830
+ .catch(e => this.log(`could not tell the owner their post was refused: ${e.message}`));
831
+ return `owner post refused (${r?.status || '?'}): ${why}`;
832
+ }
663
833
  this.log(`owner post from the outbox door published: ${r.body?.object || r.body?.id || activity.type}`);
664
834
  return null;
665
835
  }
@@ -699,3 +869,11 @@ export class Intake {
699
869
  retract(...a) { return notes.retract(this, ...a); }
700
870
  addReply(...a) { return notes.addReply(this, ...a); }
701
871
  }
872
+
873
+ // "a Like", "an Update": the kind of activity, as a sentence says it.
874
+ const article = (type) => `${/^[AEIOU]/u.test(String(type)) ? 'an' : 'a'} ${type || 'post'}`;
875
+
876
+ // Bodies the outbox door handed over that were handled, newest first.
877
+ const C2S_SEEN_MAX = 500;
878
+ const sha256Hex = async (text) => [...new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256',
879
+ new TextEncoder().encode(text)))].map(b => b.toString(16).padStart(2, '0')).join('');
@@ -91,7 +91,11 @@ export async function maybeForward(intake, activity) {
91
91
  .filter(Boolean))];
92
92
  if (!inboxes.length) return;
93
93
  intake._forwardBudget -= 1;
94
- await intake.deliverer.deliverToAll(inboxes, activity);
94
+ // Blind copies are for their sender's delivery alone and never travel on
95
+ // (§6, §7.1), from the activity or its object.
96
+ const strip = (o) => { if (!o || typeof o !== 'object') return o; const { bto, bcc, ...rest } = o; return rest; }; // eslint-disable-line no-unused-vars
97
+ const onward = { ...strip(activity), ...(activity.object && typeof activity.object === 'object' ? { object: strip(activity.object) } : {}) };
98
+ await intake.deliverer.deliverToAll(inboxes, onward);
95
99
  intake.store.write('forwarded.json', [...forwarded, id].slice(-MAX_FORWARDED));
96
100
  intake.log(`forwarded ${activity.type} ${id} to ${inboxes.length} follower inbox(es)`);
97
101
  } catch (e) {
@@ -319,10 +323,13 @@ export async function retract(intake, noteId, { collect = null } = {}) {
319
323
  if (!s.announceActivity) throw new Error('that post was never carried');
320
324
  const { undoActivity } = await import('../wire.mjs');
321
325
  const inboxes = intake.announceTargets(s.actor);
322
- await intake.deliverer.deliverToAll(inboxes,
323
- undoActivity({ urls: intake.urls, activity: s.announceActivity, serial: intake.serial++ }));
324
- if (collect) collect.push(s.announceActivity.id);
325
- else await intake.publisher.unrecordOutbox(i => i?.id === s.announceActivity.id);
326
+ const undo = undoActivity({ urls: intake.urls, activity: s.announceActivity, serial: intake.serial++ });
327
+ await intake.deliverer.deliverToAll(inboxes, undo);
328
+ // The Undo takes the carry's place in the outbox, so a reader catching up
329
+ // from it learns the carry was withdrawn.
330
+ if (collect) collect.push(undo);
331
+ else await intake.publisher.unrecordOutbox(i => i?.id === s.announceActivity.id,
332
+ { record: () => (intake.config.private ? null : undo) });
326
333
  intake.store.updateStatus(noteId, {
327
334
  announcedAt: undefined, announceActivity: undefined, retractedAt: new Date().toISOString(),
328
335
  });