fedipod 1.36.6 → 1.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -8
- package/architecture.md +8 -0
- package/bin/fedipod.mjs +6 -0
- package/cli.md +7 -1
- package/device-agent.md +3 -3
- package/gateway.md +69 -8
- package/groups.md +2 -1
- package/gui.md +3 -1
- package/lib/client/c2s.mjs +97 -43
- package/lib/client/masto/accounts.mjs +1 -0
- package/lib/client/masto/bridge.mjs +61 -0
- package/lib/client/masto/index.mjs +4 -1
- package/lib/client/masto/oauth.mjs +1 -1
- package/lib/client/masto/statuses.mjs +3 -0
- package/lib/connections/acctfeed.mjs +13 -9
- package/lib/connections/bskyfeed.mjs +13 -9
- package/lib/connections/tagfeed.mjs +14 -9
- package/lib/core/deliver.mjs +17 -1
- package/lib/core/intake/activities.mjs +18 -2
- package/lib/core/intake/group.mjs +3 -1
- package/lib/core/intake/index.mjs +226 -48
- package/lib/core/intake/notes.mjs +12 -5
- package/lib/core/intake/verify.mjs +11 -0
- package/lib/core/lease.mjs +15 -1
- package/lib/core/place.mjs +82 -0
- package/lib/core/pod-only.mjs +4 -0
- package/lib/core/publisher/collections.mjs +37 -2
- package/lib/core/publisher/index.mjs +25 -1
- package/lib/core/publisher/notes.mjs +73 -7
- package/lib/core/publisher/own.mjs +143 -0
- package/lib/core/publisher/questions.mjs +5 -3
- package/lib/core/scheduled.mjs +39 -0
- package/lib/core/social.mjs +62 -32
- package/lib/core/storage.mjs +67 -0
- package/lib/core/store.mjs +77 -5
- package/lib/core/wire.mjs +51 -15
- package/lib/device/admin/routes/lifecycle.mjs +1 -1
- package/lib/device/admin/routes/setup.mjs +17 -1
- package/lib/device/cli/commands/setup.mjs +50 -8
- package/lib/device/cli/context.mjs +1 -1
- package/lib/device/migrate.mjs +1 -1
- package/lib/device/setup.mjs +45 -8
- package/lib/gateway/account-agent.mjs +111 -0
- package/lib/gateway/copy.mjs +381 -0
- package/lib/gateway/front-core.mjs +74 -63
- package/lib/gateway/gateway-core.mjs +107 -9
- package/lib/gateway/held-mail.mjs +198 -0
- package/lib/gateway/keeper-due.mjs +39 -0
- package/lib/gateway/keeper-session.mjs +10 -0
- package/lib/gateway/keeper.mjs +72 -0
- package/lib/gateway/masto-gateway.mjs +510 -0
- package/lib/gateway/quiet.mjs +7 -2
- package/lib/gateway/relay-extras.mjs +89 -0
- package/lib/gateway/state-api.mjs +207 -0
- package/lib/gateway/token-claims.mjs +16 -0
- package/lib/pod/containers.mjs +17 -0
- package/lib/pod/location.mjs +52 -0
- package/lib/pod/notes.mjs +2 -4
- package/lib/pod/transport.mjs +214 -24
- package/lib/pod/type-index.mjs +101 -0
- package/lib/pod/urls.mjs +6 -0
- package/lib/server/embed.mjs +7 -6
- package/lib/session/README.md +5 -5
- package/lib/session/demo.html +1 -1
- package/lib/session/fedi-account.mjs +19 -10
- package/lib/session/package.json +2 -2
- package/package.json +2 -2
- package/run-agent.mjs +6 -16
- package/scripts/stage-site.mjs +17 -4
- package/web/admin/actors.js +2 -0
- package/web/admin/gateway.js +14 -1
- package/web/admin/index.html +13 -0
- package/web/admin/oauth-signin.mjs +1 -1
- package/web/admin/record.js +4 -1
- package/web/admin/setup/index.html +17 -1
- package/web/admin/setup/setup.js +18 -5
- package/web/app/README.md +2 -2
- package/web/app/admin-facade.mjs +12 -2
- package/web/app/agent.mjs +228 -51
- package/web/app/boot.mjs +75 -32
- package/web/app/copy-mode.mjs +223 -0
- package/web/app/dist/boot.js +498 -88
- package/web/app/dist/boot.js.map +4 -4
- package/web/app/dist/sw.js +4016 -2539
- package/web/app/dist/sw.js.map +4 -4
- package/web/app/index.html +16 -0
- package/web/app/signup.mjs +65 -26
- package/web/app/sw-src.mjs +15 -53
- package/web/app/update.js +3 -2
- package/web/app/warm-start.mjs +115 -0
- package/web/app-signin/app-signin.mjs +78 -0
- package/web/app-signin/index.html +41 -0
- package/web/front/run.html +7 -1
- package/web/front/run.js +30 -4
|
@@ -149,7 +149,10 @@ export class MastoApi {
|
|
|
149
149
|
// A viewer-mode agent (another agent holds the drain lease) may not act —
|
|
150
150
|
// but a user acting HERE outranks the idle active agent elsewhere, so a
|
|
151
151
|
// write attempt claims the lease and proceeds. Only a failed claim 503s.
|
|
152
|
-
|
|
152
|
+
// An agent working from a copy its gateway shares (the browser build,
|
|
153
|
+
// web/app/copy-mode.mjs) asks every time, since an app at the gateway may
|
|
154
|
+
// have taken the lease without it hearing yet.
|
|
155
|
+
if ((this.agent.viewer || this.agent.copy) && req.method !== 'GET' && req.method !== 'HEAD') {
|
|
153
156
|
const took = await this.agent.requestTakeover?.();
|
|
154
157
|
if (!took) return send(503, { error: 'another agent is active for this pod — takeover failed, try again' });
|
|
155
158
|
}
|
|
@@ -45,7 +45,7 @@ const parseRedirects = (v) => (Array.isArray(v) ? v : String(v || '').split(/\s+
|
|
|
45
45
|
// The sign-in page for an account whose owner signs in at their pod: no
|
|
46
46
|
// password of ours exists, and none is asked for. The page carries the
|
|
47
47
|
// client's request in its address, signs the owner in at the pod with the
|
|
48
|
-
// fediverse-
|
|
48
|
+
// fediverse-session library, and proves that sign-in back to /oauth/authorize.
|
|
49
49
|
function sendPodSigninPage(res, client, webId, mount = '') {
|
|
50
50
|
let asking = '';
|
|
51
51
|
if (client && (client.name || client.redirect)) {
|
|
@@ -112,6 +112,9 @@ export async function handle(api, ctx) {
|
|
|
112
112
|
};
|
|
113
113
|
sched.push(entry);
|
|
114
114
|
api.store.setScheduled(sched);
|
|
115
|
+
// Whatever keeps the time for this agent hears of it (a browser app
|
|
116
|
+
// tells its gateway, which publishes it if the app has closed by then).
|
|
117
|
+
api.agent.onScheduled?.();
|
|
115
118
|
return send(200, api.scheduledJson(entry));
|
|
116
119
|
}
|
|
117
120
|
let note;
|
|
@@ -55,18 +55,22 @@ export class AcctFeed {
|
|
|
55
55
|
});
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
start
|
|
58
|
+
// `lastSweptAt` and `onSwept`: see TagFeed.start — a restarted browser
|
|
59
|
+
// worker sweeps only when the interval is due.
|
|
60
|
+
start({ lastSweptAt = 0, onSwept = null } = {}) {
|
|
59
61
|
this.stopped = false;
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
}, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
|
|
62
|
+
const sweep = () => this.sweep()
|
|
63
|
+
.catch(e => this.log(`acctfeed: ${e.message}`))
|
|
64
|
+
.finally(() => onSwept?.(Date.now()));
|
|
65
|
+
const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
|
|
66
|
+
const tick = (ms) => {
|
|
67
|
+
this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
|
|
67
68
|
this.timer.unref?.();
|
|
68
69
|
};
|
|
69
|
-
|
|
70
|
+
const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
|
|
71
|
+
if (due > 0) { tick(due); return; }
|
|
72
|
+
sweep();
|
|
73
|
+
tick(every());
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
stop() { this.stopped = true; clearTimeout(this.timer); }
|
|
@@ -52,18 +52,22 @@ export class BskyFeed {
|
|
|
52
52
|
return this.config();
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
-
start
|
|
55
|
+
// `lastSweptAt` and `onSwept`: see TagFeed.start — a restarted browser
|
|
56
|
+
// worker sweeps only when the interval is due.
|
|
57
|
+
start({ lastSweptAt = 0, onSwept = null } = {}) {
|
|
56
58
|
this.stopped = false;
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
}, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
|
|
59
|
+
const sweep = () => this.sweep()
|
|
60
|
+
.catch(e => this.log(`bskyfeed: ${e.message}`))
|
|
61
|
+
.finally(() => onSwept?.(Date.now()));
|
|
62
|
+
const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
|
|
63
|
+
const tick = (ms) => {
|
|
64
|
+
this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
|
|
64
65
|
this.timer.unref?.();
|
|
65
66
|
};
|
|
66
|
-
|
|
67
|
+
const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
|
|
68
|
+
if (due > 0) { tick(due); return; }
|
|
69
|
+
sweep();
|
|
70
|
+
tick(every());
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
stop() { this.stopped = true; clearTimeout(this.timer); }
|
|
@@ -72,20 +72,25 @@ export class TagFeed {
|
|
|
72
72
|
return this.config();
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
|
|
75
|
+
// `lastSweptAt` and `onSwept` are for a browser worker the browser stops and
|
|
76
|
+
// restarts: it keeps the time of its last sweep, so a restart sweeps only
|
|
77
|
+
// when the interval is due rather than on every wake.
|
|
78
|
+
start({ lastSweptAt = 0, onSwept = null } = {}) {
|
|
76
79
|
this.stopped = false; // restartable, the same way Intake.start is
|
|
77
|
-
|
|
80
|
+
const sweep = () => this.sweep()
|
|
81
|
+
.catch(e => this.log(`tagfeed: ${e.message}`))
|
|
82
|
+
.finally(() => onSwept?.(Date.now()));
|
|
78
83
|
// Jittered and self-scheduling: every agent polling the same instance on
|
|
79
84
|
// the same 15-minute boundary is a beat nobody asked for.
|
|
80
|
-
const
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
.catch(e => this.log(`tagfeed: ${e.message}`))
|
|
84
|
-
.finally(() => { if (!this.stopped) tick(); });
|
|
85
|
-
}, Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3)));
|
|
85
|
+
const every = () => Math.round(this.config().intervalMin * 60_000 * (0.85 + Math.random() * 0.3));
|
|
86
|
+
const tick = (ms) => {
|
|
87
|
+
this.timer = setTimeout(() => { sweep().finally(() => { if (!this.stopped) tick(every()); }); }, ms);
|
|
86
88
|
this.timer.unref?.();
|
|
87
89
|
};
|
|
88
|
-
|
|
90
|
+
const due = lastSweptAt + this.config().intervalMin * 60_000 - Date.now();
|
|
91
|
+
if (due > 0) { tick(due); return; }
|
|
92
|
+
sweep();
|
|
93
|
+
tick(every());
|
|
89
94
|
}
|
|
90
95
|
|
|
91
96
|
// The flag is what makes this stick. Clearing the timer only cancels a sweep
|
package/lib/core/deliver.mjs
CHANGED
|
@@ -187,7 +187,21 @@ export class Deliverer {
|
|
|
187
187
|
return out;
|
|
188
188
|
}
|
|
189
189
|
|
|
190
|
+
// A fresh activity of this actor's, for the outbox its owner reads. Never
|
|
191
|
+
// allowed to stop a delivery; a retry from the queue is not fresh.
|
|
192
|
+
_sent(activity) {
|
|
193
|
+
try { this.onSent?.(activity); } catch (e) { this.log(`owner's outbox record: ${e.message}`); }
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// An inbox on a blocked domain is never delivered to, whoever still lists it.
|
|
197
|
+
_blockedInbox(inbox) {
|
|
198
|
+
try { return !!this.store.getBlocklist?.().domains.length && this.store.isBlocked?.(new URL(inbox).origin + '/'); }
|
|
199
|
+
catch { return false; }
|
|
200
|
+
}
|
|
201
|
+
|
|
190
202
|
async deliver(inbox, activity) {
|
|
203
|
+
this._sent(activity);
|
|
204
|
+
if (this._blockedInbox(inbox)) return;
|
|
191
205
|
// Proved before anything else, so the copy that goes on the queue is the
|
|
192
206
|
// copy that was signed — a retry days later must not post a bare activity.
|
|
193
207
|
const signed = await this.proofed(activity);
|
|
@@ -250,13 +264,15 @@ export class Deliverer {
|
|
|
250
264
|
}
|
|
251
265
|
|
|
252
266
|
async deliverToAll(inboxes, activity) {
|
|
267
|
+
this._sent(activity);
|
|
253
268
|
// Proved once for the whole fan-out: every recipient gets the same bytes,
|
|
254
269
|
// and one signature is computed rather than one per follower.
|
|
255
270
|
const signed = await this.proofed(activity);
|
|
256
271
|
// Shared inboxes deduplicate fan-out to the same server. Sent in batches
|
|
257
272
|
// of batchSize, and a host found cooling by one batch is not asked again
|
|
258
273
|
// by the next.
|
|
259
|
-
const targets = [...new Set(inboxes)].
|
|
274
|
+
const targets = [...new Set(inboxes)].filter((inbox) => !this._blockedInbox(inbox))
|
|
275
|
+
.map((inbox) => ({ inbox, activity: signed }));
|
|
260
276
|
for (let i = 0; i < targets.length; i += this.batchSize) {
|
|
261
277
|
const chunk = targets.slice(i, i + this.batchSize).filter((t) => !this._queueIfCooling(t.inbox, signed));
|
|
262
278
|
if (!chunk.length) continue;
|
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
import { dropFollower } from '../store.mjs';
|
|
6
6
|
import { emojisOf, pollOf, isContentType, trimActivity, reactionOf, quoteOf, quotePolicyOf, authorOf } from './activity.mjs';
|
|
7
7
|
|
|
8
|
+
// How long a follower's refollow waits before it is answered again.
|
|
9
|
+
const REACCEPT_MS = 60 * 60 * 1000;
|
|
10
|
+
|
|
8
11
|
const idOf = (v) => (typeof v === 'string' ? v : v?.id);
|
|
9
12
|
|
|
10
13
|
// §7.6 Add / §7.9 Remove. The side effect would be to add or remove the object
|
|
@@ -98,6 +101,18 @@ export async function onFollow(intake, activity, actor, { trusted = false } = {}
|
|
|
98
101
|
await intake.republish({ followers: true });
|
|
99
102
|
intake.log(`new follower: ${actor}`);
|
|
100
103
|
}
|
|
104
|
+
// A refollow from someone already following is answered again — their server
|
|
105
|
+
// may have lost our Accept — but not every time: the same Follow sent over and
|
|
106
|
+
// over would otherwise make us send, and record, an Accept for each one.
|
|
107
|
+
if (existing) {
|
|
108
|
+
const c = intake.store.getContacts();
|
|
109
|
+
const f = c.followers.find(x => x.actor === actor);
|
|
110
|
+
if (f?.acceptedAt && Date.now() - Date.parse(f.acceptedAt) < REACCEPT_MS) {
|
|
111
|
+
intake.log(`refollow from ${actor} — already accepted within the hour, not answered again`);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (f) { f.acceptedAt = new Date().toISOString(); intake.store.setContacts(c); }
|
|
115
|
+
}
|
|
101
116
|
const { acceptActivity } = await import('../wire.mjs');
|
|
102
117
|
await intake.deliverer.deliver(doc.inbox,
|
|
103
118
|
acceptActivity({ urls: intake.urls, followActivity: activity, serial: intake.serial++ }));
|
|
@@ -370,8 +385,9 @@ export async function onDelete(intake, activity, actor) {
|
|
|
370
385
|
await intake.forget(s, { collect: retracted });
|
|
371
386
|
}
|
|
372
387
|
if (retracted.length) {
|
|
373
|
-
const gone = new Set(retracted);
|
|
374
|
-
await intake.publisher.unrecordOutbox(i => gone.has(i?.id)
|
|
388
|
+
const gone = new Set(retracted.map(u => u.object.id));
|
|
389
|
+
await intake.publisher.unrecordOutbox(i => gone.has(i?.id),
|
|
390
|
+
{ record: (was) => (intake.config.private ? null : retracted.filter(u => was.includes(u.object.id))) });
|
|
375
391
|
}
|
|
376
392
|
// Both: an account deletion drops them from followers AND following.
|
|
377
393
|
await intake.republish({ followers: true, following: true });
|
|
@@ -175,7 +175,9 @@ export async function amplify(intake, noteId, { approved = false, activity = nul
|
|
|
175
175
|
// Marked carried before recorded: a failed outbox write costs one missing
|
|
176
176
|
// entry, a failed status write would carry the same post twice.
|
|
177
177
|
intake.store.updateStatus(noteId, { announcedAt: new Date().toISOString(), announceActivity: act });
|
|
178
|
-
|
|
178
|
+
// A private group's carries are written for its members alone, so they go on
|
|
179
|
+
// its owner's full outbox (recorded as sent) and never on the public one.
|
|
180
|
+
if (!intake.config.private) await intake.publisher.recordOutbox(act);
|
|
179
181
|
intake.store.setPending(intake.store.getPending().filter(p => p.noteId !== noteId));
|
|
180
182
|
intake.log(`amplified ${noteId} → ${inboxes.length} inbox(es)`);
|
|
181
183
|
// What the group carried is what the group's conversation holds. A forum
|
|
@@ -58,6 +58,11 @@ const ATTEMPTS_DOC = 'intake-attempts.json';
|
|
|
58
58
|
const ATTEMPTS_TTL_MS = 7 * 24 * 60 * 60_000;
|
|
59
59
|
const MAX_ITEM_ATTEMPTS = 5;
|
|
60
60
|
const MAX_ITEMS_PER_DRAIN = 50;
|
|
61
|
+
// A batch the gateway wrote while the owner's app was closed: at most this many
|
|
62
|
+
// deliveries, read under this cap (lib/gateway/held-mail.mjs writes 100).
|
|
63
|
+
const MAX_BATCH_ENTRIES = 100;
|
|
64
|
+
const MAX_BATCH_BYTES = 8 * 1024 * 1024;
|
|
65
|
+
const isBatch = (url) => /\/batch-[^/]*\.json$/u.test(url);
|
|
61
66
|
|
|
62
67
|
export class Intake {
|
|
63
68
|
// `ownerPost(activity, { raw, slug })` is the client-to-server dispatcher,
|
|
@@ -88,8 +93,10 @@ export class Intake {
|
|
|
88
93
|
// either case every item drained since the last successful write is gone,
|
|
89
94
|
// and what goes is the mentions, replies, join requests and dead-letter
|
|
90
95
|
// records that nothing else can rebuild.
|
|
96
|
+
// Against the store's generation when this sweep began: results dropped
|
|
97
|
+
// since (another agent acted, web/app/copy-mode.mjs) are not written down.
|
|
91
98
|
async _persisted() {
|
|
92
|
-
return this.store.commit();
|
|
99
|
+
return this.store.commit({ since: this._sweepGen });
|
|
93
100
|
}
|
|
94
101
|
|
|
95
102
|
_backOff(why) {
|
|
@@ -107,9 +114,11 @@ export class Intake {
|
|
|
107
114
|
return this.wsState === 'open' || this.wsState === 'in-process' ? POLL_PUSH_OK_MS : POLL_MS;
|
|
108
115
|
}
|
|
109
116
|
|
|
110
|
-
|
|
117
|
+
// `drainNow: false` and `subscribe: false` are for a browser worker the
|
|
118
|
+
// browser restarted: it drained moments ago, and a socket would die with it.
|
|
119
|
+
async start({ drainNow = true, subscribe = true } = {}) {
|
|
111
120
|
this.stopped = false; // restartable across demote/takeover cycles
|
|
112
|
-
await this.drain().catch(e => this.log(`drain: ${e.message}`));
|
|
121
|
+
if (drainNow) await this.drain().catch(e => this.log(`drain: ${e.message}`));
|
|
113
122
|
const tick = () => {
|
|
114
123
|
this.pollTimer = setTimeout(() => {
|
|
115
124
|
this.drain().catch(e => this.log(`drain: ${e.message}`)).finally(() => { if (!this.stopped) tick(); });
|
|
@@ -119,8 +128,8 @@ export class Intake {
|
|
|
119
128
|
tick();
|
|
120
129
|
// Embedded in the pod server, a notification socket back to that same
|
|
121
130
|
// server buys nothing — the store's own change events wake the drain.
|
|
122
|
-
if (this.push) this.
|
|
123
|
-
else this.
|
|
131
|
+
if (!this.push) this.wsState = 'in-process';
|
|
132
|
+
else if (subscribe) this.subscribe().catch(e => this.log(`subscribe: ${e.message}`));
|
|
124
133
|
}
|
|
125
134
|
|
|
126
135
|
stop() { this.stopped = true; clearTimeout(this.pollTimer); clearTimeout(this.resubTimer); this.ws?.close(); }
|
|
@@ -208,6 +217,7 @@ export class Intake {
|
|
|
208
217
|
// us, a mention, a reply to ours, or from someone we follow — and drops the
|
|
209
218
|
// rest.
|
|
210
219
|
async prune({ before, keepConcerning = false } = {}) {
|
|
220
|
+
this._sweepGen = this.store.generation;
|
|
211
221
|
const cutoff = Date.parse(before);
|
|
212
222
|
if (!Number.isFinite(cutoff)) throw new Error(`"${before}" is not a date`);
|
|
213
223
|
const all = await podInbox.list(this.remote, this.urls);
|
|
@@ -217,7 +227,29 @@ export class Intake {
|
|
|
217
227
|
|
|
218
228
|
for (const item of older) {
|
|
219
229
|
try {
|
|
220
|
-
if (item.
|
|
230
|
+
if (isBatch(item.url)) {
|
|
231
|
+
// Mail the gateway held while the app was closed: the same rule for
|
|
232
|
+
// each delivery in it, then the batch goes.
|
|
233
|
+
for (const e of (await this._batchEntries(item.url)) || []) {
|
|
234
|
+
const read = typeof e?.body === 'string' ? await readLenient(e.body) : null;
|
|
235
|
+
const activity = read?.view ?? read?.doc ?? null;
|
|
236
|
+
const receipt = activity ? await this._checkReceipt(e?.receipt) : null;
|
|
237
|
+
if (keepConcerning) {
|
|
238
|
+
const rejection = activity ? await this.handle(activity, receipt) : 'unparsable JSON';
|
|
239
|
+
if (rejection) out.dropped++; else out.applied++;
|
|
240
|
+
} else if (activity && activity.type !== 'Create') {
|
|
241
|
+
await this.handle(activity, receipt);
|
|
242
|
+
out.applied++;
|
|
243
|
+
} else {
|
|
244
|
+
out.dropped++;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
if (!await this._persisted()) {
|
|
248
|
+
this.log('state not written — stopping the prune with a batch left');
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
await podInbox.dropHandledItem(this.remote, item.url);
|
|
252
|
+
} else if (item.size > MAX_ITEM_BYTES) {
|
|
221
253
|
await podInbox.dropHandledItem(this.remote, item.url); // unreadable by the drain either way
|
|
222
254
|
out.discarded++;
|
|
223
255
|
} else {
|
|
@@ -309,6 +341,7 @@ export class Intake {
|
|
|
309
341
|
}
|
|
310
342
|
|
|
311
343
|
async _drainOnce() {
|
|
344
|
+
this._sweepGen = this.store.generation;
|
|
312
345
|
const cooling = this.drainCooldownUntil - Date.now();
|
|
313
346
|
if (cooling > 0) {
|
|
314
347
|
this.log(`inbox sweep skipped — backing off for another ${Math.ceil(cooling / 1000)}s`);
|
|
@@ -389,8 +422,29 @@ export class Intake {
|
|
|
389
422
|
return true;
|
|
390
423
|
};
|
|
391
424
|
|
|
425
|
+
let spent = 0; // deliveries taken this sweep, a batch counting each of its own
|
|
392
426
|
for (const { url, size, receipt: hasReceipt } of items) {
|
|
393
427
|
if (url.endsWith('.keep')) continue;
|
|
428
|
+
if (spent >= MAX_ITEMS_PER_DRAIN) { this._drainAgain = handled > 0 || pending.length > 0; break; }
|
|
429
|
+
if (isBatch(url)) {
|
|
430
|
+
try {
|
|
431
|
+
const took = await this._takeBatch(url);
|
|
432
|
+
spent += took.entries;
|
|
433
|
+
if (took.complete) pending.push(url);
|
|
434
|
+
} catch (e) {
|
|
435
|
+
// The batch itself could not be read: tried again next sweep, like
|
|
436
|
+
// any item, and dead-lettered only after as many tries.
|
|
437
|
+
const n = this._bumpAttempt(url, e.message);
|
|
438
|
+
this.log(`inbox batch ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
|
|
439
|
+
if (n >= MAX_ITEM_ATTEMPTS) {
|
|
440
|
+
this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: null });
|
|
441
|
+
pending.push(url);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
if (pending.length >= DELETE_BATCH && !await flush()) return;
|
|
445
|
+
continue;
|
|
446
|
+
}
|
|
447
|
+
spent++;
|
|
394
448
|
if (hasReceipt) withReceipt.add(url);
|
|
395
449
|
// The listing already carries every child's size, so this costs nothing
|
|
396
450
|
// to ask. An activity is a few kB; anything of this order is not one, and
|
|
@@ -404,7 +458,7 @@ export class Intake {
|
|
|
404
458
|
pending.push(url);
|
|
405
459
|
continue;
|
|
406
460
|
}
|
|
407
|
-
|
|
461
|
+
const at = {};
|
|
408
462
|
try {
|
|
409
463
|
const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_ITEM_BYTES, readCapped });
|
|
410
464
|
// readItem carries the rule that matters here: a pod that would not
|
|
@@ -413,56 +467,19 @@ export class Intake {
|
|
|
413
467
|
// made it "unparsable JSON" — a REJECTION, dead-lettered with both
|
|
414
468
|
// `activity` and `raw` null and then DELETEd, destroying a delivery on
|
|
415
469
|
// a transient fault with no record of what it had been.
|
|
416
|
-
const raw = got.raw;
|
|
417
|
-
// Read as JSON-LD, so a document whose terms are aliased means what it
|
|
418
|
-
// says. A document we cannot read that way is read the way it always
|
|
419
|
-
// was rather than lost, and the reason is logged.
|
|
420
|
-
const read = raw ? await readLenient(raw) : null;
|
|
421
|
-
if (read?.degraded) this.log(`inbox item ${url} grounded to read: ${read.degraded}`);
|
|
422
|
-
activity = read?.view ?? read?.doc ?? null;
|
|
423
|
-
// What the shapes make of it. This decides NOTHING — the activity is
|
|
424
|
-
// handled either way. It is written down so that which shapes fire on
|
|
425
|
-
// real traffic is a question with an answer, rather than a guess made
|
|
426
|
-
// in advance about implementations we have not met.
|
|
427
|
-
if (read?.graph) {
|
|
428
|
-
const failure = await checkShapes(read.graph);
|
|
429
|
-
if (failure) {
|
|
430
|
-
const said = describeShapeFailure(failure);
|
|
431
|
-
this.log(`inbox item ${url} does not fit its shape: ${said}`);
|
|
432
|
-
this.store.addDeadLetter({
|
|
433
|
-
inboxUrl: url, reason: `shape: ${said}`, shapeOnly: true, activity: trimActivity(activity),
|
|
434
|
-
});
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
470
|
// A gateway that verified this delivery left a receipt beside it. Read
|
|
438
471
|
// it only when a gateway is configured (no config → no fetch, so an
|
|
439
472
|
// install with no gateway pays nothing) and the listing showed one
|
|
440
473
|
// beside this item (`false` is a definite no; a listing that does not
|
|
441
474
|
// say leaves it to the read); a missing or HMAC-invalid receipt reads
|
|
442
475
|
// as null, which is exactly today's unverified behavior.
|
|
443
|
-
|
|
444
|
-
if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
|
|
445
|
-
// The owner's own post, taken at the outbox door: not mail to read
|
|
446
|
-
// but a write to make. Never archived or forwarded as if received.
|
|
447
|
-
const owned = activity && this.isOwnerPost(receipt);
|
|
448
|
-
const rejection = !activity ? 'unparsable JSON'
|
|
449
|
-
: owned ? await this.ownerPostFrom(activity, raw, receipt)
|
|
450
|
-
: await this.handle(activity, receipt);
|
|
451
|
-
if (!rejection && raw && !owned) await this._archive(url, raw, activity);
|
|
452
|
-
if (!rejection && !owned) await this._maybeForward(activity); // §7.1.2, only what we accepted
|
|
453
|
-
if (rejection) {
|
|
454
|
-
this.store.addDeadLetter({
|
|
455
|
-
inboxUrl: url, reason: rejection, activity: trimActivity(activity),
|
|
456
|
-
...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
|
|
457
|
-
});
|
|
458
|
-
this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
|
|
459
|
-
}
|
|
476
|
+
await this._take(url, got.raw, () => (hasReceipt !== false ? this._readReceipt(url) : null), at);
|
|
460
477
|
pending.push(url);
|
|
461
478
|
} catch (e) {
|
|
462
479
|
const n = this._bumpAttempt(url, e.message);
|
|
463
480
|
this.log(`inbox item ${url} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${e.message}`);
|
|
464
481
|
if (n >= MAX_ITEM_ATTEMPTS) {
|
|
465
|
-
this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(activity) });
|
|
482
|
+
this.store.addDeadLetter({ inboxUrl: url, reason: `failed ${n}x: ${e.message}`, activity: trimActivity(at.activity) });
|
|
466
483
|
// The dead letter IS the record of this item — deleting before it is
|
|
467
484
|
// written down would lose the only evidence it ever arrived, so it
|
|
468
485
|
// goes through the same commit-then-delete batch as everything else.
|
|
@@ -483,6 +500,135 @@ export class Intake {
|
|
|
483
500
|
if (handled > 0 && all.length > items.length && !this.stopped) this._drainAgain = true;
|
|
484
501
|
}
|
|
485
502
|
|
|
503
|
+
// One delivery, alone in the inbox or in a batch: read as JSON-LD, checked
|
|
504
|
+
// against its receipt, handled, and dead-lettered if refused. Throws when it
|
|
505
|
+
// could not be handled this time; `at.activity` then says what it was.
|
|
506
|
+
async _take(url, raw, receiptOf, at = {}) {
|
|
507
|
+
// Read as JSON-LD, so a document whose terms are aliased means what it
|
|
508
|
+
// says. A document we cannot read that way is read the way it always
|
|
509
|
+
// was rather than lost, and the reason is logged.
|
|
510
|
+
const read = raw ? await readLenient(raw) : null;
|
|
511
|
+
if (read?.degraded) this.log(`inbox item ${url} grounded to read: ${read.degraded}`);
|
|
512
|
+
const activity = at.activity = read?.view ?? read?.doc ?? null;
|
|
513
|
+
// What the shapes make of it. This decides NOTHING — the activity is
|
|
514
|
+
// handled either way. It is written down so that which shapes fire on
|
|
515
|
+
// real traffic is a question with an answer, rather than a guess made
|
|
516
|
+
// in advance about implementations we have not met.
|
|
517
|
+
if (read?.graph) {
|
|
518
|
+
const failure = await checkShapes(read.graph);
|
|
519
|
+
if (failure) {
|
|
520
|
+
const said = describeShapeFailure(failure);
|
|
521
|
+
this.log(`inbox item ${url} does not fit its shape: ${said}`);
|
|
522
|
+
this.store.addDeadLetter({
|
|
523
|
+
inboxUrl: url, reason: `shape: ${said}`, shapeOnly: true, activity: trimActivity(activity),
|
|
524
|
+
});
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
const receipt = activity ? await receiptOf() : null;
|
|
528
|
+
if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
|
|
529
|
+
// The owner's own post, taken at the outbox door: not mail to read
|
|
530
|
+
// but a write to make. Never archived or forwarded as if received.
|
|
531
|
+
const owned = activity && this.isOwnerPost(receipt);
|
|
532
|
+
const rejection = !activity ? 'unparsable JSON'
|
|
533
|
+
: owned ? await this.ownerPostFrom(activity, raw, receipt)
|
|
534
|
+
: await this.handle(activity, receipt);
|
|
535
|
+
if (!rejection && raw && !owned) await this._archive(url, raw, activity);
|
|
536
|
+
if (!rejection && !owned) await this._maybeForward(activity); // §7.1.2, only what we accepted
|
|
537
|
+
if (rejection) {
|
|
538
|
+
this.store.addDeadLetter({
|
|
539
|
+
inboxUrl: url, reason: rejection, activity: trimActivity(activity),
|
|
540
|
+
...(activity ? {} : { raw: raw?.slice(0, 2000) ?? null }),
|
|
541
|
+
});
|
|
542
|
+
this.log(`rejected (${rejection}) — dead-lettered: ${url}`);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// The deliveries in a batch the gateway wrote while the owner's app was
|
|
547
|
+
// closed (lib/gateway/held-mail.mjs), or null when it holds none. Throws when
|
|
548
|
+
// the pod would not give it, like readItem.
|
|
549
|
+
async _batchEntries(url) {
|
|
550
|
+
const got = await podInbox.readItem(this.remote, url, { maxBytes: MAX_BATCH_BYTES, readCapped });
|
|
551
|
+
if (got.raw === null) return [];
|
|
552
|
+
let batch = null;
|
|
553
|
+
try { batch = JSON.parse(got.raw)?.batch; } catch { batch = null; }
|
|
554
|
+
return Array.isArray(batch) ? batch.slice(0, MAX_BATCH_ENTRIES) : null;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
// Each delivery in a batch, taken as if it had arrived alone. `complete` is
|
|
558
|
+
// false while some entry is to be tried again: the batch then stays, and the
|
|
559
|
+
// entries already handled are handled again next time, harmlessly, as a
|
|
560
|
+
// re-delivered activity is.
|
|
561
|
+
async _takeBatch(url) {
|
|
562
|
+
const batch = await this._batchEntries(url);
|
|
563
|
+
if (!batch) {
|
|
564
|
+
this.store.addDeadLetter({ inboxUrl: url, reason: 'unreadable batch', activity: null });
|
|
565
|
+
return { entries: 1, complete: true };
|
|
566
|
+
}
|
|
567
|
+
let complete = true;
|
|
568
|
+
for (const [i, e] of batch.entries()) {
|
|
569
|
+
const key = `${url}#${e?.name || i}`;
|
|
570
|
+
const body = typeof e?.body === 'string' ? e.body : null;
|
|
571
|
+
if (body !== null && Buffer.byteLength(body) > MAX_ITEM_BYTES) {
|
|
572
|
+
this.store.addDeadLetter({ inboxUrl: key, reason: `oversized (${Buffer.byteLength(body)} bytes)`, activity: null });
|
|
573
|
+
continue;
|
|
574
|
+
}
|
|
575
|
+
const at = {};
|
|
576
|
+
try {
|
|
577
|
+
await this._take(key, body, () => this._checkReceipt(e?.receipt), at);
|
|
578
|
+
this._clearAttempt(key);
|
|
579
|
+
} catch (err) {
|
|
580
|
+
const n = this._bumpAttempt(key, err.message);
|
|
581
|
+
this.log(`inbox batch entry ${key} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${err.message}`);
|
|
582
|
+
if (n >= MAX_ITEM_ATTEMPTS) {
|
|
583
|
+
this.store.addDeadLetter({ inboxUrl: key, reason: `failed ${n}x: ${err.message}`, activity: trimActivity(at.activity) });
|
|
584
|
+
} else {
|
|
585
|
+
complete = false;
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
return { entries: batch.length, complete };
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Deliveries handed straight in rather than read from the pod's inbox: mail
|
|
594
|
+
* the gateway held for an account whose copy it keeps (lib/gateway/keeper.mjs).
|
|
595
|
+
* `entries` are { name, body, receipt }, each taken as if it had arrived
|
|
596
|
+
* alone. Returns the names that are done with — handled, or refused for good
|
|
597
|
+
* — once what they changed is written; one to be tried again is left out, so
|
|
598
|
+
* whoever holds it keeps it for next time.
|
|
599
|
+
*/
|
|
600
|
+
async takeHeld(entries) {
|
|
601
|
+
const since = this.store.generation;
|
|
602
|
+
const done = [];
|
|
603
|
+
this.store.hold?.();
|
|
604
|
+
try {
|
|
605
|
+
for (const e of entries) {
|
|
606
|
+
const key = `${this.urls.inbox}${e.name}`;
|
|
607
|
+
const body = typeof e.body === 'string' ? e.body : null;
|
|
608
|
+
if (body !== null && Buffer.byteLength(body) > MAX_ITEM_BYTES) {
|
|
609
|
+
this.store.addDeadLetter({ inboxUrl: key, reason: `oversized (${Buffer.byteLength(body)} bytes)`, activity: null });
|
|
610
|
+
done.push(e.name);
|
|
611
|
+
continue;
|
|
612
|
+
}
|
|
613
|
+
const at = {};
|
|
614
|
+
try {
|
|
615
|
+
await this._take(key, body, () => this._checkReceipt(e.receipt), at);
|
|
616
|
+
this._clearAttempt(key);
|
|
617
|
+
done.push(e.name);
|
|
618
|
+
} catch (err) {
|
|
619
|
+
const n = this._bumpAttempt(key, err.message);
|
|
620
|
+
this.log(`held delivery ${key} attempt ${n}/${MAX_ITEM_ATTEMPTS}: ${err.message}`);
|
|
621
|
+
if (n >= MAX_ITEM_ATTEMPTS) {
|
|
622
|
+
this.store.addDeadLetter({ inboxUrl: key, reason: `failed ${n}x: ${err.message}`, activity: trimActivity(at.activity) });
|
|
623
|
+
done.push(e.name);
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
await this._publishPending();
|
|
628
|
+
} finally { this.store.release?.(); }
|
|
629
|
+
return (await this.store.commit({ since })) ? done : [];
|
|
630
|
+
}
|
|
631
|
+
|
|
486
632
|
// The end of a sweep: publish whatever the follow graph did ONCE, then flush.
|
|
487
633
|
//
|
|
488
634
|
// publishCollections used to run per handled item — every Follow, Undo,
|
|
@@ -643,6 +789,7 @@ export class Intake {
|
|
|
643
789
|
gatewaySecret(...a) { return verify.gatewaySecret(this, ...a); }
|
|
644
790
|
_bumpGatewayStat(...a) { return verify.bumpGatewayStat(this, ...a); }
|
|
645
791
|
_readReceipt(...a) { return verify.readReceipt(this, ...a); }
|
|
792
|
+
_checkReceipt(...a) { return verify.checkReceipt(this, ...a); }
|
|
646
793
|
|
|
647
794
|
// A receipt the door stamped `c2s` for THIS actor, and only a receipt whose
|
|
648
795
|
// HMAC verified (readReceipt returns nothing else). A stranger appending an
|
|
@@ -656,10 +803,33 @@ export class Intake {
|
|
|
656
803
|
// document, not our reading of it); the graph is what it decides from.
|
|
657
804
|
// Returns null when published, else the reason it was not — a dead letter.
|
|
658
805
|
async ownerPostFrom(activity, raw, receipt) {
|
|
806
|
+
// The receipt names the body it vouches for; a body it does not name is
|
|
807
|
+
// not the owner's. A body already handled — a replayed request, or an item
|
|
808
|
+
// the pod would not let us delete — is not published twice.
|
|
809
|
+
const hash = raw ? await sha256Hex(raw) : null;
|
|
810
|
+
if (receipt.hash && receipt.hash !== hash) return 'owner post refused: the body is not the one its receipt names';
|
|
811
|
+
const seen = this.store.read('c2s-seen.json', []);
|
|
812
|
+
if (hash && seen.includes(hash)) {
|
|
813
|
+
this.log('owner post already handled — not published again');
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
659
816
|
let asSent = null;
|
|
660
817
|
try { asSent = raw ? JSON.parse(raw) : null; } catch { asSent = null; }
|
|
661
|
-
const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null
|
|
662
|
-
|
|
818
|
+
const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null,
|
|
819
|
+
// What the door told the client this will be called, when it said.
|
|
820
|
+
...(receipt.serial ? { serial: receipt.serial } : {}), ...(receipt.at ? { at: receipt.at } : {}) });
|
|
821
|
+
// Handled, one way or the other: a second sight of it changes nothing. A
|
|
822
|
+
// throw above is not handled, and is tried again.
|
|
823
|
+
if (hash) this.store.write('c2s-seen.json', [hash, ...this.store.read('c2s-seen.json', [])].slice(0, C2S_SEEN_MAX));
|
|
824
|
+
if (!r || r.status >= 300) {
|
|
825
|
+
const why = r?.body?.error || 'no reason was given';
|
|
826
|
+
// The door already answered the app "created"; the owner is the one
|
|
827
|
+
// left to tell.
|
|
828
|
+
await this.publisher.noteToSelf?.(`An app asked your account to publish ${article(activity.type)}, `
|
|
829
|
+
+ `and your account refused it: ${why}. Nothing was sent.`)
|
|
830
|
+
.catch(e => this.log(`could not tell the owner their post was refused: ${e.message}`));
|
|
831
|
+
return `owner post refused (${r?.status || '?'}): ${why}`;
|
|
832
|
+
}
|
|
663
833
|
this.log(`owner post from the outbox door published: ${r.body?.object || r.body?.id || activity.type}`);
|
|
664
834
|
return null;
|
|
665
835
|
}
|
|
@@ -699,3 +869,11 @@ export class Intake {
|
|
|
699
869
|
retract(...a) { return notes.retract(this, ...a); }
|
|
700
870
|
addReply(...a) { return notes.addReply(this, ...a); }
|
|
701
871
|
}
|
|
872
|
+
|
|
873
|
+
// "a Like", "an Update": the kind of activity, as a sentence says it.
|
|
874
|
+
const article = (type) => `${/^[AEIOU]/u.test(String(type)) ? 'an' : 'a'} ${type || 'post'}`;
|
|
875
|
+
|
|
876
|
+
// Bodies the outbox door handed over that were handled, newest first.
|
|
877
|
+
const C2S_SEEN_MAX = 500;
|
|
878
|
+
const sha256Hex = async (text) => [...new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256',
|
|
879
|
+
new TextEncoder().encode(text)))].map(b => b.toString(16).padStart(2, '0')).join('');
|
|
@@ -91,7 +91,11 @@ export async function maybeForward(intake, activity) {
|
|
|
91
91
|
.filter(Boolean))];
|
|
92
92
|
if (!inboxes.length) return;
|
|
93
93
|
intake._forwardBudget -= 1;
|
|
94
|
-
|
|
94
|
+
// Blind copies are for their sender's delivery alone and never travel on
|
|
95
|
+
// (§6, §7.1), from the activity or its object.
|
|
96
|
+
const strip = (o) => { if (!o || typeof o !== 'object') return o; const { bto, bcc, ...rest } = o; return rest; }; // eslint-disable-line no-unused-vars
|
|
97
|
+
const onward = { ...strip(activity), ...(activity.object && typeof activity.object === 'object' ? { object: strip(activity.object) } : {}) };
|
|
98
|
+
await intake.deliverer.deliverToAll(inboxes, onward);
|
|
95
99
|
intake.store.write('forwarded.json', [...forwarded, id].slice(-MAX_FORWARDED));
|
|
96
100
|
intake.log(`forwarded ${activity.type} ${id} to ${inboxes.length} follower inbox(es)`);
|
|
97
101
|
} catch (e) {
|
|
@@ -319,10 +323,13 @@ export async function retract(intake, noteId, { collect = null } = {}) {
|
|
|
319
323
|
if (!s.announceActivity) throw new Error('that post was never carried');
|
|
320
324
|
const { undoActivity } = await import('../wire.mjs');
|
|
321
325
|
const inboxes = intake.announceTargets(s.actor);
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
+
const undo = undoActivity({ urls: intake.urls, activity: s.announceActivity, serial: intake.serial++ });
|
|
327
|
+
await intake.deliverer.deliverToAll(inboxes, undo);
|
|
328
|
+
// The Undo takes the carry's place in the outbox, so a reader catching up
|
|
329
|
+
// from it learns the carry was withdrawn.
|
|
330
|
+
if (collect) collect.push(undo);
|
|
331
|
+
else await intake.publisher.unrecordOutbox(i => i?.id === s.announceActivity.id,
|
|
332
|
+
{ record: () => (intake.config.private ? null : undo) });
|
|
326
333
|
intake.store.updateStatus(noteId, {
|
|
327
334
|
announcedAt: undefined, announceActivity: undefined, retractedAt: new Date().toISOString(),
|
|
328
335
|
});
|