fedipod-server 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -28
- package/config/server.json +22 -0
- package/dist/claims.js +3 -2
- package/dist/directory.d.ts +2 -0
- package/dist/handler.d.ts +4 -1
- package/dist/handler.js +32 -3
- package/dist/handler.jsonld +4 -0
- package/dist/store-pod.js +37 -0
- package/lib/client/c2s.mjs +97 -43
- package/lib/client/masto/accounts.mjs +1 -0
- package/lib/connections/bskyfeed.mjs +6 -0
- package/lib/core/deliver.mjs +68 -22
- package/lib/core/intake/activities.mjs +18 -2
- package/lib/core/intake/group.mjs +3 -1
- package/lib/core/intake/index.mjs +50 -6
- package/lib/core/intake/notes.mjs +12 -5
- package/lib/core/lease.mjs +7 -7
- package/lib/core/place.mjs +82 -0
- package/lib/core/publisher/collections.mjs +37 -2
- package/lib/core/publisher/index.mjs +23 -1
- package/lib/core/publisher/notes.mjs +73 -7
- package/lib/core/publisher/own.mjs +143 -0
- package/lib/core/publisher/questions.mjs +5 -3
- package/lib/core/social.mjs +62 -32
- package/lib/core/store.mjs +36 -6
- package/lib/core/wire.mjs +51 -15
- package/lib/device/admin/routes/lifecycle.mjs +1 -1
- package/lib/device/admin/routes/setup.mjs +17 -1
- package/lib/device/cli/commands/setup.mjs +50 -8
- package/lib/device/cli/context.mjs +1 -1
- package/lib/device/migrate.mjs +1 -1
- package/lib/device/setup.mjs +45 -8
- package/lib/gateway/caches.mjs +36 -0
- package/lib/gateway/front-core.mjs +80 -111
- package/lib/gateway/gateway-core.mjs +91 -9
- package/lib/gateway/headers.mjs +49 -0
- package/lib/gateway/quiet.mjs +7 -2
- package/lib/gateway/relay-extras.mjs +89 -0
- package/lib/gateway/token-claims.mjs +16 -0
- package/lib/pod/containers.mjs +5 -1
- package/lib/pod/inbox.mjs +22 -3
- package/lib/pod/location.mjs +52 -0
- package/lib/pod/notes.mjs +2 -4
- package/lib/pod/transport.mjs +207 -25
- package/lib/pod/type-index.mjs +101 -0
- package/lib/pod/urls.mjs +6 -0
- package/lib/server/embed.mjs +7 -6
- package/lib/session/README.md +14 -0
- package/lib/session/fedi-account.mjs +73 -2
- package/lib/session/package.json +10 -2
- package/package.json +1 -1
- package/run-agent.mjs +18 -0
- package/web/admin/actors.js +2 -0
- package/web/admin/index.html +9 -0
- package/web/admin/record.js +4 -1
- package/web/admin/setup/index.html +17 -1
- package/web/admin/setup/setup.js +18 -5
- package/web/app/README.md +1 -1
- package/web/app/admin-facade.mjs +1 -1
- package/web/app/agent.mjs +24 -10
- package/web/app/boot.mjs +72 -32
- package/web/app/deliver-relay.mjs +47 -7
- package/web/app/dist/boot.js +466 -89
- package/web/app/dist/boot.js.map +4 -4
- package/web/app/dist/sw.js +2792 -1996
- package/web/app/dist/sw.js.map +4 -4
- package/web/app/index.html +16 -0
- package/web/app/signup.mjs +65 -26
- package/web/app/update.js +4 -0
- package/web/front/run.html +7 -1
- package/web/front/run.js +30 -4
package/lib/client/c2s.mjs
CHANGED
|
@@ -28,6 +28,7 @@ import * as social from '../core/social.mjs';
|
|
|
28
28
|
import * as wire from '../core/wire.mjs';
|
|
29
29
|
import { readLenient } from '../core/as2.mjs';
|
|
30
30
|
import { safeSlug } from '../core/publisher/notes.mjs';
|
|
31
|
+
import * as podNotes from '../pod/notes.mjs';
|
|
31
32
|
|
|
32
33
|
const MAX_BODY = 512 * 1024; // same ceiling the inbox drain enforces
|
|
33
34
|
|
|
@@ -192,9 +193,11 @@ export class C2S {
|
|
|
192
193
|
* published copy and it stays the canonical one; this is served only where
|
|
193
194
|
* that copy cannot be read by the client asking.
|
|
194
195
|
*/
|
|
195
|
-
sendLocalOutbox(res, url, origin) {
|
|
196
|
+
sendLocalOutbox(res, url, origin, { owner = false } = {}) {
|
|
196
197
|
const id = `${origin}ap/outbox`;
|
|
197
|
-
|
|
198
|
+
// newest first; the owner's view is every message, the public one what a
|
|
199
|
+
// stranger may see
|
|
200
|
+
const outbox = this.store.read(owner ? 'outbox-own.json' : 'outbox.json', []);
|
|
198
201
|
const page = url?.searchParams?.get('page') || null;
|
|
199
202
|
const ct = { 'content-type': 'application/activity+json; charset=utf-8' };
|
|
200
203
|
if (!page) {
|
|
@@ -270,12 +273,22 @@ export class C2S {
|
|
|
270
273
|
return this.sendInbox(res, url);
|
|
271
274
|
}
|
|
272
275
|
if (req.method === 'GET' || req.method === 'HEAD') {
|
|
276
|
+
// The owner, signed in, reads every message the actor produced (§5.1:
|
|
277
|
+
// the outbox is filtered by who asks). Anyone else reads the public one.
|
|
278
|
+
const owner = pathname === '/ap/outbox' && req.headers.authorization
|
|
279
|
+
? (await this.auth(req, pathname)).ok : false;
|
|
273
280
|
// Where the client surface answers on an address a stranger can reach,
|
|
274
281
|
// the pod's copies already name it and are the documents. Standalone
|
|
275
282
|
// they are not reachable by the client asking, so it is answered here.
|
|
276
283
|
const local = this.agent.publisher?.clientOrigin ? null : this.localOrigin(req);
|
|
277
284
|
if (local && pathname === '/ap/actor') return this.sendLocalActor(req, res, local);
|
|
278
|
-
if (local && pathname === '/ap/outbox') return this.sendLocalOutbox(res, url, local);
|
|
285
|
+
if (local && pathname === '/ap/outbox') return this.sendLocalOutbox(res, url, local, { owner });
|
|
286
|
+
// The owner is answered here, not sent on: a redirect loses the
|
|
287
|
+
// credential (a browser drops it across origins, and a DPoP proof names
|
|
288
|
+
// the address it was made for), so the pod would refuse them.
|
|
289
|
+
if (owner && pathname === '/ap/outbox') {
|
|
290
|
+
return this.sendLocalOutbox(res, url, this.agent.publisher.clientOrigin, { owner });
|
|
291
|
+
}
|
|
279
292
|
// The pod's copy is the document; send the reader there.
|
|
280
293
|
const target = pathname === '/ap/actor' ? this.urls.actor : this.urls.outbox;
|
|
281
294
|
res.writeHead(303, { location: target, 'cache-control': 'no-store' });
|
|
@@ -334,14 +347,22 @@ export class C2S {
|
|
|
334
347
|
visibilityOf(activity, object) {
|
|
335
348
|
const to = arr(activity.to ?? object?.to).map(idOf);
|
|
336
349
|
const cc = arr(activity.cc ?? object?.cc).map(idOf);
|
|
337
|
-
|
|
350
|
+
// Blind copies alone are still an audience: a post for them and nobody
|
|
351
|
+
// else is direct, never public.
|
|
352
|
+
const blind = arr(activity.bto ?? object?.bto).length + arr(activity.bcc ?? object?.bcc).length;
|
|
353
|
+
if (!to.length && !cc.length) return blind ? 'direct' : 'public';
|
|
338
354
|
if (to.includes(wire.PUBLIC)) return 'public';
|
|
339
355
|
if (cc.includes(wire.PUBLIC)) return 'unlisted';
|
|
340
|
-
|
|
356
|
+
// Followers named anywhere — "to alice, cc my followers" included — is a
|
|
357
|
+
// followers-only post that also reaches whoever else is named.
|
|
358
|
+
if (to.includes(this.urls.followers) || cc.includes(this.urls.followers)) return 'private';
|
|
341
359
|
return 'direct';
|
|
342
360
|
}
|
|
343
361
|
|
|
344
|
-
|
|
362
|
+
// `serial` and `at` name what this activity will make: the outbox door
|
|
363
|
+
// chose them and told the client, so they are used as given. A client
|
|
364
|
+
// posting here directly gets fresh ones.
|
|
365
|
+
async dispatch(activity, { slug = null, raw = null, serial = Date.now(), at = new Date().toISOString() } = {}) {
|
|
345
366
|
const reply = (status, body, headers = {}) => ({ status, body, headers });
|
|
346
367
|
if (!activity || typeof activity !== 'object' || Array.isArray(activity) || !activity.type) {
|
|
347
368
|
return reply(400, { error: 'a typed ActivityStreams object is required' });
|
|
@@ -349,10 +370,12 @@ export class C2S {
|
|
|
349
370
|
// A bare object arrives without an activity around it; the server supplies
|
|
350
371
|
// the Create (§6.2.1), carrying the object's own addressing up onto it.
|
|
351
372
|
if (!ACTIVITY_TYPES.has(activity.type)) {
|
|
352
|
-
|
|
373
|
+
// Every audience field is copied up onto the Create (§6.2.1).
|
|
374
|
+
const { to, cc, bto, bcc, audience } = activity;
|
|
375
|
+
activity = { type: 'Create', object: activity, to, cc, bto, bcc, audience };
|
|
353
376
|
}
|
|
354
377
|
try {
|
|
355
|
-
return await this._dispatch(activity, { slug, raw, reply });
|
|
378
|
+
return await this._dispatch(activity, { slug, raw, reply, serial, at });
|
|
356
379
|
} catch (e) {
|
|
357
380
|
this.log(`c2s ${activity?.type}: ${e.message}`);
|
|
358
381
|
return reply(422, { error: e.message || String(e) });
|
|
@@ -366,11 +389,23 @@ export class C2S {
|
|
|
366
389
|
const pick = (...fields) => [...new Set(fields.flatMap((f) => arr(activity[f] ?? object?.[f]).map(idOf)))]
|
|
367
390
|
.filter((a) => typeof a === 'string' && /^https?:\/\//u.test(a)
|
|
368
391
|
&& a !== wire.PUBLIC && a !== this.urls.followers && a !== this.urls.actor);
|
|
369
|
-
|
|
392
|
+
// `audience` is who the post is for, a group most often: delivered to and
|
|
393
|
+
// named like to and cc (§7.1).
|
|
394
|
+
return { also: pick('to', 'cc', 'audience'), deliverTo: pick('bto', 'bcc') };
|
|
370
395
|
}
|
|
371
396
|
|
|
372
|
-
async _dispatch(activity, { slug, raw, reply }) {
|
|
397
|
+
async _dispatch(activity, { slug, raw, reply, serial, at }) {
|
|
373
398
|
const agent = this.agent;
|
|
399
|
+
// §6: every post to the outbox answers 201 with the new activity's id.
|
|
400
|
+
const made = (id, body = {}) => reply(201, { id, ...body }, { location: id });
|
|
401
|
+
// What is never sent anywhere — a person's block, a pin — is still a
|
|
402
|
+
// message this actor produced, so the owner's outbox records it here.
|
|
403
|
+
const kept = (type, extra) => {
|
|
404
|
+
const id = type === 'Undo' ? wire.undoActivityId(this.urls, serial) : `${this.urls.actor}#${type.toLowerCase()}-${serial}`;
|
|
405
|
+
const act = { id, type, actor: this.urls.actor, published: at, ...extra };
|
|
406
|
+
agent.publisher.recordOwn?.(act);
|
|
407
|
+
return act.id;
|
|
408
|
+
};
|
|
374
409
|
const object = typeof activity.object === 'object' && activity.object !== null
|
|
375
410
|
? activity.object : null;
|
|
376
411
|
const objectId = idOf(activity.object);
|
|
@@ -411,6 +446,7 @@ export class C2S {
|
|
|
411
446
|
visibility,
|
|
412
447
|
spoilerText: object.summary || null,
|
|
413
448
|
sensitive: object.sensitive === true,
|
|
449
|
+
slug,
|
|
414
450
|
});
|
|
415
451
|
return reply(201,
|
|
416
452
|
{ id: wire.createActivityId(question.id), object: question.id },
|
|
@@ -445,19 +481,32 @@ export class C2S {
|
|
|
445
481
|
if (s.actor !== this.urls.actor || s.kind !== 'post') {
|
|
446
482
|
return reply(403, { error: 'not your note' });
|
|
447
483
|
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
484
|
+
// §6.3: a client's Update changes only what it sends, and a field sent
|
|
485
|
+
// as null is removed. What was sent is read from the bytes, because a
|
|
486
|
+
// JSON-LD reading drops the nulls that mean "remove this".
|
|
487
|
+
const sent = raw?.type === 'Update' && raw.object && typeof raw.object === 'object' ? raw.object : (object || {});
|
|
488
|
+
const has = (k) => Object.prototype.hasOwnProperty.call(sent, k);
|
|
489
|
+
const stored = await Promise.resolve().then(() => podNotes.read(agent.remote, s.noteId)).catch(() => null);
|
|
490
|
+
if (stored?.type && stored.type !== 'Note') {
|
|
491
|
+
await agent.publisher.updateObject(s, sent, { updated: at });
|
|
492
|
+
return made(wire.updateActivityId(s.noteId, at), { object: s.noteId });
|
|
493
|
+
}
|
|
494
|
+
const text = has('content') || sent.source?.content !== undefined
|
|
495
|
+
? String(object?.source?.content ?? object?.content ?? '') : String(s.text ?? '');
|
|
496
|
+
if (!text.trim()) return reply(422, { error: 'the edit leaves the note with no content' });
|
|
497
|
+
const attachments = has('attachment')
|
|
451
498
|
? arr(object.attachment).map((a) => ({
|
|
452
499
|
url: a?.url, mediaType: a?.mediaType,
|
|
453
500
|
...(a?.name ? { description: a.name } : {}),
|
|
454
501
|
})).filter((a) => a.url)
|
|
455
502
|
: null;
|
|
456
503
|
await agent.publisher.updateNote(s, {
|
|
457
|
-
content: text,
|
|
458
|
-
|
|
504
|
+
content: text,
|
|
505
|
+
spoilerText: has('summary') ? (object?.summary || null) : (s.spoiler || null),
|
|
506
|
+
sensitive: has('sensitive') ? object?.sensitive === true : null, attachments,
|
|
507
|
+
updated: at,
|
|
459
508
|
});
|
|
460
|
-
return
|
|
509
|
+
return made(wire.updateActivityId(s.noteId, at), { object: s.noteId });
|
|
461
510
|
}
|
|
462
511
|
|
|
463
512
|
case 'Delete': {
|
|
@@ -471,7 +520,7 @@ export class C2S {
|
|
|
471
520
|
}
|
|
472
521
|
const r = await social.deleteNote(agent, s);
|
|
473
522
|
if (!r.ok) return reply(502, { error: r.error, stillPublished: r.stillPublished });
|
|
474
|
-
return
|
|
523
|
+
return made(wire.deleteActivityId(s.noteId), { object: s.noteId });
|
|
475
524
|
}
|
|
476
525
|
|
|
477
526
|
case 'Follow': {
|
|
@@ -479,31 +528,27 @@ export class C2S {
|
|
|
479
528
|
// An acct: form or bare handle resolves through WebFinger; an https
|
|
480
529
|
// IRI is fetched directly.
|
|
481
530
|
if (/^acct:|^@|^[^/@]+@[^/@]+$/.test(objectId) && !/^https?:/.test(objectId)) {
|
|
482
|
-
const r = await social.followHandle(agent, objectId.replace(/^acct:/, ''));
|
|
531
|
+
const r = await social.followHandle(agent, objectId.replace(/^acct:/, ''), { serial });
|
|
483
532
|
const rec = this.store.getContacts().following.find((f) => f.actor === r.actor);
|
|
484
|
-
return
|
|
485
|
-
rec?.followActivity?.id ? { location: rec.followActivity.id } : {});
|
|
533
|
+
return made(rec?.followActivity?.id, { object: r.actor });
|
|
486
534
|
}
|
|
487
|
-
const doc = await social.followActor(agent, objectId);
|
|
535
|
+
const doc = await social.followActor(agent, objectId, { serial });
|
|
488
536
|
const rec = this.store.getContacts().following.find((f) => f.actor === doc.id);
|
|
489
|
-
return
|
|
490
|
-
rec?.followActivity?.id ? { location: rec.followActivity.id } : {});
|
|
537
|
+
return made(rec?.followActivity?.id, { object: doc.id });
|
|
491
538
|
}
|
|
492
539
|
|
|
493
540
|
case 'Like': {
|
|
494
541
|
const s = this.byIri(objectId);
|
|
495
542
|
if (!s) return reply(422, { error: 'that note is not held here — like what the timeline holds' });
|
|
496
|
-
const updated = await social.favourite(agent, s);
|
|
497
|
-
return
|
|
498
|
-
updated.likeActivity?.id ? { location: updated.likeActivity.id } : {});
|
|
543
|
+
const updated = await social.favourite(agent, s, { serial });
|
|
544
|
+
return made(updated.likeActivity?.id, { object: s.noteId });
|
|
499
545
|
}
|
|
500
546
|
|
|
501
547
|
case 'Announce': {
|
|
502
548
|
const s = this.byIri(objectId);
|
|
503
549
|
if (!s) return reply(422, { error: 'that note is not held here — boost what the timeline holds' });
|
|
504
|
-
const updated = await social.reblog(agent, s);
|
|
505
|
-
return
|
|
506
|
-
updated.announceActivity?.id ? { location: updated.announceActivity.id } : {});
|
|
550
|
+
const updated = await social.reblog(agent, s, { serial });
|
|
551
|
+
return made(updated.announceActivity?.id, { object: s.noteId });
|
|
507
552
|
}
|
|
508
553
|
|
|
509
554
|
case 'Undo': {
|
|
@@ -511,39 +556,47 @@ export class C2S {
|
|
|
511
556
|
// sends one, by its type+object when it re-states it instead.
|
|
512
557
|
const inner = object;
|
|
513
558
|
const innerId = idOf(activity.object);
|
|
559
|
+
const undone = wire.undoActivityId(this.urls, serial);
|
|
514
560
|
if (inner?.type === 'Block') {
|
|
515
561
|
const target = idOf(inner.object);
|
|
516
562
|
if (!target) return reply(400, { error: 'unblock whom?' });
|
|
517
|
-
await social.unblockActor(agent, target);
|
|
518
|
-
|
|
563
|
+
await social.unblockActor(agent, target, { serial });
|
|
564
|
+
// A group's unban was sent, and recorded as it went; a person's is not.
|
|
565
|
+
if (this.store.getConfig()?.kind !== 'group') {
|
|
566
|
+
kept('Undo', { object: { type: 'Block', actor: this.urls.actor, object: target } });
|
|
567
|
+
}
|
|
568
|
+
return made(undone, { object: target });
|
|
519
569
|
}
|
|
520
570
|
const statuses = this.store.getStatuses();
|
|
521
571
|
let s = innerId ? statuses.find((x) => x.likeActivity?.id === innerId) : null;
|
|
522
572
|
if (!s && inner?.type === 'Like') s = this.byIri(idOf(inner.object));
|
|
523
573
|
if (s?.favourited) {
|
|
524
|
-
const updated = await social.unfavourite(agent, s);
|
|
525
|
-
return
|
|
574
|
+
const updated = await social.unfavourite(agent, s, { serial });
|
|
575
|
+
return made(undone, { object: updated.noteId });
|
|
526
576
|
}
|
|
527
577
|
s = innerId ? statuses.find((x) => x.announceActivity?.id === innerId) : null;
|
|
528
578
|
if (!s && inner?.type === 'Announce') s = this.byIri(idOf(inner.object));
|
|
529
579
|
if (s?.reblogged) {
|
|
530
|
-
const updated = await social.unreblog(agent, s);
|
|
531
|
-
return
|
|
580
|
+
const updated = await social.unreblog(agent, s, { serial });
|
|
581
|
+
return made(undone, { object: updated.noteId });
|
|
532
582
|
}
|
|
533
583
|
const following = this.store.getContacts().following;
|
|
534
584
|
const rec = following.find((f) => f.followActivity?.id === innerId)
|
|
535
585
|
|| (inner?.type === 'Follow' ? following.find((f) => f.actor === idOf(inner.object)) : null);
|
|
536
586
|
if (rec) {
|
|
537
|
-
await social.unfollowActor(agent, rec.actor);
|
|
538
|
-
return
|
|
587
|
+
await social.unfollowActor(agent, rec.actor, { serial });
|
|
588
|
+
return made(undone, { object: rec.actor });
|
|
539
589
|
}
|
|
540
590
|
return reply(422, { error: 'nothing here matches what that Undo names' });
|
|
541
591
|
}
|
|
542
592
|
|
|
543
593
|
case 'Block': {
|
|
544
594
|
if (!objectId) return reply(400, { error: 'block whom? object must name an actor' });
|
|
545
|
-
await social.blockActor(agent, objectId);
|
|
546
|
-
|
|
595
|
+
await social.blockActor(agent, objectId, { serial });
|
|
596
|
+
// A group's ban was sent to its members; a person's never goes anywhere.
|
|
597
|
+
const id = this.store.getConfig()?.kind === 'group'
|
|
598
|
+
? `${this.urls.actor}#block-${serial}` : kept('Block', { object: objectId });
|
|
599
|
+
return made(id, { object: objectId });
|
|
547
600
|
}
|
|
548
601
|
|
|
549
602
|
case 'Add':
|
|
@@ -556,7 +609,8 @@ export class C2S {
|
|
|
556
609
|
const s = this.byIri(objectId);
|
|
557
610
|
if (!s) return reply(404, { error: 'no such note here' });
|
|
558
611
|
const updated = await social.pinStatus(agent, s, activity.type === 'Add');
|
|
559
|
-
return
|
|
612
|
+
return made(kept(activity.type, { object: updated.noteId, target: this.urls.featured }),
|
|
613
|
+
{ object: updated.noteId, pinned: !!updated.pinned });
|
|
560
614
|
}
|
|
561
615
|
|
|
562
616
|
case 'Accept':
|
|
@@ -566,10 +620,10 @@ export class C2S {
|
|
|
566
620
|
const requester = object?.actor ? idOf(object.actor) : objectId;
|
|
567
621
|
if (!requester) return reply(400, { error: 'whose request? object must name the Follow or its actor' });
|
|
568
622
|
const r = activity.type === 'Accept'
|
|
569
|
-
? await social.admitRequest(agent, requester).catch((e) => ({ error: e.message }))
|
|
570
|
-
: await social.refuseRequest(agent, requester).catch((e) => ({ error: e.message }));
|
|
623
|
+
? await social.admitRequest(agent, requester, { serial }).catch((e) => ({ error: e.message }))
|
|
624
|
+
: await social.refuseRequest(agent, requester, { serial }).catch((e) => ({ error: e.message }));
|
|
571
625
|
if (r.error) return reply(404, { error: r.error });
|
|
572
|
-
return
|
|
626
|
+
return made(`${this.urls.actor}#${activity.type.toLowerCase()}-${serial}`, { object: requester });
|
|
573
627
|
}
|
|
574
628
|
|
|
575
629
|
case 'Move':
|
|
@@ -255,6 +255,7 @@ export async function handle(api, ctx) {
|
|
|
255
255
|
if (req.method === 'POST' && !b.domains.includes(domain)) b.domains.push(domain);
|
|
256
256
|
if (req.method === 'DELETE') b.domains = b.domains.filter(d => d !== domain);
|
|
257
257
|
api.store.setBlocklist(b);
|
|
258
|
+
Promise.resolve(api.agent?.publisher?.publishCollections?.({ blocked: true })).catch(() => {});
|
|
258
259
|
return send(200, {});
|
|
259
260
|
}
|
|
260
261
|
|
|
@@ -166,6 +166,10 @@ export class BskyFeed {
|
|
|
166
166
|
this.lastSweep = new Date().toISOString();
|
|
167
167
|
const self = this.atproto.read()?.did;
|
|
168
168
|
let added = 0;
|
|
169
|
+
// One commit boundary for the sweep, as acctfeed and tagfeed: every item
|
|
170
|
+
// awaits Bluesky before the next, so the debounce never coalesced them and
|
|
171
|
+
// the timeline index went to the pod once per notification.
|
|
172
|
+
this.store.hold?.();
|
|
169
173
|
try {
|
|
170
174
|
// Our own CROSS-POSTS must not echo back into the feed — but a post or
|
|
171
175
|
// reply written natively on Bluesky is ours to see here too. The mirrors
|
|
@@ -224,6 +228,8 @@ export class BskyFeed {
|
|
|
224
228
|
} catch (e) {
|
|
225
229
|
this._backOff(e.status || 0, null);
|
|
226
230
|
return;
|
|
231
|
+
} finally {
|
|
232
|
+
this.store.release?.();
|
|
227
233
|
}
|
|
228
234
|
const all = this.store.getStatuses();
|
|
229
235
|
const mirrored = all.filter(s => s.kind === 'bsky');
|
package/lib/core/deliver.mjs
CHANGED
|
@@ -73,6 +73,10 @@ export class Deliverer {
|
|
|
73
73
|
this.edPrivate = edPrivate;
|
|
74
74
|
this.proofKeyId = proofKeyId;
|
|
75
75
|
this.log = log;
|
|
76
|
+
// How many fresh deliveries one attempt may carry. The Node deliverer
|
|
77
|
+
// opens a socket per recipient; the browser's relay deliverer sends a
|
|
78
|
+
// list, and sets this to what the relay accepts.
|
|
79
|
+
this.batchSize = 1;
|
|
76
80
|
if (!passive) this.startQueue();
|
|
77
81
|
}
|
|
78
82
|
|
|
@@ -171,33 +175,65 @@ export class Deliverer {
|
|
|
171
175
|
}
|
|
172
176
|
}
|
|
173
177
|
|
|
178
|
+
// One attempt per target, answered in order: `{ ok: true }` or `{ error }`
|
|
179
|
+
// with the error deliverNow would have thrown. Here one at a time; the
|
|
180
|
+
// relay deliverer sends the whole list in one call.
|
|
181
|
+
async deliverManyNow(targets) {
|
|
182
|
+
const out = [];
|
|
183
|
+
for (const t of targets) {
|
|
184
|
+
try { await this.deliverNow(t.inbox, t.activity); out.push({ ok: true }); }
|
|
185
|
+
catch (error) { out.push({ error }); }
|
|
186
|
+
}
|
|
187
|
+
return out;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// A fresh activity of this actor's, for the outbox its owner reads. Never
|
|
191
|
+
// allowed to stop a delivery; a retry from the queue is not fresh.
|
|
192
|
+
_sent(activity) {
|
|
193
|
+
try { this.onSent?.(activity); } catch (e) { this.log(`owner's outbox record: ${e.message}`); }
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// An inbox on a blocked domain is never delivered to, whoever still lists it.
|
|
197
|
+
_blockedInbox(inbox) {
|
|
198
|
+
try { return !!this.store.getBlocklist?.().domains.length && this.store.isBlocked?.(new URL(inbox).origin + '/'); }
|
|
199
|
+
catch { return false; }
|
|
200
|
+
}
|
|
201
|
+
|
|
174
202
|
async deliver(inbox, activity) {
|
|
203
|
+
this._sent(activity);
|
|
204
|
+
if (this._blockedInbox(inbox)) return;
|
|
175
205
|
// Proved before anything else, so the copy that goes on the queue is the
|
|
176
206
|
// copy that was signed — a retry days later must not post a bare activity.
|
|
177
207
|
const signed = await this.proofed(activity);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
208
|
+
if (this._queueIfCooling(inbox, signed)) return;
|
|
209
|
+
const [result] = await this.deliverManyNow([{ inbox, activity: signed }]);
|
|
210
|
+
await this._settle(inbox, signed, result);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// A host we already know is refusing: queue without asking again. This is
|
|
214
|
+
// the path a FRESH activity takes, so without it a fan-out to a struggling
|
|
215
|
+
// server opened one socket per follower before any of this applied.
|
|
216
|
+
_queueIfCooling(inbox, signed) {
|
|
181
217
|
const host = hostOf(inbox);
|
|
182
218
|
const until = this._cooling?.get(host);
|
|
183
|
-
if (until
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
this._enqueue({ inbox, activity: signed, attempts: 1, nextAt: Date.now() + wait });
|
|
219
|
+
if (!until || until <= Date.now()) return false;
|
|
220
|
+
this.log(`${host} is cooling — queueing ${signed.type} rather than asking again`);
|
|
221
|
+
this._enqueue({ inbox, activity: signed, attempts: 1, nextAt: until });
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// What one attempt's outcome means for the queue.
|
|
226
|
+
async _settle(inbox, signed, result) {
|
|
227
|
+
if (result.ok) { this.log(`delivered ${signed.type} → ${inbox}`); return; }
|
|
228
|
+
const e = result.error;
|
|
229
|
+
if (unsalvageable(e)) { await this._unsalvageable(inbox, e); return; }
|
|
230
|
+
this.log(`delivery failed (${e.message}) — queued`);
|
|
231
|
+
const wait = e.retryAfterMs || 60_000;
|
|
232
|
+
if (aboutTheHost(e)) {
|
|
233
|
+
this._cooling ||= new Map();
|
|
234
|
+
this._cooling.set(hostOf(inbox), Date.now() + wait);
|
|
200
235
|
}
|
|
236
|
+
this._enqueue({ inbox, activity: signed, attempts: 1, nextAt: Date.now() + wait });
|
|
201
237
|
}
|
|
202
238
|
|
|
203
239
|
// Not retried, and on a 410 the followers who received there are dropped,
|
|
@@ -228,11 +264,21 @@ export class Deliverer {
|
|
|
228
264
|
}
|
|
229
265
|
|
|
230
266
|
async deliverToAll(inboxes, activity) {
|
|
267
|
+
this._sent(activity);
|
|
231
268
|
// Proved once for the whole fan-out: every recipient gets the same bytes,
|
|
232
269
|
// and one signature is computed rather than one per follower.
|
|
233
270
|
const signed = await this.proofed(activity);
|
|
234
|
-
// Shared inboxes deduplicate fan-out to the same server.
|
|
235
|
-
|
|
271
|
+
// Shared inboxes deduplicate fan-out to the same server. Sent in batches
|
|
272
|
+
// of batchSize, and a host found cooling by one batch is not asked again
|
|
273
|
+
// by the next.
|
|
274
|
+
const targets = [...new Set(inboxes)].filter((inbox) => !this._blockedInbox(inbox))
|
|
275
|
+
.map((inbox) => ({ inbox, activity: signed }));
|
|
276
|
+
for (let i = 0; i < targets.length; i += this.batchSize) {
|
|
277
|
+
const chunk = targets.slice(i, i + this.batchSize).filter((t) => !this._queueIfCooling(t.inbox, signed));
|
|
278
|
+
if (!chunk.length) continue;
|
|
279
|
+
const results = await this.deliverManyNow(chunk);
|
|
280
|
+
for (let k = 0; k < chunk.length; k++) await this._settle(chunk[k].inbox, signed, results[k]);
|
|
281
|
+
}
|
|
236
282
|
}
|
|
237
283
|
|
|
238
284
|
// Serialized, for the same reason Intake.drain is: the tick is 60s and a
|
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
import { dropFollower } from '../store.mjs';
|
|
6
6
|
import { emojisOf, pollOf, isContentType, trimActivity, reactionOf, quoteOf, quotePolicyOf, authorOf } from './activity.mjs';
|
|
7
7
|
|
|
8
|
+
// How long a follower's refollow waits before it is answered again.
|
|
9
|
+
const REACCEPT_MS = 60 * 60 * 1000;
|
|
10
|
+
|
|
8
11
|
const idOf = (v) => (typeof v === 'string' ? v : v?.id);
|
|
9
12
|
|
|
10
13
|
// §7.6 Add / §7.9 Remove. The side effect would be to add or remove the object
|
|
@@ -98,6 +101,18 @@ export async function onFollow(intake, activity, actor, { trusted = false } = {}
|
|
|
98
101
|
await intake.republish({ followers: true });
|
|
99
102
|
intake.log(`new follower: ${actor}`);
|
|
100
103
|
}
|
|
104
|
+
// A refollow from someone already following is answered again — their server
|
|
105
|
+
// may have lost our Accept — but not every time: the same Follow sent over and
|
|
106
|
+
// over would otherwise make us send, and record, an Accept for each one.
|
|
107
|
+
if (existing) {
|
|
108
|
+
const c = intake.store.getContacts();
|
|
109
|
+
const f = c.followers.find(x => x.actor === actor);
|
|
110
|
+
if (f?.acceptedAt && Date.now() - Date.parse(f.acceptedAt) < REACCEPT_MS) {
|
|
111
|
+
intake.log(`refollow from ${actor} — already accepted within the hour, not answered again`);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (f) { f.acceptedAt = new Date().toISOString(); intake.store.setContacts(c); }
|
|
115
|
+
}
|
|
101
116
|
const { acceptActivity } = await import('../wire.mjs');
|
|
102
117
|
await intake.deliverer.deliver(doc.inbox,
|
|
103
118
|
acceptActivity({ urls: intake.urls, followActivity: activity, serial: intake.serial++ }));
|
|
@@ -370,8 +385,9 @@ export async function onDelete(intake, activity, actor) {
|
|
|
370
385
|
await intake.forget(s, { collect: retracted });
|
|
371
386
|
}
|
|
372
387
|
if (retracted.length) {
|
|
373
|
-
const gone = new Set(retracted);
|
|
374
|
-
await intake.publisher.unrecordOutbox(i => gone.has(i?.id)
|
|
388
|
+
const gone = new Set(retracted.map(u => u.object.id));
|
|
389
|
+
await intake.publisher.unrecordOutbox(i => gone.has(i?.id),
|
|
390
|
+
{ record: (was) => (intake.config.private ? null : retracted.filter(u => was.includes(u.object.id))) });
|
|
375
391
|
}
|
|
376
392
|
// Both: an account deletion drops them from followers AND following.
|
|
377
393
|
await intake.republish({ followers: true, following: true });
|
|
@@ -175,7 +175,9 @@ export async function amplify(intake, noteId, { approved = false, activity = nul
|
|
|
175
175
|
// Marked carried before recorded: a failed outbox write costs one missing
|
|
176
176
|
// entry, a failed status write would carry the same post twice.
|
|
177
177
|
intake.store.updateStatus(noteId, { announcedAt: new Date().toISOString(), announceActivity: act });
|
|
178
|
-
|
|
178
|
+
// A private group's carries are written for its members alone, so they go on
|
|
179
|
+
// its owner's full outbox (recorded as sent) and never on the public one.
|
|
180
|
+
if (!intake.config.private) await intake.publisher.recordOutbox(act);
|
|
179
181
|
intake.store.setPending(intake.store.getPending().filter(p => p.noteId !== noteId));
|
|
180
182
|
intake.log(`amplified ${noteId} → ${inboxes.length} inbox(es)`);
|
|
181
183
|
// What the group carried is what the group's conversation holds. A forum
|
|
@@ -51,6 +51,7 @@ const CHAIN_GAP_MS = 5_000; // pause between chained backlog sweeps
|
|
|
51
51
|
// enough that a crash re-does little, large enough that a flood of fast
|
|
52
52
|
// rejections does not become a pod write per item.
|
|
53
53
|
const DELETE_BATCH = 10;
|
|
54
|
+
const ORPHAN_RECEIPTS_PER_SWEEP = 20; // strays from before receipts were deleted with their items
|
|
54
55
|
// Attempt counts live in pod state, not in memory: a restart used to hand every
|
|
55
56
|
// poison item five fresh tries, and under a crash loop that is unbounded.
|
|
56
57
|
const ATTEMPTS_DOC = 'intake-attempts.json';
|
|
@@ -359,6 +360,7 @@ export class Intake {
|
|
|
359
360
|
// deadletter.json where one would do, and a flood is exactly when the pod
|
|
360
361
|
// should be asked for less rather than more.
|
|
361
362
|
const pending = [];
|
|
363
|
+
const withReceipt = new Set(); // items the listing showed a receipt beside
|
|
362
364
|
const flush = async () => {
|
|
363
365
|
if (!pending.length) return true;
|
|
364
366
|
// Written down before any of them leaves the mailbox. A failure here
|
|
@@ -379,13 +381,17 @@ export class Intake {
|
|
|
379
381
|
}
|
|
380
382
|
this._clearAttempt(url);
|
|
381
383
|
handled++;
|
|
384
|
+
// Its receipt goes with it. Left beside a deleted item, a receipt
|
|
385
|
+
// was one stray document per delivery, forever.
|
|
386
|
+
if (withReceipt.has(url)) await podInbox.dropReceiptBeside(this.remote, url);
|
|
382
387
|
await new Promise(r => setTimeout(r, DELETE_GAP_MS));
|
|
383
388
|
}
|
|
384
389
|
return true;
|
|
385
390
|
};
|
|
386
391
|
|
|
387
|
-
for (const { url, size } of items) {
|
|
392
|
+
for (const { url, size, receipt: hasReceipt } of items) {
|
|
388
393
|
if (url.endsWith('.keep')) continue;
|
|
394
|
+
if (hasReceipt) withReceipt.add(url);
|
|
389
395
|
// The listing already carries every child's size, so this costs nothing
|
|
390
396
|
// to ask. An activity is a few kB; anything of this order is not one, and
|
|
391
397
|
// reading it with an unbounded res.text() buffers whatever a stranger
|
|
@@ -430,9 +436,11 @@ export class Intake {
|
|
|
430
436
|
}
|
|
431
437
|
// A gateway that verified this delivery left a receipt beside it. Read
|
|
432
438
|
// it only when a gateway is configured (no config → no fetch, so an
|
|
433
|
-
// install with no gateway pays nothing)
|
|
434
|
-
//
|
|
435
|
-
|
|
439
|
+
// install with no gateway pays nothing) and the listing showed one
|
|
440
|
+
// beside this item (`false` is a definite no; a listing that does not
|
|
441
|
+
// say leaves it to the read); a missing or HMAC-invalid receipt reads
|
|
442
|
+
// as null, which is exactly today's unverified behavior.
|
|
443
|
+
const receipt = activity && hasReceipt !== false ? await this._readReceipt(url) : null;
|
|
436
444
|
if (activity && this.gatewaySecret()) this._bumpGatewayStat(!!receipt?.verified);
|
|
437
445
|
// The owner's own post, taken at the outbox door: not mail to read
|
|
438
446
|
// but a write to make. Never archived or forwarded as if received.
|
|
@@ -464,6 +472,11 @@ export class Intake {
|
|
|
464
472
|
if (pending.length >= DELETE_BATCH && !await flush()) return;
|
|
465
473
|
}
|
|
466
474
|
if (!await this._finishSweep(flush)) return;
|
|
475
|
+
// Receipts earlier drains left behind, a few per sweep until none remain.
|
|
476
|
+
for (const stray of podInbox.orphanReceipts(this.remote, this.urls).slice(0, ORPHAN_RECEIPTS_PER_SWEEP)) {
|
|
477
|
+
if (!await podInbox.dropStrayReceipt(this.remote, stray)) break;
|
|
478
|
+
await new Promise(r => setTimeout(r, DELETE_GAP_MS));
|
|
479
|
+
}
|
|
467
480
|
// Made progress and there is more waiting: go straight round rather than
|
|
468
481
|
// sleeping. Gated on progress so a sweep that achieved nothing — a
|
|
469
482
|
// cooldown, an unwritable store, poison at the head — cannot spin.
|
|
@@ -643,10 +656,33 @@ export class Intake {
|
|
|
643
656
|
// document, not our reading of it); the graph is what it decides from.
|
|
644
657
|
// Returns null when published, else the reason it was not — a dead letter.
|
|
645
658
|
async ownerPostFrom(activity, raw, receipt) {
|
|
659
|
+
// The receipt names the body it vouches for; a body it does not name is
|
|
660
|
+
// not the owner's. A body already handled — a replayed request, or an item
|
|
661
|
+
// the pod would not let us delete — is not published twice.
|
|
662
|
+
const hash = raw ? await sha256Hex(raw) : null;
|
|
663
|
+
if (receipt.hash && receipt.hash !== hash) return 'owner post refused: the body is not the one its receipt names';
|
|
664
|
+
const seen = this.store.read('c2s-seen.json', []);
|
|
665
|
+
if (hash && seen.includes(hash)) {
|
|
666
|
+
this.log('owner post already handled — not published again');
|
|
667
|
+
return null;
|
|
668
|
+
}
|
|
646
669
|
let asSent = null;
|
|
647
670
|
try { asSent = raw ? JSON.parse(raw) : null; } catch { asSent = null; }
|
|
648
|
-
const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null
|
|
649
|
-
|
|
671
|
+
const r = await this.ownerPost(activity, { raw: asSent, slug: receipt.slug || null,
|
|
672
|
+
// What the door told the client this will be called, when it said.
|
|
673
|
+
...(receipt.serial ? { serial: receipt.serial } : {}), ...(receipt.at ? { at: receipt.at } : {}) });
|
|
674
|
+
// Handled, one way or the other: a second sight of it changes nothing. A
|
|
675
|
+
// throw above is not handled, and is tried again.
|
|
676
|
+
if (hash) this.store.write('c2s-seen.json', [hash, ...this.store.read('c2s-seen.json', [])].slice(0, C2S_SEEN_MAX));
|
|
677
|
+
if (!r || r.status >= 300) {
|
|
678
|
+
const why = r?.body?.error || 'no reason was given';
|
|
679
|
+
// The door already answered the app "created"; the owner is the one
|
|
680
|
+
// left to tell.
|
|
681
|
+
await this.publisher.noteToSelf?.(`An app asked your account to publish ${article(activity.type)}, `
|
|
682
|
+
+ `and your account refused it: ${why}. Nothing was sent.`)
|
|
683
|
+
.catch(e => this.log(`could not tell the owner their post was refused: ${e.message}`));
|
|
684
|
+
return `owner post refused (${r?.status || '?'}): ${why}`;
|
|
685
|
+
}
|
|
650
686
|
this.log(`owner post from the outbox door published: ${r.body?.object || r.body?.id || activity.type}`);
|
|
651
687
|
return null;
|
|
652
688
|
}
|
|
@@ -686,3 +722,11 @@ export class Intake {
|
|
|
686
722
|
retract(...a) { return notes.retract(this, ...a); }
|
|
687
723
|
addReply(...a) { return notes.addReply(this, ...a); }
|
|
688
724
|
}
|
|
725
|
+
|
|
726
|
+
// "a Like", "an Update": the kind of activity, as a sentence says it.
|
|
727
|
+
const article = (type) => `${/^[AEIOU]/u.test(String(type)) ? 'an' : 'a'} ${type || 'post'}`;
|
|
728
|
+
|
|
729
|
+
// Bodies the outbox door handed over that were handled, newest first.
|
|
730
|
+
const C2S_SEEN_MAX = 500;
|
|
731
|
+
const sha256Hex = async (text) => [...new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256',
|
|
732
|
+
new TextEncoder().encode(text)))].map(b => b.toString(16).padStart(2, '0')).join('');
|
|
@@ -91,7 +91,11 @@ export async function maybeForward(intake, activity) {
|
|
|
91
91
|
.filter(Boolean))];
|
|
92
92
|
if (!inboxes.length) return;
|
|
93
93
|
intake._forwardBudget -= 1;
|
|
94
|
-
|
|
94
|
+
// Blind copies are for their sender's delivery alone and never travel on
|
|
95
|
+
// (§6, §7.1), from the activity or its object.
|
|
96
|
+
const strip = (o) => { if (!o || typeof o !== 'object') return o; const { bto, bcc, ...rest } = o; return rest; }; // eslint-disable-line no-unused-vars
|
|
97
|
+
const onward = { ...strip(activity), ...(activity.object && typeof activity.object === 'object' ? { object: strip(activity.object) } : {}) };
|
|
98
|
+
await intake.deliverer.deliverToAll(inboxes, onward);
|
|
95
99
|
intake.store.write('forwarded.json', [...forwarded, id].slice(-MAX_FORWARDED));
|
|
96
100
|
intake.log(`forwarded ${activity.type} ${id} to ${inboxes.length} follower inbox(es)`);
|
|
97
101
|
} catch (e) {
|
|
@@ -319,10 +323,13 @@ export async function retract(intake, noteId, { collect = null } = {}) {
|
|
|
319
323
|
if (!s.announceActivity) throw new Error('that post was never carried');
|
|
320
324
|
const { undoActivity } = await import('../wire.mjs');
|
|
321
325
|
const inboxes = intake.announceTargets(s.actor);
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
+
const undo = undoActivity({ urls: intake.urls, activity: s.announceActivity, serial: intake.serial++ });
|
|
327
|
+
await intake.deliverer.deliverToAll(inboxes, undo);
|
|
328
|
+
// The Undo takes the carry's place in the outbox, so a reader catching up
|
|
329
|
+
// from it learns the carry was withdrawn.
|
|
330
|
+
if (collect) collect.push(undo);
|
|
331
|
+
else await intake.publisher.unrecordOutbox(i => i?.id === s.announceActivity.id,
|
|
332
|
+
{ record: () => (intake.config.private ? null : undo) });
|
|
326
333
|
intake.store.updateStatus(noteId, {
|
|
327
334
|
announcedAt: undefined, announceActivity: undefined, retractedAt: new Date().toISOString(),
|
|
328
335
|
});
|