@takosjp/yurucommu-core 3.4.3 → 3.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/migrations/0023_delivery_resolution_outbox.sql +33 -0
- package/migrations/0024_delivery_fanout_outbox.sql +35 -0
- package/migrations/0025_delivery_endpoint_terminal_retention.sql +6 -0
- package/migrations/0026_remote_actor_fetch_failures.sql +29 -0
- package/migrations/0027_remote_actor_tombstones.sql +12 -0
- package/migrations/0028_remote_actor_delivery_fence.sql +30 -0
- package/migrations/0029_delivery_endpoint_recipients.sql +47 -0
- package/package.json +2 -1
- package/packages/api/package.json +1 -1
- package/packages/api/src/lib/api/communities.ts +2 -0
- package/packages/api/src/lib/api/fetch.ts +69 -13
- package/packages/api/src/lib/api/normalize.ts +32 -10
- package/packages/api/src/lib/api/notifications.ts +4 -1
- package/packages/api/src/lib/rtc-client.ts +127 -29
- package/src/backend/federation-helpers.ts +1 -0
- package/src/backend/index.ts +16 -3
- package/src/backend/lib/account-migration.ts +340 -0
- package/src/backend/lib/activity-delete-cascade.ts +193 -0
- package/src/backend/lib/activitypub-actor-cache.ts +615 -48
- package/src/backend/lib/activitypub-actor-identity-sql.ts +179 -0
- package/src/backend/lib/activitypub-actor-identity.ts +39 -0
- package/src/backend/lib/activitypub-validators.ts +62 -4
- package/src/backend/lib/ap-ids.ts +36 -6
- package/src/backend/lib/ap-verify.ts +7 -17
- package/src/backend/lib/blocklist-purge.ts +676 -42
- package/src/backend/lib/blocklist.ts +278 -39
- package/src/backend/lib/community-visibility.ts +47 -33
- package/src/backend/lib/delivery/fanout-outbox.ts +336 -0
- package/src/backend/lib/delivery/planner.ts +16 -12
- package/src/backend/lib/delivery/queue-batching.ts +389 -145
- package/src/backend/lib/delivery/queue-delivery.ts +47 -25
- package/src/backend/lib/delivery/queue.ts +479 -73
- package/src/backend/lib/delivery/resolution-outbox.ts +456 -0
- package/src/backend/lib/delivery/types.ts +31 -7
- package/src/backend/lib/feed-exclude.ts +40 -28
- package/src/backend/lib/follow-edge-mutations.ts +217 -0
- package/src/backend/lib/notification-eligibility.ts +15 -9
- package/src/backend/lib/notification-push.ts +2 -2
- package/src/backend/lib/oidc-id-token.ts +66 -0
- package/src/backend/lib/personal-actor-moderation.ts +239 -0
- package/src/backend/lib/post-visibility.ts +79 -16
- package/src/backend/lib/remote-activity-id.ts +61 -0
- package/src/backend/lib/unread-counts.ts +3 -0
- package/src/backend/retention.ts +38 -1
- package/src/backend/routes/account-teardown.ts +422 -156
- package/src/backend/routes/activitypub/handlers/actor-inbox-handlers.ts +104 -105
- package/src/backend/routes/activitypub/handlers/inbound-community-scope.ts +121 -0
- package/src/backend/routes/activitypub/handlers/inbound-object-identity.ts +54 -0
- package/src/backend/routes/activitypub/handlers/inbound-reply-target.ts +34 -0
- package/src/backend/routes/activitypub/handlers/inbound-story-projection.ts +438 -0
- package/src/backend/routes/activitypub/handlers/inbox-content-handlers.ts +1121 -806
- package/src/backend/routes/activitypub/handlers/inbox-follow-handlers.ts +118 -153
- package/src/backend/routes/activitypub/handlers/inbox-interaction-handlers.ts +185 -168
- package/src/backend/routes/activitypub/handlers/inbox-shared-helpers.ts +220 -32
- package/src/backend/routes/activitypub/inbound-activity-identity.ts +16 -0
- package/src/backend/routes/activitypub/inbound-activity-reference.ts +116 -0
- package/src/backend/routes/activitypub/inbound-addressing.ts +87 -0
- package/src/backend/routes/activitypub/inbox-addressing.ts +22 -19
- package/src/backend/routes/activitypub/inbox-types.ts +14 -2
- package/src/backend/routes/activitypub/inbox.ts +81 -105
- package/src/backend/routes/activitypub/outbox.ts +0 -0
- package/src/backend/routes/activitypub.ts +6 -5
- package/src/backend/routes/actors-helpers.ts +34 -8
- package/src/backend/routes/actors.ts +328 -164
- package/src/backend/routes/auth-helpers.ts +57 -9
- package/src/backend/routes/auth.ts +10 -2
- package/src/backend/routes/communities/membership-invites.ts +4 -1
- package/src/backend/routes/communities/membership-members.ts +201 -73
- package/src/backend/routes/communities/membership-requests.ts +141 -76
- package/src/backend/routes/communities/membership-shared.ts +228 -21
- package/src/backend/routes/communities/messages.ts +9 -2
- package/src/backend/routes/communities/routes.ts +48 -13
- package/src/backend/routes/dm/contacts.ts +29 -41
- package/src/backend/routes/dm/conversations-helpers.ts +9 -1
- package/src/backend/routes/dm/messages.ts +36 -42
- package/src/backend/routes/dm/read-archive.ts +4 -2
- package/src/backend/routes/dm/requests.ts +62 -54
- package/src/backend/routes/follow-helpers.ts +200 -60
- package/src/backend/routes/follow.ts +146 -140
- package/src/backend/routes/media.ts +20 -94
- package/src/backend/routes/moderation.ts +72 -4
- package/src/backend/routes/notes.ts +3 -4
- package/src/backend/routes/notifications.ts +209 -108
- package/src/backend/routes/posts/delete-cascade.ts +253 -89
- package/src/backend/routes/posts/federation.ts +373 -0
- package/src/backend/routes/posts/interactions.ts +160 -184
- package/src/backend/routes/posts/like-mutation.ts +240 -0
- package/src/backend/routes/posts/post-helpers.ts +112 -162
- package/src/backend/routes/posts/queries.ts +150 -54
- package/src/backend/routes/posts/routes.ts +169 -329
- package/src/backend/routes/posts/transformers.ts +61 -6
- package/src/backend/routes/recommendations.ts +37 -44
- package/src/backend/routes/rtc/index.ts +26 -6
- package/src/backend/routes/search.ts +39 -55
- package/src/backend/routes/stories/interactions.ts +22 -6
- package/src/backend/routes/stories/query-helpers.ts +28 -41
- package/src/backend/routes/stories/routes.ts +125 -108
- package/src/backend/routes/takos-tools/dm.ts +17 -17
- package/src/backend/routes/takos-tools/posts.ts +189 -106
- package/src/backend/routes/takos-tools/search.ts +13 -4
- package/src/backend/routes/takos-tools/timeline.ts +35 -27
- package/src/backend/routes/takos-tools-response.ts +6 -2
- package/src/backend/routes/timeline.ts +5 -5
- package/src/backend/runtime/call-signaling-do.ts +35 -4
- package/src/backend/runtime/one-time-ticket.ts +116 -0
- package/src/backend/runtime/realtime-stream-do.ts +5 -61
- package/src/backend/runtime/signaling-hub.ts +36 -3
- package/src/backend/server.ts +95 -80
- package/src/db/d1-write.ts +67 -43
- package/src/db/schema/federation.ts +42 -1
- package/src/db/schema/messaging.ts +110 -2
- package/src/db/schema.ts +1 -1
|
@@ -1,17 +1,112 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
and,
|
|
3
|
+
asc,
|
|
4
|
+
eq,
|
|
5
|
+
gt,
|
|
6
|
+
inArray,
|
|
7
|
+
isNotNull,
|
|
8
|
+
or,
|
|
9
|
+
sql,
|
|
10
|
+
type SQL,
|
|
11
|
+
} from "drizzle-orm";
|
|
12
|
+
import type { SQLiteColumn } from "drizzle-orm/sqlite-core";
|
|
13
|
+
import {
|
|
14
|
+
activities,
|
|
15
|
+
actors,
|
|
16
|
+
announces,
|
|
17
|
+
bookmarks,
|
|
18
|
+
follows,
|
|
19
|
+
likes,
|
|
20
|
+
objects,
|
|
21
|
+
runBatch,
|
|
22
|
+
storyShares,
|
|
23
|
+
storyViews,
|
|
24
|
+
storyVotes,
|
|
25
|
+
} from "../../db/index.ts";
|
|
3
26
|
import type { Database } from "../../db/index.ts";
|
|
4
27
|
import type { IObjectStorage } from "../runtime/types.ts";
|
|
5
|
-
import { chunkForInClause } from "./chunk.ts";
|
|
28
|
+
import { chunkForInClause, D1_IN_CHUNK } from "./chunk.ts";
|
|
6
29
|
import { normalizeDomain } from "./blocklist.ts";
|
|
30
|
+
import { isSameActivityPubActor } from "./activitypub-actor-identity.ts";
|
|
7
31
|
import {
|
|
8
|
-
|
|
32
|
+
deleteObjectsCascade,
|
|
9
33
|
purgeMediaBlobs,
|
|
10
34
|
} from "../routes/posts/delete-cascade.ts";
|
|
11
35
|
import { logger } from "./logger.ts";
|
|
36
|
+
import { deleteActivitiesCascade } from "./activity-delete-cascade.ts";
|
|
12
37
|
|
|
13
38
|
const log = logger.child({ component: "blocklist" });
|
|
14
39
|
|
|
40
|
+
// Actor identity equivalence cannot use a simple indexed equality lookup: the
|
|
41
|
+
// accepted cosmetic spellings include host case, default ports, fragments,
|
|
42
|
+
// and one trailing slash. Scan retained identity columns in fixed-size pages
|
|
43
|
+
// instead of re-materializing the whole table once per delete chunk.
|
|
44
|
+
const ACTOR_IDENTITY_SCAN_PAGE = 512;
|
|
45
|
+
|
|
46
|
+
// An exact-actor purge can carry both a page of retained raw actor spellings
|
|
47
|
+
// and a page of affected object/counterpart ids in the same DELETE. Keep their
|
|
48
|
+
// combined bound parameters below D1's 100-variable ceiling with headroom for
|
|
49
|
+
// any fixed predicate values. Domain purges bind two additional host values.
|
|
50
|
+
const INTERACTION_EDGE_PAGE = 40;
|
|
51
|
+
|
|
52
|
+
export interface BlocklistContentPurgeResult {
|
|
53
|
+
complete: boolean;
|
|
54
|
+
deletedObjects: number;
|
|
55
|
+
deletedActivities: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Match an HTTP(S) ActivityPub URL by its exact hostname or a real subdomain.
|
|
60
|
+
*
|
|
61
|
+
* Do not replace this with LIKE. Cloudflare D1 rejects sufficiently long LIKE
|
|
62
|
+
* patterns as too complex, which previously made a valid long domain block
|
|
63
|
+
* retain all historical content. Extracting the authority with literal
|
|
64
|
+
* instr/substr operations also keeps domain text in the URL path, lookalike
|
|
65
|
+
* suffixes, and credentials outside the match. Parse the authority down to its
|
|
66
|
+
* hostname so an explicit HTTPS port and a DNS root dot have the same meaning
|
|
67
|
+
* as the block decision path.
|
|
68
|
+
*/
|
|
69
|
+
function activityPubUrlHostMatchesDomain(
|
|
70
|
+
column: SQLiteColumn,
|
|
71
|
+
domain: string,
|
|
72
|
+
): SQL {
|
|
73
|
+
const lowerUrl = sql`lower(${column})`;
|
|
74
|
+
const schemeSeparator = sql`instr(${lowerUrl}, '://')`;
|
|
75
|
+
const scheme = sql`substr(${lowerUrl}, 1, ${schemeSeparator} - 1)`;
|
|
76
|
+
const authorityTail = sql`substr(${lowerUrl}, ${schemeSeparator} + 3)`;
|
|
77
|
+
const slash = sql`instr(${authorityTail}, '/')`;
|
|
78
|
+
const authority = sql`substr(${authorityTail}, 1, ${slash} - 1)`;
|
|
79
|
+
const closingBracket = sql`instr(${authority}, ']')`;
|
|
80
|
+
const hostWithRootDot = sql`
|
|
81
|
+
CASE
|
|
82
|
+
WHEN substr(${authority}, 1, 1) = '[' AND ${closingBracket} > 1
|
|
83
|
+
THEN substr(${authority}, 1, ${closingBracket})
|
|
84
|
+
WHEN instr(${authority}, ':') > 0
|
|
85
|
+
THEN substr(${authority}, 1, instr(${authority}, ':') - 1)
|
|
86
|
+
ELSE ${authority}
|
|
87
|
+
END
|
|
88
|
+
`;
|
|
89
|
+
const host = sql`
|
|
90
|
+
CASE
|
|
91
|
+
WHEN substr(${hostWithRootDot}, -1) = '.'
|
|
92
|
+
THEN substr(${hostWithRootDot}, 1, length(${hostWithRootDot}) - 1)
|
|
93
|
+
ELSE ${hostWithRootDot}
|
|
94
|
+
END
|
|
95
|
+
`;
|
|
96
|
+
const subdomainSuffix = `.${domain}`;
|
|
97
|
+
|
|
98
|
+
return sql`
|
|
99
|
+
${schemeSeparator} > 1
|
|
100
|
+
AND ${scheme} IN ('http', 'https')
|
|
101
|
+
AND ${slash} > 1
|
|
102
|
+
AND instr(${authority}, '@') = 0
|
|
103
|
+
AND (
|
|
104
|
+
${host} = ${domain}
|
|
105
|
+
OR substr(${host}, -length(${subdomainSuffix})) = ${subdomainSuffix}
|
|
106
|
+
)
|
|
107
|
+
`;
|
|
108
|
+
}
|
|
109
|
+
|
|
15
110
|
// Hard-delete a set of objects (with their child cascade + R2 blobs). Shared by
|
|
16
111
|
// the actor / domain purge below. apIds are EXACTLY the objects to remove.
|
|
17
112
|
async function purgeObjects(
|
|
@@ -20,90 +115,629 @@ async function purgeObjects(
|
|
|
20
115
|
media?: IObjectStorage,
|
|
21
116
|
): Promise<void> {
|
|
22
117
|
if (apIds.length === 0) return;
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
118
|
+
const parentRows = await db
|
|
119
|
+
.selectDistinct({ apId: objects.inReplyTo })
|
|
120
|
+
.from(objects)
|
|
121
|
+
.where(and(inArray(objects.apId, apIds), isNotNull(objects.inReplyTo)));
|
|
122
|
+
const parentApIds = parentRows.flatMap((row) =>
|
|
123
|
+
typeof row.apId === "string" ? [row.apId] : [],
|
|
124
|
+
);
|
|
125
|
+
const mediaKeys = await deleteObjectsCascade(db, apIds, media);
|
|
27
126
|
for (const chunk of chunkForInClause(apIds)) {
|
|
28
|
-
|
|
127
|
+
// Co-commit the authored-object delete with an exact recomputation of every
|
|
128
|
+
// surviving parent touched by this page. A retry can therefore never see a
|
|
129
|
+
// deleted reply whose contribution remains stranded in reply_count.
|
|
130
|
+
await runBatch(db, [
|
|
131
|
+
db.delete(objects).where(inArray(objects.apId, chunk)),
|
|
132
|
+
db
|
|
133
|
+
.update(objects)
|
|
134
|
+
.set({
|
|
135
|
+
replyCount: sql`(SELECT COUNT(*) FROM ${objects} AS retained_reply WHERE retained_reply.in_reply_to = ${objects.apId})`,
|
|
136
|
+
})
|
|
137
|
+
.where(inArray(objects.apId, parentApIds)),
|
|
138
|
+
]);
|
|
29
139
|
}
|
|
30
140
|
await purgeMediaBlobs(media, mediaKeys);
|
|
31
141
|
}
|
|
32
142
|
|
|
143
|
+
type ActorEdgeWhere = (column: SQLiteColumn) => SQL;
|
|
144
|
+
|
|
145
|
+
type InteractionEdge = { actorApId: string; targetApId: string };
|
|
146
|
+
|
|
147
|
+
async function drainAffectedEdges(
|
|
148
|
+
selectPage: () => Promise<InteractionEdge[]>,
|
|
149
|
+
mutatePage: (edges: InteractionEdge[]) => Promise<void>,
|
|
150
|
+
): Promise<void> {
|
|
151
|
+
while (true) {
|
|
152
|
+
const rows = await selectPage();
|
|
153
|
+
if (rows.length === 0) return;
|
|
154
|
+
await mutatePage(rows);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function exactEdgePairsWhere(
|
|
159
|
+
actorColumn: SQLiteColumn,
|
|
160
|
+
targetColumn: SQLiteColumn,
|
|
161
|
+
edges: InteractionEdge[],
|
|
162
|
+
): SQL {
|
|
163
|
+
return or(
|
|
164
|
+
...edges.map((edge) =>
|
|
165
|
+
and(eq(actorColumn, edge.actorApId), eq(targetColumn, edge.targetApId)),
|
|
166
|
+
),
|
|
167
|
+
)!;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function affectedTargetApIds(edges: InteractionEdge[]): string[] {
|
|
171
|
+
return [...new Set(edges.map((edge) => edge.targetApId))];
|
|
172
|
+
}
|
|
173
|
+
|
|
33
174
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* Best-effort; never throws into the operator's response path.
|
|
175
|
+
* Remove every retained interaction performed by one actor predicate against
|
|
176
|
+
* still-present content and reconcile denormalized counters from the surviving
|
|
177
|
+
* edges. Each page is one atomic delete+recompute unit, so a 503 retry resumes
|
|
178
|
+
* from remaining edges without leaving an already-deleted contribution in a
|
|
179
|
+
* visible count.
|
|
40
180
|
*/
|
|
41
|
-
|
|
181
|
+
async function purgeRetainedInteractionEdges(
|
|
182
|
+
db: Database,
|
|
183
|
+
actorWhere: ActorEdgeWhere,
|
|
184
|
+
): Promise<void> {
|
|
185
|
+
await drainAffectedEdges(
|
|
186
|
+
() =>
|
|
187
|
+
db
|
|
188
|
+
.select({ actorApId: likes.actorApId, targetApId: likes.objectApId })
|
|
189
|
+
.from(likes)
|
|
190
|
+
.where(actorWhere(likes.actorApId))
|
|
191
|
+
.orderBy(asc(likes.actorApId), asc(likes.objectApId))
|
|
192
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
193
|
+
(edges) => {
|
|
194
|
+
const targetApIds = affectedTargetApIds(edges);
|
|
195
|
+
return runBatch(db, [
|
|
196
|
+
db
|
|
197
|
+
.delete(likes)
|
|
198
|
+
.where(exactEdgePairsWhere(likes.actorApId, likes.objectApId, edges)),
|
|
199
|
+
db
|
|
200
|
+
.update(objects)
|
|
201
|
+
.set({
|
|
202
|
+
likeCount: sql`(SELECT COUNT(*) FROM ${likes} WHERE ${likes.objectApId} = ${objects.apId})`,
|
|
203
|
+
})
|
|
204
|
+
.where(inArray(objects.apId, targetApIds)),
|
|
205
|
+
]);
|
|
206
|
+
},
|
|
207
|
+
);
|
|
208
|
+
|
|
209
|
+
await drainAffectedEdges(
|
|
210
|
+
() =>
|
|
211
|
+
db
|
|
212
|
+
.select({
|
|
213
|
+
actorApId: announces.actorApId,
|
|
214
|
+
targetApId: announces.objectApId,
|
|
215
|
+
})
|
|
216
|
+
.from(announces)
|
|
217
|
+
.where(actorWhere(announces.actorApId))
|
|
218
|
+
.orderBy(asc(announces.actorApId), asc(announces.objectApId))
|
|
219
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
220
|
+
(edges) => {
|
|
221
|
+
const targetApIds = affectedTargetApIds(edges);
|
|
222
|
+
return runBatch(db, [
|
|
223
|
+
db
|
|
224
|
+
.delete(announces)
|
|
225
|
+
.where(
|
|
226
|
+
exactEdgePairsWhere(
|
|
227
|
+
announces.actorApId,
|
|
228
|
+
announces.objectApId,
|
|
229
|
+
edges,
|
|
230
|
+
),
|
|
231
|
+
),
|
|
232
|
+
db
|
|
233
|
+
.update(objects)
|
|
234
|
+
.set({
|
|
235
|
+
announceCount: sql`(SELECT COUNT(*) FROM ${announces} WHERE ${announces.objectApId} = ${objects.apId})`,
|
|
236
|
+
})
|
|
237
|
+
.where(inArray(objects.apId, targetApIds)),
|
|
238
|
+
]);
|
|
239
|
+
},
|
|
240
|
+
);
|
|
241
|
+
|
|
242
|
+
await drainAffectedEdges(
|
|
243
|
+
() =>
|
|
244
|
+
db
|
|
245
|
+
.select({
|
|
246
|
+
actorApId: storyShares.actorApId,
|
|
247
|
+
targetApId: storyShares.storyApId,
|
|
248
|
+
})
|
|
249
|
+
.from(storyShares)
|
|
250
|
+
.where(actorWhere(storyShares.actorApId))
|
|
251
|
+
.orderBy(asc(storyShares.actorApId), asc(storyShares.storyApId))
|
|
252
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
253
|
+
(edges) => {
|
|
254
|
+
const targetApIds = affectedTargetApIds(edges);
|
|
255
|
+
return runBatch(db, [
|
|
256
|
+
db
|
|
257
|
+
.delete(storyShares)
|
|
258
|
+
.where(
|
|
259
|
+
exactEdgePairsWhere(
|
|
260
|
+
storyShares.actorApId,
|
|
261
|
+
storyShares.storyApId,
|
|
262
|
+
edges,
|
|
263
|
+
),
|
|
264
|
+
),
|
|
265
|
+
db
|
|
266
|
+
.update(objects)
|
|
267
|
+
.set({
|
|
268
|
+
shareCount: sql`(SELECT COUNT(*) FROM ${storyShares} WHERE ${storyShares.storyApId} = ${objects.apId})`,
|
|
269
|
+
})
|
|
270
|
+
.where(inArray(objects.apId, targetApIds)),
|
|
271
|
+
]);
|
|
272
|
+
},
|
|
273
|
+
);
|
|
274
|
+
|
|
275
|
+
await drainAffectedEdges(
|
|
276
|
+
() =>
|
|
277
|
+
db
|
|
278
|
+
.select({
|
|
279
|
+
actorApId: bookmarks.actorApId,
|
|
280
|
+
targetApId: bookmarks.objectApId,
|
|
281
|
+
})
|
|
282
|
+
.from(bookmarks)
|
|
283
|
+
.where(actorWhere(bookmarks.actorApId))
|
|
284
|
+
.orderBy(asc(bookmarks.actorApId), asc(bookmarks.objectApId))
|
|
285
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
286
|
+
async (edges) => {
|
|
287
|
+
await db
|
|
288
|
+
.delete(bookmarks)
|
|
289
|
+
.where(
|
|
290
|
+
exactEdgePairsWhere(bookmarks.actorApId, bookmarks.objectApId, edges),
|
|
291
|
+
);
|
|
292
|
+
},
|
|
293
|
+
);
|
|
294
|
+
|
|
295
|
+
await drainAffectedEdges(
|
|
296
|
+
() =>
|
|
297
|
+
db
|
|
298
|
+
.select({
|
|
299
|
+
actorApId: storyViews.actorApId,
|
|
300
|
+
targetApId: storyViews.storyApId,
|
|
301
|
+
})
|
|
302
|
+
.from(storyViews)
|
|
303
|
+
.where(actorWhere(storyViews.actorApId))
|
|
304
|
+
.orderBy(asc(storyViews.actorApId), asc(storyViews.storyApId))
|
|
305
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
306
|
+
async (edges) => {
|
|
307
|
+
await db
|
|
308
|
+
.delete(storyViews)
|
|
309
|
+
.where(
|
|
310
|
+
exactEdgePairsWhere(
|
|
311
|
+
storyViews.actorApId,
|
|
312
|
+
storyViews.storyApId,
|
|
313
|
+
edges,
|
|
314
|
+
),
|
|
315
|
+
);
|
|
316
|
+
},
|
|
317
|
+
);
|
|
318
|
+
|
|
319
|
+
await drainAffectedEdges(
|
|
320
|
+
() =>
|
|
321
|
+
db
|
|
322
|
+
.select({
|
|
323
|
+
actorApId: storyVotes.actorApId,
|
|
324
|
+
targetApId: storyVotes.storyApId,
|
|
325
|
+
})
|
|
326
|
+
.from(storyVotes)
|
|
327
|
+
.where(actorWhere(storyVotes.actorApId))
|
|
328
|
+
.orderBy(asc(storyVotes.actorApId), asc(storyVotes.storyApId))
|
|
329
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
330
|
+
async (edges) => {
|
|
331
|
+
await db
|
|
332
|
+
.delete(storyVotes)
|
|
333
|
+
.where(
|
|
334
|
+
exactEdgePairsWhere(
|
|
335
|
+
storyVotes.actorApId,
|
|
336
|
+
storyVotes.storyApId,
|
|
337
|
+
edges,
|
|
338
|
+
),
|
|
339
|
+
);
|
|
340
|
+
},
|
|
341
|
+
);
|
|
342
|
+
|
|
343
|
+
// A blocked remote following a local actor contributes to follower_count;
|
|
344
|
+
// a local actor following the blocked remote contributes to following_count.
|
|
345
|
+
// Delete first, then recompute the local counterpart in the same batch.
|
|
346
|
+
await drainAffectedEdges(
|
|
347
|
+
() =>
|
|
348
|
+
db
|
|
349
|
+
.select({
|
|
350
|
+
actorApId: follows.followerApId,
|
|
351
|
+
targetApId: follows.followingApId,
|
|
352
|
+
})
|
|
353
|
+
.from(follows)
|
|
354
|
+
.where(actorWhere(follows.followerApId))
|
|
355
|
+
.orderBy(asc(follows.followerApId), asc(follows.followingApId))
|
|
356
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
357
|
+
(edges) => {
|
|
358
|
+
const targetApIds = affectedTargetApIds(edges);
|
|
359
|
+
return runBatch(db, [
|
|
360
|
+
db
|
|
361
|
+
.delete(follows)
|
|
362
|
+
.where(
|
|
363
|
+
exactEdgePairsWhere(
|
|
364
|
+
follows.followerApId,
|
|
365
|
+
follows.followingApId,
|
|
366
|
+
edges,
|
|
367
|
+
),
|
|
368
|
+
),
|
|
369
|
+
db
|
|
370
|
+
.update(actors)
|
|
371
|
+
.set({
|
|
372
|
+
followerCount: sql`(SELECT COUNT(*) FROM ${follows} WHERE ${follows.followingApId} = ${actors.apId} AND ${follows.status} = 'accepted')`,
|
|
373
|
+
})
|
|
374
|
+
.where(inArray(actors.apId, targetApIds)),
|
|
375
|
+
]);
|
|
376
|
+
},
|
|
377
|
+
);
|
|
378
|
+
|
|
379
|
+
await drainAffectedEdges(
|
|
380
|
+
() =>
|
|
381
|
+
db
|
|
382
|
+
.select({
|
|
383
|
+
actorApId: follows.followingApId,
|
|
384
|
+
targetApId: follows.followerApId,
|
|
385
|
+
})
|
|
386
|
+
.from(follows)
|
|
387
|
+
.where(actorWhere(follows.followingApId))
|
|
388
|
+
.orderBy(asc(follows.followingApId), asc(follows.followerApId))
|
|
389
|
+
.limit(INTERACTION_EDGE_PAGE),
|
|
390
|
+
(edges) => {
|
|
391
|
+
const targetApIds = affectedTargetApIds(edges);
|
|
392
|
+
return runBatch(db, [
|
|
393
|
+
db
|
|
394
|
+
.delete(follows)
|
|
395
|
+
.where(
|
|
396
|
+
exactEdgePairsWhere(
|
|
397
|
+
follows.followingApId,
|
|
398
|
+
follows.followerApId,
|
|
399
|
+
edges,
|
|
400
|
+
),
|
|
401
|
+
),
|
|
402
|
+
db
|
|
403
|
+
.update(actors)
|
|
404
|
+
.set({
|
|
405
|
+
followingCount: sql`(SELECT COUNT(*) FROM ${follows} WHERE ${follows.followerApId} = ${actors.apId} AND ${follows.status} = 'accepted')`,
|
|
406
|
+
})
|
|
407
|
+
.where(inArray(actors.apId, targetApIds)),
|
|
408
|
+
]);
|
|
409
|
+
},
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
type ActorIdentityPage = (
|
|
414
|
+
cursor: string | undefined,
|
|
415
|
+
) => Promise<{ actorApId: string }[]>;
|
|
416
|
+
|
|
417
|
+
async function purgeActorInteractionIdentityPages(
|
|
418
|
+
db: Database,
|
|
419
|
+
blockedApId: string,
|
|
420
|
+
selectPage: ActorIdentityPage,
|
|
421
|
+
): Promise<void> {
|
|
422
|
+
let cursor: string | undefined;
|
|
423
|
+
while (true) {
|
|
424
|
+
const rows = await selectPage(cursor);
|
|
425
|
+
if (rows.length === 0) return;
|
|
426
|
+
cursor = rows[rows.length - 1]!.actorApId;
|
|
427
|
+
const aliases = rows
|
|
428
|
+
.map((row) => row.actorApId)
|
|
429
|
+
.filter((actorApId) => isSameActivityPubActor(actorApId, blockedApId));
|
|
430
|
+
for (
|
|
431
|
+
let index = 0;
|
|
432
|
+
index < aliases.length;
|
|
433
|
+
index += INTERACTION_EDGE_PAGE
|
|
434
|
+
) {
|
|
435
|
+
const chunk = aliases.slice(index, index + INTERACTION_EDGE_PAGE);
|
|
436
|
+
await purgeRetainedInteractionEdges(db, (column) =>
|
|
437
|
+
inArray(column, chunk),
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
/**
|
|
444
|
+
* Scan each interaction identity column in bounded distinct pages. Keep these
|
|
445
|
+
* as separate selects: Cloudflare D1 rejects the formerly convenient eight-way
|
|
446
|
+
* UNION with `too many terms in compound SELECT`, even though libsql accepts it.
|
|
447
|
+
* Purging a discovered alias across every table before scanning the next table
|
|
448
|
+
* is safe and reduces later pages without losing aliases unique to that table.
|
|
449
|
+
*/
|
|
450
|
+
async function purgeActorInteractionEdges(
|
|
42
451
|
db: Database,
|
|
43
452
|
blockedApId: string,
|
|
453
|
+
): Promise<void> {
|
|
454
|
+
const scan = (selectPage: ActorIdentityPage) =>
|
|
455
|
+
purgeActorInteractionIdentityPages(db, blockedApId, selectPage);
|
|
456
|
+
|
|
457
|
+
await scan((cursor) =>
|
|
458
|
+
db
|
|
459
|
+
.selectDistinct({ actorApId: likes.actorApId })
|
|
460
|
+
.from(likes)
|
|
461
|
+
.where(cursor === undefined ? undefined : gt(likes.actorApId, cursor))
|
|
462
|
+
.orderBy(asc(likes.actorApId))
|
|
463
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
464
|
+
);
|
|
465
|
+
await scan((cursor) =>
|
|
466
|
+
db
|
|
467
|
+
.selectDistinct({ actorApId: announces.actorApId })
|
|
468
|
+
.from(announces)
|
|
469
|
+
.where(cursor === undefined ? undefined : gt(announces.actorApId, cursor))
|
|
470
|
+
.orderBy(asc(announces.actorApId))
|
|
471
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
472
|
+
);
|
|
473
|
+
await scan((cursor) =>
|
|
474
|
+
db
|
|
475
|
+
.selectDistinct({ actorApId: bookmarks.actorApId })
|
|
476
|
+
.from(bookmarks)
|
|
477
|
+
.where(cursor === undefined ? undefined : gt(bookmarks.actorApId, cursor))
|
|
478
|
+
.orderBy(asc(bookmarks.actorApId))
|
|
479
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
480
|
+
);
|
|
481
|
+
await scan((cursor) =>
|
|
482
|
+
db
|
|
483
|
+
.selectDistinct({ actorApId: storyShares.actorApId })
|
|
484
|
+
.from(storyShares)
|
|
485
|
+
.where(
|
|
486
|
+
cursor === undefined ? undefined : gt(storyShares.actorApId, cursor),
|
|
487
|
+
)
|
|
488
|
+
.orderBy(asc(storyShares.actorApId))
|
|
489
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
490
|
+
);
|
|
491
|
+
await scan((cursor) =>
|
|
492
|
+
db
|
|
493
|
+
.selectDistinct({ actorApId: storyViews.actorApId })
|
|
494
|
+
.from(storyViews)
|
|
495
|
+
.where(
|
|
496
|
+
cursor === undefined ? undefined : gt(storyViews.actorApId, cursor),
|
|
497
|
+
)
|
|
498
|
+
.orderBy(asc(storyViews.actorApId))
|
|
499
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
500
|
+
);
|
|
501
|
+
await scan((cursor) =>
|
|
502
|
+
db
|
|
503
|
+
.selectDistinct({ actorApId: storyVotes.actorApId })
|
|
504
|
+
.from(storyVotes)
|
|
505
|
+
.where(
|
|
506
|
+
cursor === undefined ? undefined : gt(storyVotes.actorApId, cursor),
|
|
507
|
+
)
|
|
508
|
+
.orderBy(asc(storyVotes.actorApId))
|
|
509
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
510
|
+
);
|
|
511
|
+
await scan((cursor) =>
|
|
512
|
+
db
|
|
513
|
+
.selectDistinct({ actorApId: follows.followerApId })
|
|
514
|
+
.from(follows)
|
|
515
|
+
.where(
|
|
516
|
+
cursor === undefined ? undefined : gt(follows.followerApId, cursor),
|
|
517
|
+
)
|
|
518
|
+
.orderBy(asc(follows.followerApId))
|
|
519
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
520
|
+
);
|
|
521
|
+
await scan((cursor) =>
|
|
522
|
+
db
|
|
523
|
+
.selectDistinct({ actorApId: follows.followingApId })
|
|
524
|
+
.from(follows)
|
|
525
|
+
.where(
|
|
526
|
+
cursor === undefined ? undefined : gt(follows.followingApId, cursor),
|
|
527
|
+
)
|
|
528
|
+
.orderBy(asc(follows.followingApId))
|
|
529
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE),
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/**
|
|
534
|
+
* Delete a predicate's objects in bounded, retryable units.
|
|
535
|
+
*
|
|
536
|
+
* Selecting every matching id before the first delete made a large domain
|
|
537
|
+
* block consume memory in direct proportion to retained history. Keeping each
|
|
538
|
+
* unit at D1_IN_CHUNK also means a failed statement leaves at most one page
|
|
539
|
+
* incomplete; retrying the idempotent operator block resumes from the rows that
|
|
540
|
+
* remain instead of rebuilding an unbounded in-memory id set. A stable AP-ID
|
|
541
|
+
* keyset prevents unmatched history before the current page from being scanned
|
|
542
|
+
* again after each successful delete.
|
|
543
|
+
*/
|
|
544
|
+
async function purgeMatchingObjects(
|
|
545
|
+
db: Database,
|
|
546
|
+
where: SQL,
|
|
44
547
|
media?: IObjectStorage,
|
|
548
|
+
onPageDeleted?: (count: number) => void,
|
|
45
549
|
): Promise<void> {
|
|
46
|
-
|
|
550
|
+
let cursor: string | undefined;
|
|
551
|
+
while (true) {
|
|
47
552
|
const rows = await db
|
|
48
553
|
.select({ apId: objects.apId })
|
|
49
554
|
.from(objects)
|
|
50
|
-
.where(
|
|
555
|
+
.where(
|
|
556
|
+
cursor === undefined ? where : and(where, gt(objects.apId, cursor)),
|
|
557
|
+
)
|
|
558
|
+
.orderBy(asc(objects.apId))
|
|
559
|
+
.limit(D1_IN_CHUNK);
|
|
560
|
+
if (rows.length === 0) return;
|
|
561
|
+
cursor = rows[rows.length - 1]!.apId;
|
|
51
562
|
await purgeObjects(
|
|
52
563
|
db,
|
|
53
|
-
rows.map((
|
|
564
|
+
rows.map((row) => row.apId),
|
|
54
565
|
media,
|
|
55
566
|
);
|
|
56
|
-
|
|
567
|
+
onPageDeleted?.(rows.length);
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
async function purgeMatchingActivities(
|
|
572
|
+
db: Database,
|
|
573
|
+
where: SQL,
|
|
574
|
+
onPageDeleted?: (count: number) => void,
|
|
575
|
+
): Promise<void> {
|
|
576
|
+
let cursor: string | undefined;
|
|
577
|
+
while (true) {
|
|
578
|
+
const rows = await db
|
|
579
|
+
.select({ apId: activities.apId })
|
|
580
|
+
.from(activities)
|
|
581
|
+
.where(
|
|
582
|
+
cursor === undefined ? where : and(where, gt(activities.apId, cursor)),
|
|
583
|
+
)
|
|
584
|
+
.orderBy(asc(activities.apId))
|
|
585
|
+
.limit(D1_IN_CHUNK);
|
|
586
|
+
if (rows.length === 0) return;
|
|
587
|
+
cursor = rows[rows.length - 1]!.apId;
|
|
588
|
+
await deleteActivitiesCascade(
|
|
589
|
+
db,
|
|
590
|
+
inArray(
|
|
591
|
+
activities.apId,
|
|
592
|
+
rows.map((row) => row.apId),
|
|
593
|
+
),
|
|
594
|
+
);
|
|
595
|
+
onPageDeleted?.(rows.length);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Scan retained objects once by stable AP-ID and remove matching actor rows in
|
|
601
|
+
* D1-sized units. The operator block is active before this cleanup starts, so
|
|
602
|
+
* the scan does not have to restart from the beginning after every delete.
|
|
603
|
+
*/
|
|
604
|
+
async function purgeActorObjects(
|
|
605
|
+
db: Database,
|
|
606
|
+
blockedApId: string,
|
|
607
|
+
media?: IObjectStorage,
|
|
608
|
+
onPageDeleted?: (count: number) => void,
|
|
609
|
+
): Promise<void> {
|
|
610
|
+
let cursor: string | undefined;
|
|
611
|
+
while (true) {
|
|
612
|
+
const rows = await db
|
|
613
|
+
.select({ apId: objects.apId, attributedTo: objects.attributedTo })
|
|
614
|
+
.from(objects)
|
|
615
|
+
.where(cursor === undefined ? undefined : gt(objects.apId, cursor))
|
|
616
|
+
.orderBy(asc(objects.apId))
|
|
617
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE);
|
|
618
|
+
if (rows.length === 0) return;
|
|
619
|
+
cursor = rows[rows.length - 1]!.apId;
|
|
620
|
+
|
|
621
|
+
const targetApIds = rows
|
|
622
|
+
.filter((row) => isSameActivityPubActor(row.attributedTo, blockedApId))
|
|
623
|
+
.map((row) => row.apId);
|
|
624
|
+
for (const chunk of chunkForInClause(targetApIds)) {
|
|
625
|
+
await purgeObjects(db, chunk, media);
|
|
626
|
+
onPageDeleted?.(chunk.length);
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
async function purgeActorActivities(
|
|
632
|
+
db: Database,
|
|
633
|
+
blockedApId: string,
|
|
634
|
+
onPageDeleted?: (count: number) => void,
|
|
635
|
+
): Promise<void> {
|
|
636
|
+
let cursor: string | undefined;
|
|
637
|
+
while (true) {
|
|
638
|
+
const rows = await db
|
|
639
|
+
.select({ apId: activities.apId, actorApId: activities.actorApId })
|
|
640
|
+
.from(activities)
|
|
641
|
+
.where(cursor === undefined ? undefined : gt(activities.apId, cursor))
|
|
642
|
+
.orderBy(asc(activities.apId))
|
|
643
|
+
.limit(ACTOR_IDENTITY_SCAN_PAGE);
|
|
644
|
+
if (rows.length === 0) return;
|
|
645
|
+
cursor = rows[rows.length - 1]!.apId;
|
|
646
|
+
|
|
647
|
+
const targetApIds = rows
|
|
648
|
+
.filter((row) => isSameActivityPubActor(row.actorApId, blockedApId))
|
|
649
|
+
.map((row) => row.apId);
|
|
650
|
+
for (const chunk of chunkForInClause(targetApIds)) {
|
|
651
|
+
await deleteActivitiesCascade(db, inArray(activities.apId, chunk));
|
|
652
|
+
onPageDeleted?.(chunk.length);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/**
|
|
658
|
+
* Purge a blocked REMOTE actor's already-ingested content. The operator
|
|
659
|
+
* blocklist is otherwise ingest/delivery-only, so without this a defederated
|
|
660
|
+
* actor's prior posts/replies/stories stay live in timelines, search, and
|
|
661
|
+
* object serving — contradicting the operator's "they're gone" expectation.
|
|
662
|
+
* Removes the actor's authored objects (cascade) + their activity ledger rows.
|
|
663
|
+
* Failures are logged and returned to the operator route. The block mutation
|
|
664
|
+
* remains active, while a non-success response makes retry converge cleanup.
|
|
665
|
+
*/
|
|
666
|
+
export async function purgeActorContent(
|
|
667
|
+
db: Database,
|
|
668
|
+
blockedApId: string,
|
|
669
|
+
media?: IObjectStorage,
|
|
670
|
+
): Promise<BlocklistContentPurgeResult> {
|
|
671
|
+
let deletedObjects = 0;
|
|
672
|
+
let deletedActivities = 0;
|
|
673
|
+
try {
|
|
674
|
+
// Clean still-visible local counterpart state first. If a later media/
|
|
675
|
+
// authored-content page fails, the active block no longer leaves the
|
|
676
|
+
// blocked actor's reactions or relationship counts exposed while the
|
|
677
|
+
// operator retries the idempotent cleanup.
|
|
678
|
+
await purgeActorInteractionEdges(db, blockedApId);
|
|
679
|
+
await purgeActorObjects(db, blockedApId, media, (count) => {
|
|
680
|
+
deletedObjects += count;
|
|
681
|
+
});
|
|
682
|
+
await purgeActorActivities(db, blockedApId, (count) => {
|
|
683
|
+
deletedActivities += count;
|
|
684
|
+
});
|
|
685
|
+
return { complete: true, deletedObjects, deletedActivities };
|
|
57
686
|
} catch (err) {
|
|
58
687
|
log.warn("blocklist.purgeActorContent failed", {
|
|
59
688
|
event: "blocklist.purge_actor_failed",
|
|
60
689
|
actor: blockedApId,
|
|
61
690
|
error: err,
|
|
62
691
|
});
|
|
692
|
+
return { complete: false, deletedObjects, deletedActivities };
|
|
63
693
|
}
|
|
64
694
|
}
|
|
65
695
|
|
|
66
696
|
/**
|
|
67
697
|
* Purge already-ingested content authored by any actor on a blocked DOMAIN (the
|
|
68
|
-
* host itself OR a subdomain). Host-
|
|
698
|
+
* host itself OR a subdomain). Host-boundary matching means `evil.com` matches
|
|
69
699
|
* `https://evil.com/...` and `https://node1.evil.com/...` but NOT `notevil.com`.
|
|
70
|
-
*
|
|
71
|
-
*
|
|
700
|
+
* Failures are logged and returned to the operator route so its response can
|
|
701
|
+
* require a retry. Local content is never matched (local objects carry the
|
|
702
|
+
* local host; the operator never blocks their own domain).
|
|
72
703
|
*/
|
|
73
704
|
export async function purgeDomainContent(
|
|
74
705
|
db: Database,
|
|
75
706
|
domainOrUrl: string,
|
|
76
707
|
media?: IObjectStorage,
|
|
77
|
-
): Promise<
|
|
708
|
+
): Promise<BlocklistContentPurgeResult> {
|
|
78
709
|
const domain = normalizeDomain(domainOrUrl);
|
|
79
|
-
if (!domain)
|
|
710
|
+
if (!domain) {
|
|
711
|
+
return { complete: true, deletedObjects: 0, deletedActivities: 0 };
|
|
712
|
+
}
|
|
713
|
+
let deletedObjects = 0;
|
|
714
|
+
let deletedActivities = 0;
|
|
80
715
|
try {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
like(objects.attributedTo, `https://%.${domain}/%`),
|
|
716
|
+
await purgeRetainedInteractionEdges(db, (column) =>
|
|
717
|
+
activityPubUrlHostMatchesDomain(column, domain),
|
|
84
718
|
);
|
|
85
|
-
|
|
86
|
-
.select({ apId: objects.apId })
|
|
87
|
-
.from(objects)
|
|
88
|
-
.where(hostMatch);
|
|
89
|
-
await purgeObjects(
|
|
719
|
+
await purgeMatchingObjects(
|
|
90
720
|
db,
|
|
91
|
-
|
|
721
|
+
activityPubUrlHostMatchesDomain(objects.attributedTo, domain),
|
|
92
722
|
media,
|
|
723
|
+
(count) => {
|
|
724
|
+
deletedObjects += count;
|
|
725
|
+
},
|
|
93
726
|
);
|
|
94
|
-
await
|
|
95
|
-
|
|
96
|
-
.
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
727
|
+
await purgeMatchingActivities(
|
|
728
|
+
db,
|
|
729
|
+
activityPubUrlHostMatchesDomain(activities.actorApId, domain),
|
|
730
|
+
(count) => {
|
|
731
|
+
deletedActivities += count;
|
|
732
|
+
},
|
|
733
|
+
);
|
|
734
|
+
return { complete: true, deletedObjects, deletedActivities };
|
|
102
735
|
} catch (err) {
|
|
103
736
|
log.warn("blocklist.purgeDomainContent failed", {
|
|
104
737
|
event: "blocklist.purge_domain_failed",
|
|
105
738
|
domain,
|
|
106
739
|
error: err,
|
|
107
740
|
});
|
|
741
|
+
return { complete: false, deletedObjects, deletedActivities };
|
|
108
742
|
}
|
|
109
743
|
}
|