@takosjp/yurucommu-core 3.4.3 → 3.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/migrations/0023_delivery_resolution_outbox.sql +33 -0
  2. package/migrations/0024_delivery_fanout_outbox.sql +35 -0
  3. package/migrations/0025_delivery_endpoint_terminal_retention.sql +6 -0
  4. package/migrations/0026_remote_actor_fetch_failures.sql +29 -0
  5. package/migrations/0027_remote_actor_tombstones.sql +12 -0
  6. package/migrations/0028_remote_actor_delivery_fence.sql +30 -0
  7. package/migrations/0029_delivery_endpoint_recipients.sql +47 -0
  8. package/package.json +2 -1
  9. package/packages/api/package.json +1 -1
  10. package/packages/api/src/lib/api/communities.ts +2 -0
  11. package/packages/api/src/lib/api/fetch.ts +69 -13
  12. package/packages/api/src/lib/api/normalize.ts +32 -10
  13. package/packages/api/src/lib/api/notifications.ts +4 -1
  14. package/packages/api/src/lib/rtc-client.ts +127 -29
  15. package/src/backend/federation-helpers.ts +1 -0
  16. package/src/backend/index.ts +16 -3
  17. package/src/backend/lib/account-migration.ts +340 -0
  18. package/src/backend/lib/activity-delete-cascade.ts +193 -0
  19. package/src/backend/lib/activitypub-actor-cache.ts +615 -48
  20. package/src/backend/lib/activitypub-actor-identity-sql.ts +179 -0
  21. package/src/backend/lib/activitypub-actor-identity.ts +39 -0
  22. package/src/backend/lib/activitypub-validators.ts +62 -4
  23. package/src/backend/lib/ap-ids.ts +36 -6
  24. package/src/backend/lib/ap-verify.ts +7 -17
  25. package/src/backend/lib/blocklist-purge.ts +676 -42
  26. package/src/backend/lib/blocklist.ts +278 -39
  27. package/src/backend/lib/community-visibility.ts +47 -33
  28. package/src/backend/lib/delivery/fanout-outbox.ts +336 -0
  29. package/src/backend/lib/delivery/planner.ts +16 -12
  30. package/src/backend/lib/delivery/queue-batching.ts +389 -145
  31. package/src/backend/lib/delivery/queue-delivery.ts +47 -25
  32. package/src/backend/lib/delivery/queue.ts +479 -73
  33. package/src/backend/lib/delivery/resolution-outbox.ts +456 -0
  34. package/src/backend/lib/delivery/types.ts +31 -7
  35. package/src/backend/lib/feed-exclude.ts +40 -28
  36. package/src/backend/lib/follow-edge-mutations.ts +217 -0
  37. package/src/backend/lib/notification-eligibility.ts +15 -9
  38. package/src/backend/lib/notification-push.ts +2 -2
  39. package/src/backend/lib/oidc-id-token.ts +66 -0
  40. package/src/backend/lib/personal-actor-moderation.ts +239 -0
  41. package/src/backend/lib/post-visibility.ts +79 -16
  42. package/src/backend/lib/remote-activity-id.ts +61 -0
  43. package/src/backend/lib/unread-counts.ts +3 -0
  44. package/src/backend/retention.ts +38 -1
  45. package/src/backend/routes/account-teardown.ts +422 -156
  46. package/src/backend/routes/activitypub/handlers/actor-inbox-handlers.ts +104 -105
  47. package/src/backend/routes/activitypub/handlers/inbound-community-scope.ts +121 -0
  48. package/src/backend/routes/activitypub/handlers/inbound-object-identity.ts +54 -0
  49. package/src/backend/routes/activitypub/handlers/inbound-reply-target.ts +34 -0
  50. package/src/backend/routes/activitypub/handlers/inbound-story-projection.ts +438 -0
  51. package/src/backend/routes/activitypub/handlers/inbox-content-handlers.ts +1121 -806
  52. package/src/backend/routes/activitypub/handlers/inbox-follow-handlers.ts +118 -153
  53. package/src/backend/routes/activitypub/handlers/inbox-interaction-handlers.ts +185 -168
  54. package/src/backend/routes/activitypub/handlers/inbox-shared-helpers.ts +220 -32
  55. package/src/backend/routes/activitypub/inbound-activity-identity.ts +16 -0
  56. package/src/backend/routes/activitypub/inbound-activity-reference.ts +116 -0
  57. package/src/backend/routes/activitypub/inbound-addressing.ts +87 -0
  58. package/src/backend/routes/activitypub/inbox-addressing.ts +22 -19
  59. package/src/backend/routes/activitypub/inbox-types.ts +14 -2
  60. package/src/backend/routes/activitypub/inbox.ts +81 -105
  61. package/src/backend/routes/activitypub/outbox.ts +0 -0
  62. package/src/backend/routes/activitypub.ts +6 -5
  63. package/src/backend/routes/actors-helpers.ts +34 -8
  64. package/src/backend/routes/actors.ts +328 -164
  65. package/src/backend/routes/auth-helpers.ts +57 -9
  66. package/src/backend/routes/auth.ts +10 -2
  67. package/src/backend/routes/communities/membership-invites.ts +4 -1
  68. package/src/backend/routes/communities/membership-members.ts +201 -73
  69. package/src/backend/routes/communities/membership-requests.ts +141 -76
  70. package/src/backend/routes/communities/membership-shared.ts +228 -21
  71. package/src/backend/routes/communities/messages.ts +9 -2
  72. package/src/backend/routes/communities/routes.ts +48 -13
  73. package/src/backend/routes/dm/contacts.ts +29 -41
  74. package/src/backend/routes/dm/conversations-helpers.ts +9 -1
  75. package/src/backend/routes/dm/messages.ts +36 -42
  76. package/src/backend/routes/dm/read-archive.ts +4 -2
  77. package/src/backend/routes/dm/requests.ts +62 -54
  78. package/src/backend/routes/follow-helpers.ts +200 -60
  79. package/src/backend/routes/follow.ts +146 -140
  80. package/src/backend/routes/media.ts +20 -94
  81. package/src/backend/routes/moderation.ts +72 -4
  82. package/src/backend/routes/notes.ts +3 -4
  83. package/src/backend/routes/notifications.ts +209 -108
  84. package/src/backend/routes/posts/delete-cascade.ts +253 -89
  85. package/src/backend/routes/posts/federation.ts +373 -0
  86. package/src/backend/routes/posts/interactions.ts +160 -184
  87. package/src/backend/routes/posts/like-mutation.ts +240 -0
  88. package/src/backend/routes/posts/post-helpers.ts +112 -162
  89. package/src/backend/routes/posts/queries.ts +150 -54
  90. package/src/backend/routes/posts/routes.ts +169 -329
  91. package/src/backend/routes/posts/transformers.ts +61 -6
  92. package/src/backend/routes/recommendations.ts +37 -44
  93. package/src/backend/routes/rtc/index.ts +26 -6
  94. package/src/backend/routes/search.ts +39 -55
  95. package/src/backend/routes/stories/interactions.ts +22 -6
  96. package/src/backend/routes/stories/query-helpers.ts +28 -41
  97. package/src/backend/routes/stories/routes.ts +125 -108
  98. package/src/backend/routes/takos-tools/dm.ts +17 -17
  99. package/src/backend/routes/takos-tools/posts.ts +189 -106
  100. package/src/backend/routes/takos-tools/search.ts +13 -4
  101. package/src/backend/routes/takos-tools/timeline.ts +35 -27
  102. package/src/backend/routes/takos-tools-response.ts +6 -2
  103. package/src/backend/routes/timeline.ts +5 -5
  104. package/src/backend/runtime/call-signaling-do.ts +35 -4
  105. package/src/backend/runtime/one-time-ticket.ts +116 -0
  106. package/src/backend/runtime/realtime-stream-do.ts +5 -61
  107. package/src/backend/runtime/signaling-hub.ts +36 -3
  108. package/src/backend/server.ts +95 -80
  109. package/src/db/d1-write.ts +67 -43
  110. package/src/db/schema/federation.ts +42 -1
  111. package/src/db/schema/messaging.ts +110 -2
  112. package/src/db/schema.ts +1 -1
@@ -8,11 +8,17 @@
8
8
 
9
9
  import { and, eq } from "drizzle-orm";
10
10
  import type { Database } from "../../db/index.ts";
11
- import { blocks, follows } from "../../db/index.ts";
11
+ import { follows, objectRecipients } from "../../db/index.ts";
12
12
  import { canViewerReadObject } from "./community-visibility.ts";
13
13
  import { safeJsonParse } from "../federation-helpers.ts";
14
+ import {
15
+ personalActorIsBlockedBy,
16
+ personalActorIsSuppressedBy,
17
+ } from "./personal-actor-moderation.ts";
18
+ import { isActorBlockedStrict } from "./blocklist.ts";
14
19
 
15
20
  export type ReadGateObject = {
21
+ apId?: string;
16
22
  visibility: string;
17
23
  attributedTo: string;
18
24
  toJson: string;
@@ -65,6 +71,26 @@ async function hasAcceptedFollow(
65
71
  return Boolean(row);
66
72
  }
67
73
 
74
+ /** Resolve a private bto/bcc recipient without exposing that identity in JSON. */
75
+ async function hasProjectedRecipient(
76
+ db: Database,
77
+ objectApId: string,
78
+ viewerApId: string,
79
+ ): Promise<boolean> {
80
+ const row = await db
81
+ .select({ objectApId: objectRecipients.objectApId })
82
+ .from(objectRecipients)
83
+ .where(
84
+ and(
85
+ eq(objectRecipients.objectApId, objectApId),
86
+ eq(objectRecipients.recipientApId, viewerApId),
87
+ eq(objectRecipients.type, "to"),
88
+ ),
89
+ )
90
+ .get();
91
+ return Boolean(row);
92
+ }
93
+
68
94
  /**
69
95
  * Per-post visibility decision EXCLUDING the private-community membership gate.
70
96
  * This is the single source of truth for the public / unlisted / followers /
@@ -83,6 +109,7 @@ export function passesPostVisibilitySync(
83
109
  obj: ReadGateObject,
84
110
  viewerApId: string | null | undefined,
85
111
  isAcceptedFollower: (authorApId: string) => boolean,
112
+ isProjectedRecipient: (objectApId: string) => boolean = () => false,
86
113
  now: string = new Date().toISOString(),
87
114
  ): boolean {
88
115
  // A Story's stored visibility ("public") does NOT encode its reach: a personal
@@ -99,17 +126,21 @@ export function passesPostVisibilitySync(
99
126
  if (obj.visibility === "direct") {
100
127
  if (!viewerApId) return false;
101
128
  if (obj.attributedTo === viewerApId) return true;
102
- return isExplicitRecipient(obj, viewerApId);
129
+ return (
130
+ isExplicitRecipient(obj, viewerApId) ||
131
+ (!!obj.apId && isProjectedRecipient(obj.apId))
132
+ );
103
133
  }
104
134
 
105
135
  if (obj.visibility === "followers") {
106
136
  if (!viewerApId) return false;
107
137
  if (obj.attributedTo === viewerApId) return true;
108
138
  if (isExplicitRecipient(obj, viewerApId)) return true;
139
+ if (obj.apId && isProjectedRecipient(obj.apId)) return true;
109
140
  return isAcceptedFollower(obj.attributedTo);
110
141
  }
111
142
 
112
- return true; // public / unlisted
143
+ return obj.visibility === "public" || obj.visibility === "unlisted";
113
144
  }
114
145
 
115
146
  /**
@@ -130,6 +161,13 @@ export async function canViewerReadObjectFull(
130
161
  ): Promise<boolean> {
131
162
  const now = new Date().toISOString();
132
163
 
164
+ // Instance moderation is a read boundary, not only an ingest/delivery
165
+ // boundary. A failed or partial purge intentionally leaves retained objects
166
+ // for retry/reversible unblock; while their author is operator-blocked those
167
+ // rows must be inert through every single-object reader and interaction that
168
+ // consumes this canonical gate.
169
+ if (await isActorBlockedStrict(db, obj.attributedTo)) return false;
170
+
133
171
  // Story author + expiry shortcuts need no community/follow query.
134
172
  if (obj.type === "Story") {
135
173
  if (viewerApId && obj.attributedTo === viewerApId) return true;
@@ -150,18 +188,36 @@ export async function canViewerReadObjectFull(
150
188
  // Resolve the single accepted-follow edge only when a follower-gated branch
151
189
  // actually needs it (personal story, or a followers-only post with no explicit
152
190
  // to/cc recipient), then defer to the shared per-post predicate.
191
+ const needsProjection =
192
+ !!viewerApId &&
193
+ !!obj.apId &&
194
+ obj.attributedTo !== viewerApId &&
195
+ obj.type !== "Story" &&
196
+ (obj.visibility === "direct" || obj.visibility === "followers") &&
197
+ !isExplicitRecipient(obj, viewerApId);
198
+ const projectedRecipient = needsProjection
199
+ ? await hasProjectedRecipient(db, obj.apId!, viewerApId!)
200
+ : false;
201
+
153
202
  const needsFollow =
154
203
  !!viewerApId &&
155
204
  obj.attributedTo !== viewerApId &&
156
205
  ((obj.type === "Story" && !obj.communityApId) ||
157
206
  (obj.type !== "Story" &&
158
207
  obj.visibility === "followers" &&
159
- !isExplicitRecipient(obj, viewerApId)));
208
+ !isExplicitRecipient(obj, viewerApId) &&
209
+ !projectedRecipient));
160
210
  const following = needsFollow
161
211
  ? await hasAcceptedFollow(db, viewerApId, obj.attributedTo)
162
212
  : false;
163
213
 
164
- return passesPostVisibilitySync(obj, viewerApId, () => following, now);
214
+ return passesPostVisibilitySync(
215
+ obj,
216
+ viewerApId,
217
+ () => following,
218
+ () => projectedRecipient,
219
+ now,
220
+ );
165
221
  }
166
222
 
167
223
  /**
@@ -176,15 +232,22 @@ export async function actorIsBlockedBy(
176
232
  targetApId: string,
177
233
  actorApId: string,
178
234
  ): Promise<boolean> {
179
- const row = await db
180
- .select({ blockerApId: blocks.blockerApId })
181
- .from(blocks)
182
- .where(
183
- and(
184
- eq(blocks.blockerApId, targetApId),
185
- eq(blocks.blockedApId, actorApId),
186
- ),
187
- )
188
- .get();
189
- return Boolean(row);
235
+ return personalActorIsBlockedBy(db, targetApId, actorApId);
236
+ }
237
+
238
+ /**
239
+ * True if `targetApId` has blocked OR muted `actorApId` and therefore does not
240
+ * want an inbound write from that actor reflected on a locally-owned object.
241
+ * Keep this separate from `actorIsBlockedBy`: block remains the existence-
242
+ * hiding authorization rule used by reads and direct local API responses,
243
+ * while mute is an owner-selected write-suppression rule for federated
244
+ * interactions and other targeted inbound writes such as Like, Announce,
245
+ * Follow, Reply, DM, and Mention.
246
+ */
247
+ export async function actorSuppressesInteractionFrom(
248
+ db: Database,
249
+ targetApId: string,
250
+ actorApId: string,
251
+ ): Promise<boolean> {
252
+ return personalActorIsSuppressedBy(db, targetApId, actorApId);
190
253
  }
@@ -0,0 +1,61 @@
1
+ import { isLocal } from "./ap-ids.ts";
2
+ import { isSafeRemoteUrl } from "./ssrf.ts";
3
+
4
+ export const MAX_REMOTE_ACTIVITY_ID_LENGTH = 2_048;
5
+
6
+ /** A bounded HTTP(S) Activity IRI suitable for an indexed/local lookup. */
7
+ export function isBoundedHttpActivityId(value: string): boolean {
8
+ if (
9
+ value.length === 0 ||
10
+ value.length > MAX_REMOTE_ACTIVITY_ID_LENGTH ||
11
+ value.trim() !== value ||
12
+ /[\u0000-\u001f\u007f]/u.test(value)
13
+ ) {
14
+ return false;
15
+ }
16
+ try {
17
+ const parsed = new URL(value);
18
+ return (
19
+ (parsed.protocol === "http:" || parsed.protocol === "https:") &&
20
+ !parsed.username &&
21
+ !parsed.password
22
+ );
23
+ } catch {
24
+ return false;
25
+ }
26
+ }
27
+
28
+ /**
29
+ * Return whether a peer-controlled Activity id is safe to reuse in an
30
+ * outbound protocol reference.
31
+ *
32
+ * HTTP-signature verification binds the envelope actor, not an arbitrary id.
33
+ * A reusable id must therefore be a bounded, credential-free HTTP(S) URL on
34
+ * that actor's exact URL origin. Comparing only `host` is insufficient: it
35
+ * treats an HTTPS downgrade, a non-HTTP scheme, and credential-bearing URL
36
+ * spellings as if the actor had authority over them.
37
+ */
38
+ export function isTrustedRemoteActivityId(
39
+ value: string,
40
+ actorApId: string,
41
+ localBaseUrl?: string,
42
+ ): boolean {
43
+ if (
44
+ !isBoundedHttpActivityId(value) ||
45
+ actorApId.length === 0 ||
46
+ actorApId.length > MAX_REMOTE_ACTIVITY_ID_LENGTH ||
47
+ actorApId.trim() !== actorApId ||
48
+ /[\u0000-\u001f\u007f]/u.test(actorApId) ||
49
+ !isSafeRemoteUrl(value) ||
50
+ !isSafeRemoteUrl(actorApId)
51
+ ) {
52
+ return false;
53
+ }
54
+ if (localBaseUrl && isLocal(value, localBaseUrl)) return false;
55
+
56
+ try {
57
+ return new URL(value).origin === new URL(actorApId).origin;
58
+ } catch {
59
+ return false;
60
+ }
61
+ }
@@ -25,6 +25,7 @@ import {
25
25
  type Database,
26
26
  } from "../../db/index.ts";
27
27
  import { notificationEligibilityWhere } from "./notification-eligibility.ts";
28
+ import { operatorActorNotBlockedSql } from "./blocklist.ts";
28
29
 
29
30
  export interface YurumeUnreadCounts {
30
31
  readonly dm: number;
@@ -50,6 +51,7 @@ export async function yurumeUnreadCounts(
50
51
  AND o.type = 'Note'
51
52
  AND o.conversation IS NOT NULL
52
53
  AND o.attributed_to != ${actorApId}
54
+ AND ${operatorActorNotBlockedSql(sql.raw("o.attributed_to"))}
53
55
  AND o.published > COALESCE(r.last_read_at, '1970-01-01T00:00:00Z')
54
56
  AND o.conversation NOT IN (
55
57
  SELECT conversation_id FROM dm_archived_conversations
@@ -72,6 +74,7 @@ export async function yurumeUnreadCounts(
72
74
  ON r.community_ap_id = cm.community_ap_id
73
75
  AND r.actor_ap_id = ${actorApId}
74
76
  WHERE cm.actor_ap_id = ${actorApId}
77
+ AND ${operatorActorNotBlockedSql(sql.raw("o.attributed_to"))}
75
78
  AND o.published > COALESCE(
76
79
  r.last_read_at,
77
80
  cm.joined_at,
@@ -1,14 +1,28 @@
1
1
  import type { Env } from "./types.ts";
2
2
  import { enqueuePendingNotificationPushJobs } from "./lib/notification-push.ts";
3
+ import { enqueuePendingDeliveryFanoutJobs } from "./lib/delivery/fanout-outbox.ts";
4
+ import { enqueuePendingDeliveryEndpointJobs } from "./lib/delivery/queue.ts";
5
+ import { enqueuePendingDeliveryResolutionJobs } from "./lib/delivery/resolution-outbox.ts";
3
6
  import { reapDrainedTombstones } from "./routes/actors.ts";
4
7
  import { cleanupExpiredStories } from "./routes/stories/query-helpers.ts";
8
+ import { reapRemoteActorFetchFailures } from "./lib/activitypub-actor-cache.ts";
5
9
 
6
10
  export type YurucommuRetentionStep =
7
- "expired_stories" | "drained_tombstones" | "notification_push";
11
+ | "expired_stories"
12
+ | "drained_tombstones"
13
+ | "remote_actor_fetch_failures"
14
+ | "delivery_fanout"
15
+ | "delivery_endpoint"
16
+ | "delivery_resolution"
17
+ | "notification_push";
8
18
 
9
19
  export interface YurucommuRetentionResult {
10
20
  readonly expiredStories: number;
11
21
  readonly reapedTombstones: number;
22
+ readonly reapedRemoteActorFetchFailures: number;
23
+ readonly enqueuedDeliveryFanoutJobs: number;
24
+ readonly enqueuedDeliveryEndpointJobs: number;
25
+ readonly enqueuedDeliveryResolutionJobs: number;
12
26
  readonly enqueuedNotificationPushJobs: number;
13
27
  }
14
28
 
@@ -34,6 +48,9 @@ export class YurucommuRetentionError extends Error {
34
48
  * handling:
35
49
  * - Story expiry uses the canonical object cascade and purges its media last.
36
50
  * - Tombstones remain until every Delete delivery has drained.
51
+ * - Federation delivery republishes pending follower/community fanout,
52
+ * due endpoint jobs, and unresolved-recipient outbox rows whose first Queue
53
+ * RPC or owning worker was lost.
37
54
  * - Notification push performs bounded pusher/job retention, stale-job
38
55
  * recovery, and enqueues due durable outbox rows when a queue is available.
39
56
  *
@@ -54,6 +71,22 @@ export async function runYurucommuRetention(
54
71
  const reapedTombstones = await retentionStep("drained_tombstones", () =>
55
72
  reapDrainedTombstones(env.DB_INSTANCE),
56
73
  );
74
+ const reapedRemoteActorFetchFailures = await retentionStep(
75
+ "remote_actor_fetch_failures",
76
+ () => reapRemoteActorFetchFailures(env.DB_INSTANCE),
77
+ );
78
+ const enqueuedDeliveryFanoutJobs = await retentionStep(
79
+ "delivery_fanout",
80
+ () => enqueuePendingDeliveryFanoutJobs(env),
81
+ );
82
+ const enqueuedDeliveryEndpointJobs = await retentionStep(
83
+ "delivery_endpoint",
84
+ () => enqueuePendingDeliveryEndpointJobs(env),
85
+ );
86
+ const enqueuedDeliveryResolutionJobs = await retentionStep(
87
+ "delivery_resolution",
88
+ () => enqueuePendingDeliveryResolutionJobs(env),
89
+ );
57
90
  const enqueuedNotificationPushJobs = await retentionStep(
58
91
  "notification_push",
59
92
  () => enqueuePendingNotificationPushJobs(env),
@@ -62,6 +95,10 @@ export async function runYurucommuRetention(
62
95
  return {
63
96
  expiredStories,
64
97
  reapedTombstones,
98
+ reapedRemoteActorFetchFailures,
99
+ enqueuedDeliveryFanoutJobs,
100
+ enqueuedDeliveryEndpointJobs,
101
+ enqueuedDeliveryResolutionJobs,
65
102
  enqueuedNotificationPushJobs,
66
103
  };
67
104
  }